From f32deba38fec76178fbb4130786d556b9a77171c Mon Sep 17 00:00:00 2001
From: sneak <sneak@sneak.berlin>
Date: Thu, 9 Apr 2020 00:38:19 -0700
Subject: [PATCH]  should backoff reasonably now

---
 database/imconnector.go |  6 ++++--
 database/model.go       |  1 +
 instance/instance.go    | 48 +++++++++++++++++++++++++++++------------
 3 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/database/imconnector.go b/database/imconnector.go
index e9af486..fab76f8 100644
--- a/database/imconnector.go
+++ b/database/imconnector.go
@@ -2,9 +2,8 @@ package database
 
 import (
 	"git.eeqj.de/sneak/feta/instance"
-	"github.com/rs/zerolog/log"
-
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Manager) SaveInstance(i *instance.Instance) error {
@@ -20,6 +19,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			UUID:                       i.UUID,
 			Disabled:                   i.Disabled,
 			ErrorCount:                 i.ErrorCount,
+			ConsecutiveErrorCount:      i.ConsecutiveErrorCount,
 			FSMState:                   i.Status(),
 			Fetching:                   i.Fetching,
 			HighestID:                  i.HighestID,
@@ -46,6 +46,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		m.db.Where("UUID = ?", i.UUID).First(&ei)
 		ei.Disabled = i.Disabled
 		ei.ErrorCount = i.ErrorCount
+		ei.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 		ei.FSMState = i.Status()
 		ei.Fetching = i.Fetching
 		ei.HighestID = i.HighestID
@@ -74,6 +75,7 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
 			x.UUID = i.UUID
 			x.Disabled = i.Disabled
 			x.ErrorCount = i.ErrorCount
+			x.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 			x.InitialFSMState = i.FSMState
 			x.Fetching = i.Fetching
 			x.HighestID = i.HighestID
diff --git a/database/model.go b/database/model.go
index 93c942d..23cdee2 100644
--- a/database/model.go
+++ b/database/model.go
@@ -27,6 +27,7 @@ type StoredToot struct {
 type APInstance struct {
 	gorm.Model
 	UUID                       uuid.UUID `gorm:"type:uuid;primary_key;"`
+	ConsecutiveErrorCount      uint
 	ErrorCount                 uint
 	SuccessCount               uint
 	HighestID                  uint
diff --git a/instance/instance.go b/instance/instance.go
index fd6eed6..6183b9f 100644
--- a/instance/instance.go
+++ b/instance/instance.go
@@ -17,18 +17,19 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-//import "github.com/gin-gonic/gin"
-
 const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
-const instanceNodeinfoTimeout = time.Second * 50
-const instanceHTTPTimeout = time.Second * 120
-const instanceSpiderInterval = time.Second * 120
-const instanceErrorInterval = time.Second * 60 * 30
+const instanceNodeinfoTimeout = time.Second * 60 * 2        // 2m
+const instanceHTTPTimeout = time.Second * 60 * 2            // 2m
+const instanceSpiderInterval = time.Second * 60 * 2         // 2m
+const instanceErrorInterval = time.Second * 60 * 60         // 1h
+const instancePersistentErrorInterval = time.Second * 86400 // 1d
+const zeroInterval = time.Second * 0                        // 0s
 
 // Instance stores all the information we know about an instance
 type Instance struct {
 	Disabled                   bool
 	ErrorCount                 uint
+	ConsecutiveErrorCount      uint
 	FSM                        *fsm.FSM
 	Fetching                   bool
 	HighestID                  uint
@@ -121,10 +122,26 @@ func (i *Instance) Unlock() {
 	i.structLock.Unlock()
 }
 
-func (i *Instance) bumpFetch() {
+func (i *Instance) bumpFetchError() {
 	i.Lock()
-	defer i.Unlock()
-	i.NextFetch = time.Now().Add(120 * time.Second)
+	probablyDead := i.ConsecutiveErrorCount > 3
+	i.Unlock()
+
+	if probablyDead {
+		// if three consecutive fetch errors happen, only try once per day:
+		i.setNextFetchAfter(instancePersistentErrorInterval)
+	} else {
+		// otherwise give them 1h
+		i.setNextFetchAfter(instanceErrorInterval)
+	}
+}
+
+func (i *Instance) bumpFetchSuccess() {
+	i.setNextFetchAfter(instanceSpiderInterval)
+}
+
+func (i *Instance) scheduleFetchImmediate() {
+	i.setNextFetchAfter(zeroInterval)
 }
 
 func (i *Instance) setNextFetchAfter(d time.Duration) {
@@ -139,8 +156,7 @@ func (i *Instance) Fetch() {
 	i.fetchingLock.Lock()
 	defer i.fetchingLock.Unlock()
 
-	i.setNextFetchAfter(instanceErrorInterval)
-
+	i.bumpFetchError()
 	err := i.DetectNodeTypeIfNecessary()
 	if err != nil {
 		log.Debug().
@@ -149,8 +165,7 @@ func (i *Instance) Fetch() {
 			Msg("unable to fetch instance metadata")
 		return
 	}
-
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.scheduleFetchImmediate()
 	log.Info().
 		Str("hostname", i.Hostname).
 		Msg("instance now ready for fetch")
@@ -207,12 +222,14 @@ func (i *Instance) registerError() {
 	i.Lock()
 	defer i.Unlock()
 	i.ErrorCount++
+	i.ConsecutiveErrorCount++
 }
 
 func (i *Instance) registerSuccess() {
 	i.Lock()
 	defer i.Unlock()
 	i.SuccessCount++
+	i.ConsecutiveErrorCount = 0
 }
 
 // Up returns true if the success count is >0
@@ -405,9 +422,12 @@ func (i *Instance) fetchRecentToots() error {
 
 	// it turns out pleroma supports the mastodon api so we'll just use that
 	// for everything for now
+
+	// FIXME would be nice to support non-https
 	url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true",
 		i.Hostname)
 
+	// FIXME support broken/expired certs
 	var c = &http.Client{
 		Timeout: instanceHTTPTimeout,
 	}
@@ -461,7 +481,7 @@ func (i *Instance) fetchRecentToots() error {
 		Msgf("got and parsed toots")
 	i.registerSuccess()
 	i.Event("TOOTS_FETCHED")
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.bumpFetchSuccess()
 
 	// this should go fast as either the channel is buffered bigly or the
 	// ingester receives fast and does its own buffering, but run it in its