should backoff reasonably now

instance/instance.go

@@ -17,18 +17,19 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-//import "github.com/gin-gonic/gin"
-
 const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
-const instanceNodeinfoTimeout = time.Second * 50
-const instanceHTTPTimeout = time.Second * 120
-const instanceSpiderInterval = time.Second * 120
-const instanceErrorInterval = time.Second * 60 * 30
+const instanceNodeinfoTimeout = time.Second * 60 * 2        // 2m
+const instanceHTTPTimeout = time.Second * 60 * 2            // 2m
+const instanceSpiderInterval = time.Second * 60 * 2         // 2m
+const instanceErrorInterval = time.Second * 60 * 60         // 1h
+const instancePersistentErrorInterval = time.Second * 86400 // 1d
+const zeroInterval = time.Second * 0                        // 0s
 
 // Instance stores all the information we know about an instance
 type Instance struct {
 	Disabled                   bool
 	ErrorCount                 uint
+	ConsecutiveErrorCount      uint
 	FSM                        *fsm.FSM
 	Fetching                   bool
 	HighestID                  uint
@@ -121,10 +122,26 @@ func (i *Instance) Unlock() {
 	i.structLock.Unlock()
 }
 
-func (i *Instance) bumpFetch() {
+func (i *Instance) bumpFetchError() {
 	i.Lock()
-	defer i.Unlock()
-	i.NextFetch = time.Now().Add(120 * time.Second)
+	probablyDead := i.ConsecutiveErrorCount > 3
+	i.Unlock()
+
+	if probablyDead {
+		// if three consecutive fetch errors happen, only try once per day:
+		i.setNextFetchAfter(instancePersistentErrorInterval)
+	} else {
+		// otherwise give them 1h
+		i.setNextFetchAfter(instanceErrorInterval)
+	}
+}
+
+func (i *Instance) bumpFetchSuccess() {
+	i.setNextFetchAfter(instanceSpiderInterval)
+}
+
+func (i *Instance) scheduleFetchImmediate() {
+	i.setNextFetchAfter(zeroInterval)
 }
 
 func (i *Instance) setNextFetchAfter(d time.Duration) {
@@ -139,8 +156,7 @@ func (i *Instance) Fetch() {
 	i.fetchingLock.Lock()
 	defer i.fetchingLock.Unlock()
 
-	i.setNextFetchAfter(instanceErrorInterval)
-
+	i.bumpFetchError()
 	err := i.DetectNodeTypeIfNecessary()
 	if err != nil {
 		log.Debug().
@@ -149,8 +165,7 @@ func (i *Instance) Fetch() {
 			Msg("unable to fetch instance metadata")
 		return
 	}
-
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.scheduleFetchImmediate()
 	log.Info().
 		Str("hostname", i.Hostname).
 		Msg("instance now ready for fetch")
@@ -207,12 +222,14 @@ func (i *Instance) registerError() {
 	i.Lock()
 	defer i.Unlock()
 	i.ErrorCount++
+	i.ConsecutiveErrorCount++
 }
 
 func (i *Instance) registerSuccess() {
 	i.Lock()
 	defer i.Unlock()
 	i.SuccessCount++
+	i.ConsecutiveErrorCount = 0
 }
 
 // Up returns true if the success count is >0
@@ -405,9 +422,12 @@ func (i *Instance) fetchRecentToots() error {
 
 	// it turns out pleroma supports the mastodon api so we'll just use that
 	// for everything for now
+
+	// FIXME would be nice to support non-https
 	url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true",
 		i.Hostname)
 
+	// FIXME support broken/expired certs
 	var c = &http.Client{
 		Timeout: instanceHTTPTimeout,
 	}
@@ -461,7 +481,7 @@ func (i *Instance) fetchRecentToots() error {
 		Msgf("got and parsed toots")
 	i.registerSuccess()
 	i.Event("TOOTS_FETCHED")
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.bumpFetchSuccess()
 
 	// this should go fast as either the channel is buffered bigly or the
 	// ingester receives fast and does its own buffering, but run it in its