add /instances route

* permdisable instance on 6 failures
* restore in proper state on db load

.gitignore

@@ -3,3 +3,4 @@ output/
 feta.sqlite
 .lintsetup
 out
+debug.log

Makefile

@@ -26,7 +26,7 @@ endif
 default: build
 
 debug: build
-	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN)
+	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log
 
 run: build
 	./$(FN)

database/imconnector.go

@@ -2,9 +2,8 @@ package database
 
 import (
 	"git.eeqj.de/sneak/feta/instance"
-	"github.com/rs/zerolog/log"
-
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Manager) SaveInstance(i *instance.Instance) error {
@@ -20,6 +19,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			UUID:                       i.UUID,
 			Disabled:                   i.Disabled,
 			ErrorCount:                 i.ErrorCount,
+			ConsecutiveErrorCount:      i.ConsecutiveErrorCount,
 			FSMState:                   i.Status(),
 			Fetching:                   i.Fetching,
 			HighestID:                  i.HighestID,
@@ -46,6 +46,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		m.db.Where("UUID = ?", i.UUID).First(&ei)
 		ei.Disabled = i.Disabled
 		ei.ErrorCount = i.ErrorCount
+		ei.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 		ei.FSMState = i.Status()
 		ei.Fetching = i.Fetching
 		ei.HighestID = i.HighestID
@@ -74,6 +75,7 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
 			x.UUID = i.UUID
 			x.Disabled = i.Disabled
 			x.ErrorCount = i.ErrorCount
+			x.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 			x.InitialFSMState = i.FSMState
 			x.Fetching = i.Fetching
 			x.HighestID = i.HighestID

database/model.go

@@ -27,6 +27,7 @@ type StoredToot struct {
 type APInstance struct {
 	gorm.Model
 	UUID                       uuid.UUID `gorm:"type:uuid;primary_key;"`
+	ConsecutiveErrorCount      uint
 	ErrorCount                 uint
 	SuccessCount               uint
 	HighestID                  uint

database/readshortcuts.go

@@ -14,6 +14,16 @@ func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
 	}
 }
 
+func (m *Manager) TotalTootCount() (uint, error) {
+	var c uint
+	e := m.db.Model(&StoredToot{}).Count(&c)
+	if e.Error != nil {
+		return 0, e.Error
+	} else {
+		return c, nil
+	}
+}
+
 func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
 	var i APInstance
 	e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)

instance/instance.go

@@ -17,18 +17,19 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-//import "github.com/gin-gonic/gin"
-
 const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
-const instanceNodeinfoTimeout = time.Second * 50
+const instanceNodeinfoTimeout = time.Second * 60 * 2        // 2m
-const instanceHTTPTimeout = time.Second * 120
+const instanceHTTPTimeout = time.Second * 60 * 2            // 2m
-const instanceSpiderInterval = time.Second * 120
+const instanceSpiderInterval = time.Second * 60 * 2         // 2m
-const instanceErrorInterval = time.Second * 60 * 30
+const instanceErrorInterval = time.Second * 60 * 60         // 1h
+const instancePersistentErrorInterval = time.Second * 86400 // 1d
+const zeroInterval = time.Second * 0                        // 0s
 
 // Instance stores all the information we know about an instance
 type Instance struct {
 	Disabled                   bool
 	ErrorCount                 uint
+	ConsecutiveErrorCount      uint
 	FSM                        *fsm.FSM
 	Fetching                   bool
 	HighestID                  uint
@@ -61,6 +62,10 @@ func New(options ...func(i *Instance)) *Instance {
 		opt(i)
 	}
 
+	if i.InitialFSMState == "FETCHING" {
+		i.InitialFSMState = "READY_FOR_TOOTFETCH"
+	}
+
 	i.FSM = fsm.NewFSM(
 		i.InitialFSMState,
 		fsm.Events{
@@ -74,11 +79,13 @@ func New(options ...func(i *Instance)) *Instance {
 			{Name: "EARLY_FETCH_ERROR", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "EARLY_ERROR"},
 			{Name: "TOOT_FETCH_ERROR", Src: []string{"FETCHING"}, Dst: "TOOT_FETCH_ERROR"},
 			{Name: "TOOTS_FETCHED", Src: []string{"FETCHING"}, Dst: "READY_FOR_TOOTFETCH"},
+			{Name: "DISABLEMENT", Src: []string{"WEIRD_NODE", "EARLY_ERROR", "TOOT_FETCH_ERROR"}, Dst: "DISABLED"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) { i.fsmEnterState(e) },
 		},
 	)
+
 	return i
 }
 
@@ -121,10 +128,36 @@ func (i *Instance) Unlock() {
 	i.structLock.Unlock()
 }
 
-func (i *Instance) bumpFetch() {
+func (i *Instance) bumpFetchError() {
 	i.Lock()
-	defer i.Unlock()
+	probablyDead := i.ConsecutiveErrorCount > 3
-	i.NextFetch = time.Now().Add(120 * time.Second)
+	shouldDisable := i.ConsecutiveErrorCount > 6
+	i.Unlock()
+
+	if shouldDisable {
+		// auf wiedersehen, felicia
+		i.Lock()
+		i.Disabled = true
+		i.Unlock()
+		i.Event("DISABLEMENT")
+		return
+	}
+
+	if probablyDead {
+		// if three consecutive fetch errors happen, only try once per day:
+		i.setNextFetchAfter(instancePersistentErrorInterval)
+	} else {
+		// otherwise give them 1h
+		i.setNextFetchAfter(instanceErrorInterval)
+	}
+}
+
+func (i *Instance) bumpFetchSuccess() {
+	i.setNextFetchAfter(instanceSpiderInterval)
+}
+
+func (i *Instance) scheduleFetchImmediate() {
+	i.setNextFetchAfter(zeroInterval)
 }
 
 func (i *Instance) setNextFetchAfter(d time.Duration) {
@@ -139,8 +172,7 @@ func (i *Instance) Fetch() {
 	i.fetchingLock.Lock()
 	defer i.fetchingLock.Unlock()
 
-	i.setNextFetchAfter(instanceErrorInterval)
+	i.bumpFetchError()
-
 	err := i.DetectNodeTypeIfNecessary()
 	if err != nil {
 		log.Debug().
@@ -149,8 +181,7 @@ func (i *Instance) Fetch() {
 			Msg("unable to fetch instance metadata")
 		return
 	}
-
+	i.scheduleFetchImmediate()
-	i.setNextFetchAfter(instanceSpiderInterval)
 	log.Info().
 		Str("hostname", i.Hostname).
 		Msg("instance now ready for fetch")
@@ -207,12 +238,14 @@ func (i *Instance) registerError() {
 	i.Lock()
 	defer i.Unlock()
 	i.ErrorCount++
+	i.ConsecutiveErrorCount++
 }
 
 func (i *Instance) registerSuccess() {
 	i.Lock()
 	defer i.Unlock()
 	i.SuccessCount++
+	i.ConsecutiveErrorCount = 0
 }
 
 // Up returns true if the success count is >0
@@ -405,9 +438,12 @@ func (i *Instance) fetchRecentToots() error {
 
 	// it turns out pleroma supports the mastodon api so we'll just use that
 	// for everything for now
+
+	// FIXME would be nice to support non-https
 	url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true",
 		i.Hostname)
 
+	// FIXME support broken/expired certs
 	var c = &http.Client{
 		Timeout: instanceHTTPTimeout,
 	}
@@ -461,7 +497,7 @@ func (i *Instance) fetchRecentToots() error {
 		Msgf("got and parsed toots")
 	i.registerSuccess()
 	i.Event("TOOTS_FETCHED")
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.bumpFetchSuccess()
 
 	// this should go fast as either the channel is buffered bigly or the
 	// ingester receives fast and does its own buffering, but run it in its

process/handlers.go

@@ -9,7 +9,6 @@ import (
 
 	u "git.eeqj.de/sneak/goutil"
 	"github.com/flosch/pongo2"
-	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 	"github.com/labstack/echo"
 )
@@ -23,7 +22,12 @@ func (a *Server) instances() []hash {
 		i := make(hash)
 		// TODO move this locking onto a method on Instance that just
 		// returns a new hash
-		// FIXME figure out why a very short lock here deadlocks
+
+		//this only locks the FSM, not the whole instance struct
+		i["status"] = v.Status()
+
+		// now do a quick lock of the whole instance just to copy out the
+		// attrs
 		v.Lock()
 		i["hostname"] = v.Hostname
 		i["uuid"] = v.UUID.String()
@@ -31,9 +35,8 @@ func (a *Server) instances() []hash {
 		i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
 		i["successCount"] = v.SuccessCount
 		i["errorCount"] = v.ErrorCount
+		i["consecutiveErrorCount"] = v.ConsecutiveErrorCount
 		i["identified"] = v.Identified
-		//this only locks the FSM, not the whole instance struct
-		i["status"] = v.Status()
 		i["software"] = "unknown"
 		i["version"] = "unknown"
 		if v.Identified {
@@ -41,6 +44,7 @@ func (a *Server) instances() []hash {
 			i["version"] = v.ServerVersionString
 		}
 		v.Unlock()
+
 		resp = append(resp, i)
 	}
 
@@ -54,7 +58,7 @@ func (a *Server) instances() []hash {
 	return resp
 }
 
-func (a *Server) instanceSummary() map[string]int {
+func (a *Server) instanceStatusSummary() map[string]int {
 	resp := make(map[string]int)
 	for _, v := range a.feta.manager.ListInstances() {
 		v.Lock()
@@ -68,26 +72,6 @@ func (a *Server) instanceSummary() map[string]int {
 	return resp
 }
 
-/*
-func (a *Server) getInstanceListHandler() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-
-		result := &gin.H{
-			"instances": a.instances(),
-		}
-
-		json, err := json.Marshal(result)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		w.Write(json)
-	}
-}
-*/
-
 func (a *Server) notFoundHandler(c echo.Context) error {
 	return c.String(http.StatusNotFound, "404 not found")
 }
@@ -117,17 +101,33 @@ func (a *Server) instanceHandler(c echo.Context) error {
 }
 
 func (a *Server) indexHandler(c echo.Context) error {
+	count, err := a.feta.dbm.TotalTootCount()
+	if err != nil {
+		count = 0
+	}
 	tc := pongo2.Context{
-		"time":      time.Now().UTC().Format(time.RFC3339Nano),
+		"time":                  time.Now().UTC().Format(time.RFC3339Nano),
-		"gitrev":    a.feta.version,
+		"gitrev":                a.feta.version,
-		"instances": a.instances(),
+		"tootCount":             count,
+		"instances":             a.instances(),
+		"instanceStatusSummary": a.instanceStatusSummary(),
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }
 
+func (a *Server) instanceListHandler(c echo.Context) error {
+	il := a.instances()
+	tc := pongo2.Context{
+		"time":      time.Now().UTC().Format(time.RFC3339Nano),
+		"gitrev":    a.feta.version,
+		"instances": il,
+	}
+	return c.Render(http.StatusOK, "instancelist.html", tc)
+}
+
 func (a *Server) statsHandler(c echo.Context) error {
-	index := &gin.H{
+	index := &hash{
-		"server": &gin.H{
+		"server": &hash{
 			"now":        time.Now().UTC().Format(time.RFC3339),
 			"uptime":     a.feta.uptime().String(),
 			"goroutines": runtime.NumGoroutine(),
@@ -135,14 +135,14 @@ func (a *Server) statsHandler(c echo.Context) error {
 			"version":    a.feta.version,
 			"buildarch":  a.feta.buildarch,
 		},
-		"instanceSummary": a.instanceSummary(),
+		"instanceStatusSummary": a.instanceStatusSummary(),
 	}
 
 	return c.JSONPretty(http.StatusOK, index, "    ")
 }
 
 func (a *Server) healthCheckHandler(c echo.Context) error {
-	resp := &gin.H{
+	resp := &hash{
 		"status": "ok",
 		"now":    time.Now().UTC().Format(time.RFC3339),
 		"uptime": a.feta.uptime().String(),

process/server.go

@@ -96,6 +96,7 @@ func (s *Server) initRouter() {
 	// Routes
 	s.e.GET("/", s.indexHandler)
 	s.e.GET("/instance/:uuid", s.instanceHandler)
+	s.e.GET("/instances", s.instanceListHandler)
 	s.e.GET("/stats.json", s.statsHandler)
 	s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
 	//a.e.GET("/about", s.aboutHandler)

view/index.html

@@ -4,34 +4,44 @@
 <div class="col-lg-12">
 
 
-    <h2>indexer stats</h2>
+<h2>feta overview</h2>
+
+
+<div class="card m-5">
+  <h5 class="card-header">Instances</h5>
+  <div class="card-body">
+      <h5 class="card-title">Tracking {{ instances | length }} instances
+          across the Fediverse.</h5>
+    <!-- 
+        <p class="card-text">
+        </p> -->
+
+    <a href="/instances" class="btn btn-primary">View Instance List</a>
+  </div>
+</div>
+
+<div class="card m-5">
+  <h5 class="card-header">Toots</h5>
+  <div class="card-body">
+      <h5 class="card-title">I have {{ tootCount }} toots
+          in my database.</h5>
+    <a href="/toots" class="btn btn-primary">View Latest Toots</a>
+  </div>
+</div>
+
+<div class="card m-5">
+  <h5 class="card-header">Recent Events</h5>
+  <div class="card-body">
+      <h5 class="card-title">Last n System Events</h5>
+
+        <p class="card-text"> Discovered instance toot1.example.com </p>
+        <p class="card-text"> Discovered instance toot2.example.com </p>
+        <p class="card-text"> Discovered instance toot3.example.com </p>
+        <p class="card-text"> Discovered instance toot4.example.com </p>
+
+  </div>
+</div>
 
-<table class="table table-striped table-hover">
-  <thead class="thead-dark">
-<tr>
-  <th scope="col">instance id</th>
-  <th scope="col">hostname</th>
-  <th scope="col">status</th>
-  <th scope="col">tootCount</th>
-  <th scope="col">Detail</th>
-</tr>
-</thead>
-<tbody>
-{% for instance in instances %}
-    <tr>
-        <td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
-        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
-        <td>{{instance.status}}</td>
-        <td>{{instance.tootCount}}</td>
-        <td><a
-                href="/instance/{{instance.uuid}}"
-                class="btn btn-info">
-                <i class="fab fa-mastodon"></i>
-            </button></td>
-    </tr>
-{% endfor %}
- </tbody>
-</table>
 
 </div>
 {% endblock %}

view/instance.html

@@ -0,0 +1,24 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+
+<h2>instance {{instance.hostname}}</h2>
+
+<div class="card m-5">
+  <div class="card-header">
+      <a href="/instance/{{instance.uuid}}">{{ instance.hostname }}</a>
+      ({{instance.tootCount}} toots)
+  </div>
+  <div class="card-body">
+      <h5 class="card-title">{{instance.status}}</h5>
+    <p class="card-text">First Stat</p>
+    <p class="card-text">Second Stat</p>
+    <p class="card-text">Third Stat</p>
+    <a href="https://{{instance.hostname}}" class="btn btn-primary">View Instance Website</a>
+  </div>
+</div>
+
+</div>
+{% endblock %}

view/instancelist.html

@@ -0,0 +1,36 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+<h2>instance list</h2>
+
+<table class="table table-striped table-hover">
+  <thead class="thead-dark">
+<tr>
+  <th scope="col">hostname</th>
+  <th scope="col">status</th>
+  <th scope="col">tootCount</th>
+  <th scope="col">nextFetch</th>
+  <th scope="col">Detail</th>
+</tr>
+</thead>
+<tbody>
+{% for instance in instances %}
+    <tr>
+        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
+        <td>{{instance.status}}</td>
+        <td>{{instance.tootCount}}</td>
+        <td>{{instance.nextFetch}}</td>
+        <td><a
+                href="/instance/{{instance.uuid}}"
+                class="btn btn-info">
+                <i class="fab fa-mastodon"></i>
+            </button></td>
+    </tr>
+{% endfor %}
+ </tbody>
+</table>
+
+</div>
+{% endblock %}