.gitignore

@@ -3,4 +3,3 @@ output/
 feta.sqlite
 .lintsetup
 out
-debug.log

Makefile

@@ -26,7 +26,7 @@ endif
 default: build
 
 debug: build
-	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log
+	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN)
 
 run: build
 	./$(FN)

database/imconnector.go

@@ -2,8 +2,9 @@ package database
 
 import (
 	"git.eeqj.de/sneak/feta/instance"
-	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
+
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
 )
 
 func (m *Manager) SaveInstance(i *instance.Instance) error {
@@ -19,7 +20,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			UUID:                       i.UUID,
 			Disabled:                   i.Disabled,
 			ErrorCount:                 i.ErrorCount,
-			ConsecutiveErrorCount:      i.ConsecutiveErrorCount,
 			FSMState:                   i.Status(),
 			Fetching:                   i.Fetching,
 			HighestID:                  i.HighestID,
@@ -46,7 +46,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		m.db.Where("UUID = ?", i.UUID).First(&ei)
 		ei.Disabled = i.Disabled
 		ei.ErrorCount = i.ErrorCount
-		ei.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 		ei.FSMState = i.Status()
 		ei.Fetching = i.Fetching
 		ei.HighestID = i.HighestID
@@ -75,7 +74,6 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
 			x.UUID = i.UUID
 			x.Disabled = i.Disabled
 			x.ErrorCount = i.ErrorCount
-			x.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 			x.InitialFSMState = i.FSMState
 			x.Fetching = i.Fetching
 			x.HighestID = i.HighestID

database/model.go

@@ -27,7 +27,6 @@ type StoredToot struct {
 type APInstance struct {
 	gorm.Model
 	UUID                       uuid.UUID `gorm:"type:uuid;primary_key;"`
-	ConsecutiveErrorCount      uint
 	ErrorCount                 uint
 	SuccessCount               uint
 	HighestID                  uint

database/readshortcuts.go

@@ -14,16 +14,6 @@ func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
 	}
 }
 
-func (m *Manager) TotalTootCount() (uint, error) {
-	var c uint
-	e := m.db.Model(&StoredToot{}).Count(&c)
-	if e.Error != nil {
-		return 0, e.Error
-	} else {
-		return c, nil
-	}
-}
-
 func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
 	var i APInstance
 	e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)

instance/instance.go

@@ -17,19 +17,18 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+//import "github.com/gin-gonic/gin"
+
 const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
-const instanceNodeinfoTimeout = time.Second * 60 * 2        // 2m
+const instanceNodeinfoTimeout = time.Second * 50
-const instanceHTTPTimeout = time.Second * 60 * 2            // 2m
+const instanceHTTPTimeout = time.Second * 120
-const instanceSpiderInterval = time.Second * 60 * 2         // 2m
+const instanceSpiderInterval = time.Second * 120
-const instanceErrorInterval = time.Second * 60 * 60         // 1h
+const instanceErrorInterval = time.Second * 60 * 30
-const instancePersistentErrorInterval = time.Second * 86400 // 1d
-const zeroInterval = time.Second * 0                        // 0s
 
 // Instance stores all the information we know about an instance
 type Instance struct {
 	Disabled                   bool
 	ErrorCount                 uint
-	ConsecutiveErrorCount      uint
 	FSM                        *fsm.FSM
 	Fetching                   bool
 	HighestID                  uint
@@ -62,10 +61,6 @@ func New(options ...func(i *Instance)) *Instance {
 		opt(i)
 	}
 
-	if i.InitialFSMState == "FETCHING" {
-		i.InitialFSMState = "READY_FOR_TOOTFETCH"
-	}
-
 	i.FSM = fsm.NewFSM(
 		i.InitialFSMState,
 		fsm.Events{
@@ -79,13 +74,11 @@ func New(options ...func(i *Instance)) *Instance {
 			{Name: "EARLY_FETCH_ERROR", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "EARLY_ERROR"},
 			{Name: "TOOT_FETCH_ERROR", Src: []string{"FETCHING"}, Dst: "TOOT_FETCH_ERROR"},
 			{Name: "TOOTS_FETCHED", Src: []string{"FETCHING"}, Dst: "READY_FOR_TOOTFETCH"},
-			{Name: "DISABLEMENT", Src: []string{"WEIRD_NODE", "EARLY_ERROR", "TOOT_FETCH_ERROR"}, Dst: "DISABLED"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) { i.fsmEnterState(e) },
 		},
 	)
-
 	return i
 }
 
@@ -128,36 +121,10 @@ func (i *Instance) Unlock() {
 	i.structLock.Unlock()
 }
 
-func (i *Instance) bumpFetchError() {
+func (i *Instance) bumpFetch() {
 	i.Lock()
-	probablyDead := i.ConsecutiveErrorCount > 3
+	defer i.Unlock()
-	shouldDisable := i.ConsecutiveErrorCount > 6
+	i.NextFetch = time.Now().Add(120 * time.Second)
-	i.Unlock()
-
-	if shouldDisable {
-		// auf wiedersehen, felicia
-		i.Lock()
-		i.Disabled = true
-		i.Unlock()
-		i.Event("DISABLEMENT")
-		return
-	}
-
-	if probablyDead {
-		// if three consecutive fetch errors happen, only try once per day:
-		i.setNextFetchAfter(instancePersistentErrorInterval)
-	} else {
-		// otherwise give them 1h
-		i.setNextFetchAfter(instanceErrorInterval)
-	}
-}
-
-func (i *Instance) bumpFetchSuccess() {
-	i.setNextFetchAfter(instanceSpiderInterval)
-}
-
-func (i *Instance) scheduleFetchImmediate() {
-	i.setNextFetchAfter(zeroInterval)
 }
 
 func (i *Instance) setNextFetchAfter(d time.Duration) {
@@ -172,7 +139,8 @@ func (i *Instance) Fetch() {
 	i.fetchingLock.Lock()
 	defer i.fetchingLock.Unlock()
 
-	i.bumpFetchError()
+	i.setNextFetchAfter(instanceErrorInterval)
+
 	err := i.DetectNodeTypeIfNecessary()
 	if err != nil {
 		log.Debug().
@@ -181,7 +149,8 @@ func (i *Instance) Fetch() {
 			Msg("unable to fetch instance metadata")
 		return
 	}
-	i.scheduleFetchImmediate()
+
+	i.setNextFetchAfter(instanceSpiderInterval)
 	log.Info().
 		Str("hostname", i.Hostname).
 		Msg("instance now ready for fetch")
@@ -238,14 +207,12 @@ func (i *Instance) registerError() {
 	i.Lock()
 	defer i.Unlock()
 	i.ErrorCount++
-	i.ConsecutiveErrorCount++
 }
 
 func (i *Instance) registerSuccess() {
 	i.Lock()
 	defer i.Unlock()
 	i.SuccessCount++
-	i.ConsecutiveErrorCount = 0
 }
 
 // Up returns true if the success count is >0
@@ -438,12 +405,9 @@ func (i *Instance) fetchRecentToots() error {
 
 	// it turns out pleroma supports the mastodon api so we'll just use that
 	// for everything for now
-
-	// FIXME would be nice to support non-https
 	url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true",
 		i.Hostname)
 
-	// FIXME support broken/expired certs
 	var c = &http.Client{
 		Timeout: instanceHTTPTimeout,
 	}
@@ -497,7 +461,7 @@ func (i *Instance) fetchRecentToots() error {
 		Msgf("got and parsed toots")
 	i.registerSuccess()
 	i.Event("TOOTS_FETCHED")
-	i.bumpFetchSuccess()
+	i.setNextFetchAfter(instanceSpiderInterval)
 
 	// this should go fast as either the channel is buffered bigly or the
 	// ingester receives fast and does its own buffering, but run it in its

process/handlers.go

@@ -9,6 +9,7 @@ import (
 
 	u "git.eeqj.de/sneak/goutil"
 	"github.com/flosch/pongo2"
+	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 	"github.com/labstack/echo"
 )
@@ -22,12 +23,7 @@ func (a *Server) instances() []hash {
 		i := make(hash)
 		// TODO move this locking onto a method on Instance that just
 		// returns a new hash
-
+		// FIXME figure out why a very short lock here deadlocks
-		//this only locks the FSM, not the whole instance struct
-		i["status"] = v.Status()
-
-		// now do a quick lock of the whole instance just to copy out the
-		// attrs
 		v.Lock()
 		i["hostname"] = v.Hostname
 		i["uuid"] = v.UUID.String()
@@ -35,8 +31,9 @@ func (a *Server) instances() []hash {
 		i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
 		i["successCount"] = v.SuccessCount
 		i["errorCount"] = v.ErrorCount
-		i["consecutiveErrorCount"] = v.ConsecutiveErrorCount
 		i["identified"] = v.Identified
+		//this only locks the FSM, not the whole instance struct
+		i["status"] = v.Status()
 		i["software"] = "unknown"
 		i["version"] = "unknown"
 		if v.Identified {
@@ -44,7 +41,6 @@ func (a *Server) instances() []hash {
 			i["version"] = v.ServerVersionString
 		}
 		v.Unlock()
-
 		resp = append(resp, i)
 	}
 
@@ -58,7 +54,7 @@ func (a *Server) instances() []hash {
 	return resp
 }
 
-func (a *Server) instanceStatusSummary() map[string]int {
+func (a *Server) instanceSummary() map[string]int {
 	resp := make(map[string]int)
 	for _, v := range a.feta.manager.ListInstances() {
 		v.Lock()
@@ -72,6 +68,26 @@ func (a *Server) instanceStatusSummary() map[string]int {
 	return resp
 }
 
+/*
+func (a *Server) getInstanceListHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		result := &gin.H{
+			"instances": a.instances(),
+		}
+
+		json, err := json.Marshal(result)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(json)
+	}
+}
+*/
+
 func (a *Server) notFoundHandler(c echo.Context) error {
 	return c.String(http.StatusNotFound, "404 not found")
 }
@@ -101,33 +117,17 @@ func (a *Server) instanceHandler(c echo.Context) error {
 }
 
 func (a *Server) indexHandler(c echo.Context) error {
-	count, err := a.feta.dbm.TotalTootCount()
-	if err != nil {
-		count = 0
-	}
 	tc := pongo2.Context{
-		"time":                  time.Now().UTC().Format(time.RFC3339Nano),
+		"time":      time.Now().UTC().Format(time.RFC3339Nano),
-		"gitrev":                a.feta.version,
+		"gitrev":    a.feta.version,
-		"tootCount":             count,
+		"instances": a.instances(),
-		"instances":             a.instances(),
-		"instanceStatusSummary": a.instanceStatusSummary(),
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }
 
-func (a *Server) instanceListHandler(c echo.Context) error {
-	il := a.instances()
-	tc := pongo2.Context{
-		"time":      time.Now().UTC().Format(time.RFC3339Nano),
-		"gitrev":    a.feta.version,
-		"instances": il,
-	}
-	return c.Render(http.StatusOK, "instancelist.html", tc)
-}
-
 func (a *Server) statsHandler(c echo.Context) error {
-	index := &hash{
+	index := &gin.H{
-		"server": &hash{
+		"server": &gin.H{
 			"now":        time.Now().UTC().Format(time.RFC3339),
 			"uptime":     a.feta.uptime().String(),
 			"goroutines": runtime.NumGoroutine(),
@@ -135,14 +135,14 @@ func (a *Server) statsHandler(c echo.Context) error {
 			"version":    a.feta.version,
 			"buildarch":  a.feta.buildarch,
 		},
-		"instanceStatusSummary": a.instanceStatusSummary(),
+		"instanceSummary": a.instanceSummary(),
 	}
 
 	return c.JSONPretty(http.StatusOK, index, "    ")
 }
 
 func (a *Server) healthCheckHandler(c echo.Context) error {
-	resp := &hash{
+	resp := &gin.H{
 		"status": "ok",
 		"now":    time.Now().UTC().Format(time.RFC3339),
 		"uptime": a.feta.uptime().String(),

process/server.go

@@ -96,7 +96,6 @@ func (s *Server) initRouter() {
 	// Routes
 	s.e.GET("/", s.indexHandler)
 	s.e.GET("/instance/:uuid", s.instanceHandler)
-	s.e.GET("/instances", s.instanceListHandler)
 	s.e.GET("/stats.json", s.statsHandler)
 	s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
 	//a.e.GET("/about", s.aboutHandler)

view/index.html

@@ -4,44 +4,34 @@
 <div class="col-lg-12">
 
 
-<h2>feta overview</h2>
+    <h2>indexer stats</h2>
-
-
-<div class="card m-5">
-  <h5 class="card-header">Instances</h5>
-  <div class="card-body">
-      <h5 class="card-title">Tracking {{ instances | length }} instances
-          across the Fediverse.</h5>
-    <!-- 
-        <p class="card-text">
-        </p> -->
-
-    <a href="/instances" class="btn btn-primary">View Instance List</a>
-  </div>
-</div>
-
-<div class="card m-5">
-  <h5 class="card-header">Toots</h5>
-  <div class="card-body">
-      <h5 class="card-title">I have {{ tootCount }} toots
-          in my database.</h5>
-    <a href="/toots" class="btn btn-primary">View Latest Toots</a>
-  </div>
-</div>
-
-<div class="card m-5">
-  <h5 class="card-header">Recent Events</h5>
-  <div class="card-body">
-      <h5 class="card-title">Last n System Events</h5>
-
-        <p class="card-text"> Discovered instance toot1.example.com </p>
-        <p class="card-text"> Discovered instance toot2.example.com </p>
-        <p class="card-text"> Discovered instance toot3.example.com </p>
-        <p class="card-text"> Discovered instance toot4.example.com </p>
-
-  </div>
-</div>
 
+<table class="table table-striped table-hover">
+  <thead class="thead-dark">
+<tr>
+  <th scope="col">instance id</th>
+  <th scope="col">hostname</th>
+  <th scope="col">status</th>
+  <th scope="col">tootCount</th>
+  <th scope="col">Detail</th>
+</tr>
+</thead>
+<tbody>
+{% for instance in instances %}
+    <tr>
+        <td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
+        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
+        <td>{{instance.status}}</td>
+        <td>{{instance.tootCount}}</td>
+        <td><a
+                href="/instance/{{instance.uuid}}"
+                class="btn btn-info">
+                <i class="fab fa-mastodon"></i>
+            </button></td>
+    </tr>
+{% endfor %}
+ </tbody>
+</table>
 
 </div>
 {% endblock %}

view/instance.html

@@ -1,24 +0,0 @@
-{% extends "page.html" %}
-
-{% block content %}
-<div class="col-lg-12">
-
-
-<h2>instance {{instance.hostname}}</h2>
-
-<div class="card m-5">
-  <div class="card-header">
-      <a href="/instance/{{instance.uuid}}">{{ instance.hostname }}</a>
-      ({{instance.tootCount}} toots)
-  </div>
-  <div class="card-body">
-      <h5 class="card-title">{{instance.status}}</h5>
-    <p class="card-text">First Stat</p>
-    <p class="card-text">Second Stat</p>
-    <p class="card-text">Third Stat</p>
-    <a href="https://{{instance.hostname}}" class="btn btn-primary">View Instance Website</a>
-  </div>
-</div>
-
-</div>
-{% endblock %}

view/instancelist.html

@@ -1,36 +0,0 @@
-{% extends "page.html" %}
-
-{% block content %}
-<div class="col-lg-12">
-
-<h2>instance list</h2>
-
-<table class="table table-striped table-hover">
-  <thead class="thead-dark">
-<tr>
-  <th scope="col">hostname</th>
-  <th scope="col">status</th>
-  <th scope="col">tootCount</th>
-  <th scope="col">nextFetch</th>
-  <th scope="col">Detail</th>
-</tr>
-</thead>
-<tbody>
-{% for instance in instances %}
-    <tr>
-        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
-        <td>{{instance.status}}</td>
-        <td>{{instance.tootCount}}</td>
-        <td>{{instance.nextFetch}}</td>
-        <td><a
-                href="/instance/{{instance.uuid}}"
-                class="btn btn-info">
-                <i class="fab fa-mastodon"></i>
-            </button></td>
-    </tr>
-{% endfor %}
- </tbody>
-</table>
-
-</div>
-{% endblock %}