add /instances route

* permdisable instance on 6 failures
* restore in proper state on db load

.gitignore

@@ -3,3 +3,4 @@ output/
 feta.sqlite
 .lintsetup
 out
+debug.log

Dockerfile

@@ -14,19 +14,10 @@ RUN tar cfz go-src.tgz src && du -sh *
 
 FROM alpine
 
-
- 54     viper.SetDefault("Debug", false)
- 55     viper.SetDefault("TootsToDisk", false)
- 56     viper.SetDefault("TootsToDB", true)
- 57     viper.SetDefault("HostDiscoveryParallelism", 5)
- 58     viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d"))
- 59     viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))
- 60     viper.SetDefault("LogReportInterval", time.Second*10)
-
 # here are the levers
 ENV FETA_HOSTDISCOVERYPARALLELISM 20
 ENV FETA_FSSTORAGELOCATION /state/tootstore
-ENV FETA_DBSTORAGELOCATION /state/feta.state.sqlite3
+ENV FETA_DBURL sqlite:///state/feta.state.sqlite3
 ENV FETA_TOOTSTODISK false
 ENV FETA_TOOTSTODB true
 ENV FETA_DEBUG false

Makefile

@@ -26,7 +26,7 @@ endif
 default: build
 
 debug: build
-	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN)
+	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log
 
 run: build
 	./$(FN)

README.md

@@ -20,6 +20,25 @@ archives the fediverse
 
 [![Build Status](https://drone.datavi.be/api/badges/sneak/feta/status.svg)](https://drone.datavi.be/sneak/feta)
 
+# getting started
+## sqlite
+
+*using default file location:*
+`./feta`
+
+*using file:*
+`FETA_DBURL=sqlite://<abs path to file>  ./feta`
+
+*using memory:*
+`FETA_DBURL=sqlite://:memory:  ./feta`
+
+## postgres
+
+1. `docker-compose up .`
+2. `FETA_DBURL=postgres://feta_user:password@localhost:5432/feta?sslmode=disable  ./feta`
+
+Access the pgweb dashboard at `http://localhost:8081`
+
 # ethics statement
 
 It seems that some splinter groups are not well acquainted with the norms of
@@ -41,6 +60,7 @@ legitimately-obtained files from the hard drives of other people.
 
 # Author
 
-Jeffrey Paul &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt;
+Jeffrey Paul &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt; and
+others
 
 [@sneak@sneak.berlin](https://s.sneak.berlin/@sneak)

database/imconnector.go

@@ -2,9 +2,8 @@ package database
 
 import (
 	"git.eeqj.de/sneak/feta/instance"
-	"github.com/rs/zerolog/log"
-
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Manager) SaveInstance(i *instance.Instance) error {
@@ -20,6 +19,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			UUID:                       i.UUID,
 			Disabled:                   i.Disabled,
 			ErrorCount:                 i.ErrorCount,
+			ConsecutiveErrorCount:      i.ConsecutiveErrorCount,
 			FSMState:                   i.Status(),
 			Fetching:                   i.Fetching,
 			HighestID:                  i.HighestID,
@@ -27,6 +27,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			Identified:                 i.Identified,
 			Implementation:             i.Implementation,
 			NextFetch:                  i.NextFetch,
+			LastError:                  i.LastError,
 			NodeInfoURL:                i.NodeInfoURL,
 			ServerImplementationString: i.ServerImplementationString,
 			ServerVersionString:        i.ServerVersionString,
@@ -45,10 +46,12 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		m.db.Where("UUID = ?", i.UUID).First(&ei)
 		ei.Disabled = i.Disabled
 		ei.ErrorCount = i.ErrorCount
+		ei.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 		ei.FSMState = i.Status()
 		ei.Fetching = i.Fetching
 		ei.HighestID = i.HighestID
 		ei.Hostname = i.Hostname
+		ei.LastError = i.LastError
 		ei.Identified = i.Identified
 		ei.Implementation = string(i.Implementation)
 		ei.NextFetch = i.NextFetch
@@ -56,7 +59,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		ei.ServerImplementationString = i.ServerImplementationString
 		ei.ServerVersionString = i.ServerVersionString
 		ei.SuccessCount = i.SuccessCount
-
 		r := m.db.Save(&ei)
 		return r.Error
 	}
@@ -73,10 +75,12 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
 			x.UUID = i.UUID
 			x.Disabled = i.Disabled
 			x.ErrorCount = i.ErrorCount
+			x.ConsecutiveErrorCount = i.ConsecutiveErrorCount
 			x.InitialFSMState = i.FSMState
 			x.Fetching = i.Fetching
 			x.HighestID = i.HighestID
 			x.Hostname = i.Hostname
+			x.LastError = i.LastError
 			x.Identified = i.Identified
 			x.Implementation = i.Implementation
 			x.NextFetch = i.NextFetch

database/manager.go

@@ -1,23 +1,30 @@
 package database
 
 import (
+	"net/url"
 	"path/filepath"
+	"strings"
 	"sync"
 
-	"github.com/jinzhu/gorm"
-
 	u "git.eeqj.de/sneak/goutil"
+	"github.com/golang/groupcache/lru"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 )
 
+const cacheEntries = 1000000
+
+// Manager coordinates database and cache reads and writes
 type Manager struct {
 	db                            *gorm.DB
 	cachelock                     sync.Mutex
-	recentlyInsertedTootHashCache *map[string]bool
+	recentlyInsertedTootHashCache *lru.Cache
 }
 
+// New creates new Manager
 func New() *Manager {
 	m := new(Manager)
 	m.init()
@@ -25,34 +32,65 @@ func New() *Manager {
 }
 
 func (m *Manager) init() {
-	m.open()
+	m.open(viper.GetString("DBURL"))
 	// breaks stuff, do not use:
 	//m.db.SingularTable(true)
 	m.db.LogMode(false)
 	if viper.GetBool("Debug") {
 		m.db.LogMode(true)
 	}
-
-	h := make(map[string]bool)
-	m.recentlyInsertedTootHashCache = &h
+	m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
 }
 
-func (m *Manager) open() {
+func (m *Manager) open(dbURL string) {
 	log.Info().Msg("opening database")
-	dirname := filepath.Dir(viper.GetString("DbStorageLocation"))
-	err := u.Mkdirp(dirname)
+	dsn, err := url.Parse(dbURL)
 	if err != nil {
 		log.Panic().
 			Err(err).
-			Msg("db path erro")
+			Msg("error parsing dbURL")
 	}
-
-	db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation"))
+	log.Info().
+		Str("scheme", dsn.Scheme).
+		Str("user", dsn.User.Username()).
+		Str("host", dsn.Host).
+		Str("db", dsn.Path).
+		Str("args", dsn.RawQuery).
+		Msg("db connection values")
+	switch {
+	case strings.HasPrefix(dbURL, "postgres://"):
+		log.Info().Msg("using postgres db")
+		db, err := gorm.Open("postgres", dbURL)
 		if err != nil {
 			log.Panic().
 				Err(err).
 				Msg("failed to open database")
 		}
 		m.db = db
+	case strings.HasPrefix(dbURL, "sqlite://"):
+		log.Info().Msg("using sqlite db")
+		if !strings.HasSuffix(dbURL, ":memory:") {
+			dirname := filepath.Dir(strings.TrimPrefix(dbURL, "sqlite://"))
+			err := u.Mkdirp(dirname)
+			if err != nil {
+				log.Panic().
+					Err(err).
+					Msg("db path error")
+			}
+		}
+		db, err := gorm.Open("sqlite3", strings.TrimPrefix(dbURL, "sqlite://"))
+		if err != nil {
+			log.Panic().
+				Err(err).
+				Str("dbURL", dbURL).
+				Msg("failed to open database")
+		}
+		m.db = db
+	default:
+		log.Panic().
+			Str("driver", dsn.Scheme).
+			Msg("unsupported driver in database url, must be 'postgres' or 'sqlite'")
+	}
+
 	m.doMigrations()
 }

database/model.go

@@ -21,12 +21,13 @@ type StoredToot struct {
 	Content       []byte
 	TextContent   []byte
 	URL           string
-	Hostname      string
+	Hostname      string `gorm:"index:hostnameindex"`
 }
 
 type APInstance struct {
 	gorm.Model
 	UUID                       uuid.UUID `gorm:"type:uuid;primary_key;"`
+	ConsecutiveErrorCount      uint
 	ErrorCount                 uint
 	SuccessCount               uint
 	HighestID                  uint
@@ -34,6 +35,7 @@ type APInstance struct {
 	Identified                 bool
 	Fetching                   bool
 	Disabled                   bool
+	LastError                  string
 	Implementation             string
 	NextFetch                  time.Time
 	NodeInfoURL                string

database/readshortcuts.go

@@ -0,0 +1,34 @@
+package database
+
+import (
+	"github.com/google/uuid"
+)
+
+func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
+	var c uint
+	e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
+	if e.Error != nil {
+		return 0, e.Error
+	} else {
+		return c, nil
+	}
+}
+
+func (m *Manager) TotalTootCount() (uint, error) {
+	var c uint
+	e := m.db.Model(&StoredToot{}).Count(&c)
+	if e.Error != nil {
+		return 0, e.Error
+	} else {
+		return c, nil
+	}
+}
+
+func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
+	var i APInstance
+	e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
+	if e.Error != nil {
+		return nil, e.Error
+	}
+	return &i, nil
+}

database/storageconnector.go

@@ -4,35 +4,24 @@ import (
 	"fmt"
 	"html"
 	"strings"
-
 	"git.eeqj.de/sneak/feta/toot"
-
 	"github.com/google/uuid"
 	hstg "github.com/grokify/html-strip-tags-go"
-	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 )
 
 func (m *Manager) TootInsertHashCacheSize() uint {
 	m.cachelock.Lock()
 	defer m.cachelock.Unlock()
-	return uint(len(*m.recentlyInsertedTootHashCache))
+	return uint(m.recentlyInsertedTootHashCache.Len())
 }
 
 func (m *Manager) TootExists(t *toot.Toot) bool {
 	var try StoredToot
 
-	// first, nuke cache if it gets too big to prevent
-	// unlimited memory usage
+	// check cache
 	m.cachelock.Lock()
-	if len(*m.recentlyInsertedTootHashCache) > 1000000 {
-		emptycache := make(map[string]bool)
-		m.recentlyInsertedTootHashCache = &emptycache
-	}
-	m.cachelock.Unlock()
-
-	m.cachelock.Lock()
-	// check map
-	if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
+	if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
 		m.cachelock.Unlock()
 		return true
 	}
@@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
 
 	// put it in the cache to avoid relying on db for dedupe:
 	m.cachelock.Lock()
-	(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
+	m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
 	m.cachelock.Unlock()
 
 	//panic(fmt.Sprintf("%+v", t))

docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.1'
+
+services:
+  postgres:
+    image: postgres:12
+    restart: always
+    container_name: postgres
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: feta_user
+      POSTGRES_DB: feta
+  pgweb:
+    image: sosedoff/pgweb
+    restart: always
+    container_name: pgweb
+    ports: 
+        - "8081:8081" 
+    links: 
+        - postgres:postgres
+    environment:
+        - DATABASE_URL=postgres://feta_user:password@postgres:5432/feta?sslmode=disable
+    depends_on:
+        - postgres

go.mod

@@ -4,8 +4,6 @@ go 1.14
 
 require (
 	git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
-	github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
-	github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
 	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gin-gonic/gin v1.6.2
@@ -37,10 +35,9 @@ require (
 	github.com/valyala/fasttemplate v1.1.0 // indirect
 	github.com/ziflex/lecho v1.2.0
 	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
-	golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 // indirect
 	gopkg.in/ini.v1 v1.55.0 // indirect
+	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
 )

go.sum

@@ -56,6 +56,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -114,6 +115,7 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20=
 github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI=

instance/instance.go

@@ -17,18 +17,19 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-//import "github.com/gin-gonic/gin"
-
 const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
-const instanceNodeinfoTimeout = time.Second * 50
-const instanceHTTPTimeout = time.Second * 120
-const instanceSpiderInterval = time.Second * 120
-const instanceErrorInterval = time.Second * 60 * 30
+const instanceNodeinfoTimeout = time.Second * 60 * 2        // 2m
+const instanceHTTPTimeout = time.Second * 60 * 2            // 2m
+const instanceSpiderInterval = time.Second * 60 * 2         // 2m
+const instanceErrorInterval = time.Second * 60 * 60         // 1h
+const instancePersistentErrorInterval = time.Second * 86400 // 1d
+const zeroInterval = time.Second * 0                        // 0s
 
 // Instance stores all the information we know about an instance
 type Instance struct {
 	Disabled                   bool
 	ErrorCount                 uint
+	ConsecutiveErrorCount      uint
 	FSM                        *fsm.FSM
 	Fetching                   bool
 	HighestID                  uint
@@ -37,6 +38,7 @@ type Instance struct {
 	Implementation             string
 	InitialFSMState            string
 	NextFetch                  time.Time
+	LastError                  string
 	NodeInfoURL                string
 	ServerImplementationString string
 	ServerVersionString        string
@@ -60,6 +62,10 @@ func New(options ...func(i *Instance)) *Instance {
 		opt(i)
 	}
 
+	if i.InitialFSMState == "FETCHING" {
+		i.InitialFSMState = "READY_FOR_TOOTFETCH"
+	}
+
 	i.FSM = fsm.NewFSM(
 		i.InitialFSMState,
 		fsm.Events{
@@ -73,11 +79,13 @@ func New(options ...func(i *Instance)) *Instance {
 			{Name: "EARLY_FETCH_ERROR", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "EARLY_ERROR"},
 			{Name: "TOOT_FETCH_ERROR", Src: []string{"FETCHING"}, Dst: "TOOT_FETCH_ERROR"},
 			{Name: "TOOTS_FETCHED", Src: []string{"FETCHING"}, Dst: "READY_FOR_TOOTFETCH"},
+			{Name: "DISABLEMENT", Src: []string{"WEIRD_NODE", "EARLY_ERROR", "TOOT_FETCH_ERROR"}, Dst: "DISABLED"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) { i.fsmEnterState(e) },
 		},
 	)
+
 	return i
 }
 
@@ -120,10 +128,36 @@ func (i *Instance) Unlock() {
 	i.structLock.Unlock()
 }
 
-func (i *Instance) bumpFetch() {
+func (i *Instance) bumpFetchError() {
 	i.Lock()
-	defer i.Unlock()
-	i.NextFetch = time.Now().Add(120 * time.Second)
+	probablyDead := i.ConsecutiveErrorCount > 3
+	shouldDisable := i.ConsecutiveErrorCount > 6
+	i.Unlock()
+
+	if shouldDisable {
+		// auf wiedersehen, felicia
+		i.Lock()
+		i.Disabled = true
+		i.Unlock()
+		i.Event("DISABLEMENT")
+		return
+	}
+
+	if probablyDead {
+		// if three consecutive fetch errors happen, only try once per day:
+		i.setNextFetchAfter(instancePersistentErrorInterval)
+	} else {
+		// otherwise give them 1h
+		i.setNextFetchAfter(instanceErrorInterval)
+	}
+}
+
+func (i *Instance) bumpFetchSuccess() {
+	i.setNextFetchAfter(instanceSpiderInterval)
+}
+
+func (i *Instance) scheduleFetchImmediate() {
+	i.setNextFetchAfter(zeroInterval)
 }
 
 func (i *Instance) setNextFetchAfter(d time.Duration) {
@@ -138,8 +172,7 @@ func (i *Instance) Fetch() {
 	i.fetchingLock.Lock()
 	defer i.fetchingLock.Unlock()
 
-	i.setNextFetchAfter(instanceErrorInterval)
-
+	i.bumpFetchError()
 	err := i.DetectNodeTypeIfNecessary()
 	if err != nil {
 		log.Debug().
@@ -148,8 +181,7 @@ func (i *Instance) Fetch() {
 			Msg("unable to fetch instance metadata")
 		return
 	}
-
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.scheduleFetchImmediate()
 	log.Info().
 		Str("hostname", i.Hostname).
 		Msg("instance now ready for fetch")
@@ -206,12 +238,14 @@ func (i *Instance) registerError() {
 	i.Lock()
 	defer i.Unlock()
 	i.ErrorCount++
+	i.ConsecutiveErrorCount++
 }
 
 func (i *Instance) registerSuccess() {
 	i.Lock()
 	defer i.Unlock()
 	i.SuccessCount++
+	i.ConsecutiveErrorCount = 0
 }
 
 // Up returns true if the success count is >0
@@ -404,9 +438,12 @@ func (i *Instance) fetchRecentToots() error {
 
 	// it turns out pleroma supports the mastodon api so we'll just use that
 	// for everything for now
+
+	// FIXME would be nice to support non-https
 	url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true",
 		i.Hostname)
 
+	// FIXME support broken/expired certs
 	var c = &http.Client{
 		Timeout: instanceHTTPTimeout,
 	}
@@ -460,7 +497,7 @@ func (i *Instance) fetchRecentToots() error {
 		Msgf("got and parsed toots")
 	i.registerSuccess()
 	i.Event("TOOTS_FETCHED")
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.bumpFetchSuccess()
 
 	// this should go fast as either the channel is buffered bigly or the
 	// ingester receives fast and does its own buffering, but run it in its

process/feta.go

@@ -1,6 +1,7 @@
 package process
 
 import (
+	"fmt"
 	"os"
 	"time"
 
@@ -9,6 +10,7 @@ import (
 	"git.eeqj.de/sneak/feta/locator"
 	"git.eeqj.de/sneak/feta/manager"
 	"git.eeqj.de/sneak/feta/storage"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/k0kubun/pp"
 	"github.com/mattn/go-isatty"
@@ -56,7 +58,7 @@ func (f *Feta) configure() {
 	viper.SetDefault("TootsToDB", true)
 	viper.SetDefault("HostDiscoveryParallelism", 5)
 	viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d"))
-	viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))
+	viper.SetDefault("DBURL", fmt.Sprintf("sqlite://%s", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db")))
 	viper.SetDefault("LogReportInterval", time.Second*10)
 
 	if err := viper.ReadInConfig(); err != nil {

process/handlers.go

@@ -9,7 +9,7 @@ import (
 
 	u "git.eeqj.de/sneak/goutil"
 	"github.com/flosch/pongo2"
-	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
 	"github.com/labstack/echo"
 )
 
@@ -20,15 +20,23 @@ func (a *Server) instances() []hash {
 	now := time.Now()
 	for _, v := range a.feta.manager.ListInstances() {
 		i := make(hash)
-		// FIXME figure out why a very short lock here deadlocks
+		// TODO move this locking onto a method on Instance that just
+		// returns a new hash
+
+		//this only locks the FSM, not the whole instance struct
+		i["status"] = v.Status()
+
+		// now do a quick lock of the whole instance just to copy out the
+		// attrs
 		v.Lock()
 		i["hostname"] = v.Hostname
+		i["uuid"] = v.UUID.String()
 		i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
 		i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
 		i["successCount"] = v.SuccessCount
 		i["errorCount"] = v.ErrorCount
+		i["consecutiveErrorCount"] = v.ConsecutiveErrorCount
 		i["identified"] = v.Identified
-		i["status"] = v.Status() //FIXME maybe this is why
 		i["software"] = "unknown"
 		i["version"] = "unknown"
 		if v.Identified {
@@ -36,12 +44,21 @@ func (a *Server) instances() []hash {
 			i["version"] = v.ServerVersionString
 		}
 		v.Unlock()
+
 		resp = append(resp, i)
 	}
+
+	for _, item := range resp {
+		count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
+		item["tootCount"] = 0
+		if err != nil {
+			item["tootCount"] = count
+		}
+	}
 	return resp
 }
 
-func (a *Server) instanceSummary() map[string]int {
+func (a *Server) instanceStatusSummary() map[string]int {
 	resp := make(map[string]int)
 	for _, v := range a.feta.manager.ListInstances() {
 		v.Lock()
@@ -55,37 +72,62 @@ func (a *Server) instanceSummary() map[string]int {
 	return resp
 }
 
-/*
-func (a *Server) getInstanceListHandler() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-
-		result := &gin.H{
-			"instances": a.instances(),
-		}
-
-		json, err := json.Marshal(result)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		w.Write(json)
-	}
+func (a *Server) notFoundHandler(c echo.Context) error {
+	return c.String(http.StatusNotFound, "404 not found")
+}
+
+func (a *Server) instanceHandler(c echo.Context) error {
+	tu := c.Param("uuid")
+	u, err := uuid.Parse(tu)
+	if err != nil {
+		return a.notFoundHandler(c)
+	}
+
+	tc := pongo2.Context{}
+	instances := a.feta.manager.ListInstances()
+	found := false
+	for _, item := range instances {
+		if item.UUID == u {
+			tc["instance"] = item
+			found = true
+		}
+	}
+
+	if !found {
+		return a.notFoundHandler(c)
+	}
+
+	return c.Render(http.StatusOK, "instance.html", tc)
 }
-*/
 
 func (a *Server) indexHandler(c echo.Context) error {
+	count, err := a.feta.dbm.TotalTootCount()
+	if err != nil {
+		count = 0
+	}
 	tc := pongo2.Context{
 		"time":                  time.Now().UTC().Format(time.RFC3339Nano),
 		"gitrev":                a.feta.version,
+		"tootCount":             count,
+		"instances":             a.instances(),
+		"instanceStatusSummary": a.instanceStatusSummary(),
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }
 
+func (a *Server) instanceListHandler(c echo.Context) error {
+	il := a.instances()
+	tc := pongo2.Context{
+		"time":      time.Now().UTC().Format(time.RFC3339Nano),
+		"gitrev":    a.feta.version,
+		"instances": il,
+	}
+	return c.Render(http.StatusOK, "instancelist.html", tc)
+}
+
 func (a *Server) statsHandler(c echo.Context) error {
-	index := &gin.H{
-		"server": &gin.H{
+	index := &hash{
+		"server": &hash{
 			"now":        time.Now().UTC().Format(time.RFC3339),
 			"uptime":     a.feta.uptime().String(),
 			"goroutines": runtime.NumGoroutine(),
@@ -93,14 +135,14 @@ func (a *Server) statsHandler(c echo.Context) error {
 			"version":    a.feta.version,
 			"buildarch":  a.feta.buildarch,
 		},
-		"instanceSummary": a.instanceSummary(),
+		"instanceStatusSummary": a.instanceStatusSummary(),
 	}
 
 	return c.JSONPretty(http.StatusOK, index, "    ")
 }
 
 func (a *Server) healthCheckHandler(c echo.Context) error {
-	resp := &gin.H{
+	resp := &hash{
 		"status": "ok",
 		"now":    time.Now().UTC().Format(time.RFC3339),
 		"uptime": a.feta.uptime().String(),

process/server.go

@@ -95,6 +95,8 @@ func (s *Server) initRouter() {
 
 	// Routes
 	s.e.GET("/", s.indexHandler)
+	s.e.GET("/instance/:uuid", s.instanceHandler)
+	s.e.GET("/instances", s.instanceListHandler)
 	s.e.GET("/stats.json", s.statsHandler)
 	s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
 	//a.e.GET("/about", s.aboutHandler)

seeds/seeds.go

@@ -315,6 +315,7 @@ var SeedInstances = [...]string{
 	"snuskete.net",
 	"social.1in9.net",
 	"social.adlerweb.info",
+	"social.art-software.fr",
 	"social.au2pb.net",
 	"social.avareborn.de",
 	"social.azkware.net",

view/base.html

@@ -10,6 +10,7 @@
       <meta name="author" content="">
     -->
     <title>{{ htmltitle }}</title>
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
   <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
 
 <style>

view/index.html

@@ -4,26 +4,44 @@
 <div class="col-lg-12">
 
 
-    <h2>indexer stats</h2>
+<h2>feta overview</h2>
+
+
+<div class="card m-5">
+  <h5 class="card-header">Instances</h5>
+  <div class="card-body">
+      <h5 class="card-title">Tracking {{ instances | length }} instances
+          across the Fediverse.</h5>
+    <!-- 
+        <p class="card-text">
+        </p> -->
+
+    <a href="/instances" class="btn btn-primary">View Instance List</a>
+  </div>
+</div>
+
+<div class="card m-5">
+  <h5 class="card-header">Toots</h5>
+  <div class="card-body">
+      <h5 class="card-title">I have {{ tootCount }} toots
+          in my database.</h5>
+    <a href="/toots" class="btn btn-primary">View Latest Toots</a>
+  </div>
+</div>
+
+<div class="card m-5">
+  <h5 class="card-header">Recent Events</h5>
+  <div class="card-body">
+      <h5 class="card-title">Last n System Events</h5>
+
+        <p class="card-text"> Discovered instance toot1.example.com </p>
+        <p class="card-text"> Discovered instance toot2.example.com </p>
+        <p class="card-text"> Discovered instance toot3.example.com </p>
+        <p class="card-text"> Discovered instance toot4.example.com </p>
+
+  </div>
+</div>
 
-<table class="table table-striped table-hover">
-  <thead class="thead-dark">
-<tr>
-  <th scope="col">Key</th>
-  <th scope="col">Value</th>
-</tr>
-</thead>
-<tbody>
-{% for stat in stats %}
-    <tr>
-        {{stat.key}}
-    </tr>
-    <tr>
-        {{stat.value}}
-    </tr>
-{% endfor %}
- </tbody>
-</table>
 
 </div>
 {% endblock %}

view/instance.html

@@ -0,0 +1,24 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+
+<h2>instance {{instance.hostname}}</h2>
+
+<div class="card m-5">
+  <div class="card-header">
+      <a href="/instance/{{instance.uuid}}">{{ instance.hostname }}</a>
+      ({{instance.tootCount}} toots)
+  </div>
+  <div class="card-body">
+      <h5 class="card-title">{{instance.status}}</h5>
+    <p class="card-text">First Stat</p>
+    <p class="card-text">Second Stat</p>
+    <p class="card-text">Third Stat</p>
+    <a href="https://{{instance.hostname}}" class="btn btn-primary">View Instance Website</a>
+  </div>
+</div>
+
+</div>
+{% endblock %}

view/instancelist.html

@@ -0,0 +1,36 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+<h2>instance list</h2>
+
+<table class="table table-striped table-hover">
+  <thead class="thead-dark">
+<tr>
+  <th scope="col">hostname</th>
+  <th scope="col">status</th>
+  <th scope="col">tootCount</th>
+  <th scope="col">nextFetch</th>
+  <th scope="col">Detail</th>
+</tr>
+</thead>
+<tbody>
+{% for instance in instances %}
+    <tr>
+        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
+        <td>{{instance.status}}</td>
+        <td>{{instance.tootCount}}</td>
+        <td>{{instance.nextFetch}}</td>
+        <td><a
+                href="/instance/{{instance.uuid}}"
+                class="btn btn-info">
+                <i class="fab fa-mastodon"></i>
+            </button></td>
+    </tr>
+{% endfor %}
+ </tbody>
+</table>
+
+</div>
+{% endblock %}