add /instances route

* permdisable instance on 6 failures
* restore in proper state on db load

.circleci/config.yml

@@ -1,10 +0,0 @@
-version: 2
-jobs:
- build:
-   machine: true
-   steps:
-     - checkout
-     - run: |
-         make dist
-     - store_artifacts:
-         path: output

.drone.yml

@@ -6,11 +6,8 @@ steps:
   image: plugins/docker
   settings:
     repo: sneak/feta
-    username:
+    dry_run: true
-      from_secret: docker_username
+    #auto_tag: true
-    password:
-      from_secret: docker_password
-    auto_tag: true
     tags:
       - ${DRONE_COMMIT_SHA}
       - ${DRONE_BRANCH}

.gitignore

@@ -2,3 +2,5 @@ feta
 output/
 feta.sqlite
 .lintsetup
+out
+debug.log

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13 as builder
+FROM golang:1.14 as builder
 
 WORKDIR /go/src/git.eeqj.de/sneak/feta
 COPY . .
@@ -7,18 +7,31 @@ COPY . .
 RUN make build
 
 WORKDIR /go
-RUN tar cvfz go-src.tgz src && du -sh *
+RUN tar cfz go-src.tgz src && du -sh *
 
 # this container doesn't do anything except hold the build artifact
 # and make sure it compiles.
 
 FROM alpine
 
+# here are the levers
+ENV FETA_HOSTDISCOVERYPARALLELISM 20
+ENV FETA_FSSTORAGELOCATION /state/tootstore
+ENV FETA_DBURL sqlite:///state/feta.state.sqlite3
+ENV FETA_TOOTSTODISK false
+ENV FETA_TOOTSTODB true
+ENV FETA_DEBUG false
+
 COPY --from=builder /go/src/git.eeqj.de/sneak/feta/feta /bin/feta
+RUN mkdir /app && mkdir /state
+COPY --from=builder /go/src/git.eeqj.de/sneak/feta/view /app/view
 
 # put the source in there too for safekeeping
 COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
 
+VOLUME /state
+WORKDIR /app
+
 CMD /bin/feta
 
 # FIXME add testing

Makefile

@@ -26,7 +26,7 @@ endif
 default: build
 
 debug: build
-	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN)
+	GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log
 
 run: build
 	./$(FN)

README.md

@@ -2,9 +2,42 @@
 
 archives the fediverse
 
+# todo
+
+* scan toots for mentions and feed to locator
+* put toots in a separate db file
+* test with a real database
+* save instancelist to store more often (maybe on each new one added not
+  during initial load)
+* verify instances load properly on startup
+* do some simple in-memory dedupe for toot storage
+* make some templates using pongo2 and a simple website
+* update json APIs
+* index hashtags
+* index seen urls
+
 # status
 
-[![CircleCI](https://circleci.com/gh/sneak/feta.svg?style=svg)](https://circleci.com/gh/sneak/feta)
+[![Build Status](https://drone.datavi.be/api/badges/sneak/feta/status.svg)](https://drone.datavi.be/sneak/feta)
+
+# getting started
+## sqlite
+
+*using default file location:*
+`./feta`
+
+*using file:*
+`FETA_DBURL=sqlite://<abs path to file>  ./feta`
+
+*using memory:*
+`FETA_DBURL=sqlite://:memory:  ./feta`
+
+## postgres
+
+1. `docker-compose up .`
+2. `FETA_DBURL=postgres://feta_user:password@localhost:5432/feta?sslmode=disable  ./feta`
+
+Access the pgweb dashboard at `http://localhost:8081`
 
 # ethics statement
 
@@ -27,6 +60,7 @@ legitimately-obtained files from the hard drives of other people.
 
 # Author
 
-Jeffrey Paul &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt;
+Jeffrey Paul &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt; and
+others
 
 [@sneak@sneak.berlin](https://s.sneak.berlin/@sneak)

database/dbmodel.go

@@ -1,37 +0,0 @@
-package database
-
-import (
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/jinzhu/gorm"
-
-	_ "github.com/jinzhu/gorm/dialects/sqlite"
-)
-
-// NB that when you add a model below you must add it to this list!
-
-func (m *Manager) doMigrations() {
-	m.db.AutoMigrate(&instance{})
-}
-
-type instance struct {
-	gorm.Model
-	Identifier                 uuid.UUID
-	ErrorCount                 uint
-	SuccessCount               uint
-	HighestID                  int
-	Hostname                   string
-	Identified                 bool
-	Fetching                   bool
-	Disabled                   bool
-	Implementation             string
-	NextFetch                  time.Time
-	NodeInfoURL                string
-	ServerVersionString        string
-	ServerImplementationString string
-}
-
-func (m *Manager) ListInstances() {
-	// FIXME have this produce a list of Instance
-}

database/imconnector.go

@@ -0,0 +1,96 @@
+package database
+
+import (
+	"git.eeqj.de/sneak/feta/instance"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Manager) SaveInstance(i *instance.Instance) error {
+	i.Lock()
+	defer i.Unlock()
+	var x APInstance
+	if m.db.Where("UUID = ?", i.UUID).First(&x).RecordNotFound() {
+		log.Info().
+			Str("hostname", i.Hostname).
+			Msg("instance not in db, inserting")
+		// item does not exist in db yet, must insert
+		ni := APInstance{
+			UUID:                       i.UUID,
+			Disabled:                   i.Disabled,
+			ErrorCount:                 i.ErrorCount,
+			ConsecutiveErrorCount:      i.ConsecutiveErrorCount,
+			FSMState:                   i.Status(),
+			Fetching:                   i.Fetching,
+			HighestID:                  i.HighestID,
+			Hostname:                   i.Hostname,
+			Identified:                 i.Identified,
+			Implementation:             i.Implementation,
+			NextFetch:                  i.NextFetch,
+			LastError:                  i.LastError,
+			NodeInfoURL:                i.NodeInfoURL,
+			ServerImplementationString: i.ServerImplementationString,
+			ServerVersionString:        i.ServerVersionString,
+			SuccessCount:               i.SuccessCount,
+		}
+		r := m.db.Create(&ni)
+		return r.Error
+	} else {
+		log.Info().
+			Str("hostname", i.Hostname).
+			Str("id", i.UUID.String()).
+			Msg("instance found in db, updating")
+		// exists in db, update db
+		var ei APInstance
+		// EI EI uh-oh
+		m.db.Where("UUID = ?", i.UUID).First(&ei)
+		ei.Disabled = i.Disabled
+		ei.ErrorCount = i.ErrorCount
+		ei.ConsecutiveErrorCount = i.ConsecutiveErrorCount
+		ei.FSMState = i.Status()
+		ei.Fetching = i.Fetching
+		ei.HighestID = i.HighestID
+		ei.Hostname = i.Hostname
+		ei.LastError = i.LastError
+		ei.Identified = i.Identified
+		ei.Implementation = string(i.Implementation)
+		ei.NextFetch = i.NextFetch
+		ei.NodeInfoURL = i.NodeInfoURL
+		ei.ServerImplementationString = i.ServerImplementationString
+		ei.ServerVersionString = i.ServerVersionString
+		ei.SuccessCount = i.SuccessCount
+		r := m.db.Save(&ei)
+		return r.Error
+	}
+}
+
+func (m *Manager) ListInstances() ([]*instance.Instance, error) {
+	output := make([]*instance.Instance, 0)
+
+	var results []APInstance
+	m.db.Find(&results)
+
+	for _, i := range results {
+		newinst := instance.New(func(x *instance.Instance) {
+			x.UUID = i.UUID
+			x.Disabled = i.Disabled
+			x.ErrorCount = i.ErrorCount
+			x.ConsecutiveErrorCount = i.ConsecutiveErrorCount
+			x.InitialFSMState = i.FSMState
+			x.Fetching = i.Fetching
+			x.HighestID = i.HighestID
+			x.Hostname = i.Hostname
+			x.LastError = i.LastError
+			x.Identified = i.Identified
+			x.Implementation = i.Implementation
+			x.NextFetch = i.NextFetch
+			x.NodeInfoURL = i.NodeInfoURL
+			x.ServerImplementationString = i.ServerImplementationString
+			x.ServerVersionString = i.ServerVersionString
+			x.SuccessCount = i.SuccessCount
+		})
+		output = append(output, newinst)
+	}
+
+	return output, nil
+}

database/manager.go

@@ -1,21 +1,30 @@
 package database
 
 import (
-	"errors"
+	"net/url"
-	"os"
 	"path/filepath"
+	"strings"
+	"sync"
 
+	u "git.eeqj.de/sneak/goutil"
+	"github.com/golang/groupcache/lru"
 	"github.com/jinzhu/gorm"
-
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 )
 
+const cacheEntries = 1000000
+
+// Manager coordinates database and cache reads and writes
 type Manager struct {
-	db *gorm.DB
+	db                            *gorm.DB
+	cachelock                     sync.Mutex
+	recentlyInsertedTootHashCache *lru.Cache
 }
 
+// New creates new Manager
 func New() *Manager {
 	m := new(Manager)
 	m.init()
@@ -23,40 +32,65 @@ func New() *Manager {
 }
 
 func (m *Manager) init() {
-	m.open()
+	m.open(viper.GetString("DBURL"))
+	// breaks stuff, do not use:
+	//m.db.SingularTable(true)
+	m.db.LogMode(false)
+	if viper.GetBool("Debug") {
+		m.db.LogMode(true)
+	}
+	m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
 }
 
-func mkdirp(p string) error {
+func (m *Manager) open(dbURL string) {
-	src, err := os.Stat(p)
-	if os.IsNotExist(err) {
-		errDir := os.MkdirAll(p, 0755)
-		if errDir != nil {
-			return err
-		}
-		return nil
-	}
-	if src.Mode().IsRegular() {
-		return errors.New("file already exists at path")
-	}
-	return nil
-}
-
-func (m *Manager) open() {
 	log.Info().Msg("opening database")
-	dirname := filepath.Dir(viper.GetString("DbStorageLocation"))
+	dsn, err := url.Parse(dbURL)
-	err := mkdirp(dirname)
 	if err != nil {
 		log.Panic().
 			Err(err).
-			Msg("db path erro")
+			Msg("error parsing dbURL")
+	}
+	log.Info().
+		Str("scheme", dsn.Scheme).
+		Str("user", dsn.User.Username()).
+		Str("host", dsn.Host).
+		Str("db", dsn.Path).
+		Str("args", dsn.RawQuery).
+		Msg("db connection values")
+	switch {
+	case strings.HasPrefix(dbURL, "postgres://"):
+		log.Info().Msg("using postgres db")
+		db, err := gorm.Open("postgres", dbURL)
+		if err != nil {
+			log.Panic().
+				Err(err).
+				Msg("failed to open database")
+		}
+		m.db = db
+	case strings.HasPrefix(dbURL, "sqlite://"):
+		log.Info().Msg("using sqlite db")
+		if !strings.HasSuffix(dbURL, ":memory:") {
+			dirname := filepath.Dir(strings.TrimPrefix(dbURL, "sqlite://"))
+			err := u.Mkdirp(dirname)
+			if err != nil {
+				log.Panic().
+					Err(err).
+					Msg("db path error")
+			}
+		}
+		db, err := gorm.Open("sqlite3", strings.TrimPrefix(dbURL, "sqlite://"))
+		if err != nil {
+			log.Panic().
+				Err(err).
+				Str("dbURL", dbURL).
+				Msg("failed to open database")
+		}
+		m.db = db
+	default:
+		log.Panic().
+			Str("driver", dsn.Scheme).
+			Msg("unsupported driver in database url, must be 'postgres' or 'sqlite'")
 	}
 
-	db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation"))
-	if err != nil {
-		log.Panic().
-			Err(err).
-			Msg("failed to open database")
-	}
-	m.db = db
 	m.doMigrations()
 }

database/model.go

@@ -0,0 +1,52 @@
+package database
+
+import (
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jinzhu/gorm"
+	"github.com/rs/zerolog/log"
+
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+)
+
+type StoredToot struct {
+	gorm.Model
+	UUID uuid.UUID `gorm:"type:uuid;primary_key;"`
+	//Original      string    `sql:"type:text"`
+	Original      []byte
+	Hash          string `gorm:"unique_index"`
+	ServerCreated time.Time
+	Acct          string
+	Content       []byte
+	TextContent   []byte
+	URL           string
+	Hostname      string `gorm:"index:hostnameindex"`
+}
+
+type APInstance struct {
+	gorm.Model
+	UUID                       uuid.UUID `gorm:"type:uuid;primary_key;"`
+	ConsecutiveErrorCount      uint
+	ErrorCount                 uint
+	SuccessCount               uint
+	HighestID                  uint
+	Hostname                   string `gorm:"type:varchar(100);unique_index"`
+	Identified                 bool
+	Fetching                   bool
+	Disabled                   bool
+	LastError                  string
+	Implementation             string
+	NextFetch                  time.Time
+	NodeInfoURL                string
+	ServerVersionString        string
+	ServerImplementationString string
+	FSMState                   string
+}
+
+// NB that when you add a model below you must add it to this list!
+func (m *Manager) doMigrations() {
+	log.Info().Msg("doing database migrations if required")
+	m.db.AutoMigrate(&APInstance{})
+	m.db.AutoMigrate(&StoredToot{})
+}

database/readshortcuts.go

@@ -0,0 +1,34 @@
+package database
+
+import (
+	"github.com/google/uuid"
+)
+
+func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
+	var c uint
+	e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
+	if e.Error != nil {
+		return 0, e.Error
+	} else {
+		return c, nil
+	}
+}
+
+func (m *Manager) TotalTootCount() (uint, error) {
+	var c uint
+	e := m.db.Model(&StoredToot{}).Count(&c)
+	if e.Error != nil {
+		return 0, e.Error
+	} else {
+		return c, nil
+	}
+}
+
+func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
+	var i APInstance
+	e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
+	if e.Error != nil {
+		return nil, e.Error
+	}
+	return &i, nil
+}

database/storageconnector.go

@@ -0,0 +1,74 @@
+package database
+
+import (
+	"fmt"
+	"html"
+	"strings"
+	"git.eeqj.de/sneak/feta/toot"
+	"github.com/google/uuid"
+	hstg "github.com/grokify/html-strip-tags-go"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
+)
+
+func (m *Manager) TootInsertHashCacheSize() uint {
+	m.cachelock.Lock()
+	defer m.cachelock.Unlock()
+	return uint(m.recentlyInsertedTootHashCache.Len())
+}
+
+func (m *Manager) TootExists(t *toot.Toot) bool {
+	var try StoredToot
+
+	// check cache
+	m.cachelock.Lock()
+	if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
+		m.cachelock.Unlock()
+		return true
+	}
+	m.cachelock.Unlock()
+
+	if m.db.Where("Hash = ?", t.GetHash()).First(&try).RecordNotFound() {
+		return false
+	} else {
+		return true
+	}
+}
+
+func (m *Manager) StoreToot(t *toot.Toot) error {
+	nt := new(StoredToot)
+	nt.UUID = uuid.New()
+	nt.ServerCreated = t.Parsed.CreatedAt
+	nt.Original = t.Original
+	// FIXME add better validation to the parsed stuff here
+	nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost))
+	nt.URL = t.Parsed.URL
+	nt.Content = []byte(t.Parsed.Content)
+	// FIXME replace tags with spaces, don't just strip them, otherwise text
+	// gets messed up.
+	nt.TextContent = []byte(html.UnescapeString(hstg.StripTags(t.Parsed.Content)))
+	nt.Hostname = strings.ToLower(t.FromHost)
+	nt.Hash = t.GetHash()
+
+	//TODO: detect hashtags and insert hashtag records
+	//TODO: detect URLs and insert URL records
+
+	r := m.db.Create(&nt)
+
+	// put it in the cache to avoid relying on db for dedupe:
+	m.cachelock.Lock()
+	m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
+	m.cachelock.Unlock()
+
+	//panic(fmt.Sprintf("%+v", t))
+	return r.Error
+}
+
+func (m *Manager) StoreToots(tc []*toot.Toot) error {
+	for _, item := range tc {
+		err := m.StoreToot(item)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.1'
+
+services:
+  postgres:
+    image: postgres:12
+    restart: always
+    container_name: postgres
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: feta_user
+      POSTGRES_DB: feta
+  pgweb:
+    image: sosedoff/pgweb
+    restart: always
+    container_name: pgweb
+    ports: 
+        - "8081:8081" 
+    links: 
+        - postgres:postgres
+    environment:
+        - DATABASE_URL=postgres://feta_user:password@postgres:5432/feta?sslmode=disable
+    depends_on:
+        - postgres

go.mod

@@ -3,16 +3,41 @@ module git.eeqj.de/sneak/feta
 go 1.14
 
 require (
-	github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
+	git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
+	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gin-gonic/gin v1.6.2
+	github.com/golang/protobuf v1.3.5 // indirect
 	github.com/google/uuid v1.1.1
+	github.com/grokify/html-strip-tags-go v0.0.0-20200322061010-ea0c1cf2f119
 	github.com/jinzhu/gorm v1.9.12
+	github.com/juju/errors v0.0.0-20200330140219-3fe23663418f // indirect
 	github.com/k0kubun/pp v3.0.1+incompatible
+	github.com/labstack/echo v3.3.10+incompatible
+	github.com/labstack/gommon v0.3.0
 	github.com/looplab/fsm v0.1.0
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mattn/go-isatty v0.0.12
+	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
+	github.com/mayowa/echo-pongo2 v0.0.0-20170410154925-661ce95e1767
+	github.com/minio/sha256-simd v0.1.1 // indirect
+	github.com/mitchellh/mapstructure v1.2.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/multiformats/go-multihash v0.0.13
+	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/rs/zerolog v1.18.0
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.6.2
+	github.com/valyala/fasttemplate v1.1.0 // indirect
+	github.com/ziflex/lecho v1.2.0
+	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+	golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
+	gopkg.in/ini.v1 v1.55.0 // indirect
+	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
 )

go.sum

@@ -1,5 +1,11 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+git.eeqj.de/sneak/goutil v0.0.0-20200330224009-e54964f792cd h1:LlQYjhr5NA5WK9f/s0QnnxHZE3YbnAhSkqQ/sJCdHk8=
+git.eeqj.de/sneak/goutil v0.0.0-20200330224009-e54964f792cd/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw=
+git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5 h1:0hBq85ulrB0pfC+lxb/UAMfGRHkykDMis0E8i2Di3zI=
+git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 h1:onmsMcmF/EIYUxqy4bWwV/tmPhJx6Y3bBNlnwANFhS4=
+github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0/go.mod h1:VTH/oZwsQNNdzVhP2R366GZGFfksyOchw5rcIj2QpLI=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -16,18 +22,24 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1 h1:qwfOp+dwJnhdRFWsXkRMb+EZz0BgMQ8VD77OgBjuRUQ=
 github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1/go.mod h1:AAlcXL9Ejp3TUsJRWJtjbIpK3p1L9z987raCTYL17j4=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc=
+github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM=
 github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -44,12 +56,15 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -57,9 +72,13 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grokify/html-strip-tags-go v0.0.0-20200322061010-ea0c1cf2f119 h1:h3iGUlU8HyW4baKd6D+h1mwOHnM2kwskSuG6Bv4tSbc=
+github.com/grokify/html-strip-tags-go v0.0.0-20200322061010-ea0c1cf2f119/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4 h1:60gBOooTSmNtrqNaRvrDbi8VAne0REaek2agjnITKSw=
+github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q=
@@ -71,6 +90,12 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok=
+github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/errors v0.0.0-20200330140219-3fe23663418f h1:MCOvExGLpaSIzLYB4iQXEHP4jYVU6vmzLNQPdMVrxnM=
+github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
+github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/k0kubun/pp v1.3.0 h1:r9td75hcmetrcVbmsZRjnxcIbI9mhm+/N6iWyG4TWe0=
 github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
@@ -82,30 +107,53 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI=
+github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
+github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
+github.com/labstack/gommon v0.2.9/go.mod h1:E8ZTmW9vw5az5/ZyHWCp0Lw4OH2ecsaBP1C/NKavGG4=
+github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20=
 github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
 github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
+github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mayowa/echo-pongo2 v0.0.0-20170410154925-661ce95e1767 h1:T8fARjLT0o9OkyYmCAm3UuPVWm6/8yTAt4rmmWVTORI=
+github.com/mayowa/echo-pongo2 v0.0.0-20170410154925-661ce95e1767/go.mod h1:JCIHkkBgcmXEr/rc3YhaxQ9MgZ4GJe7wjEV/is2uuag=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
+github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
@@ -116,6 +164,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
+github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -130,6 +180,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg=
 github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
 github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -141,16 +192,25 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
 github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -162,9 +222,18 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4=
+github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+github.com/ziflex/lecho v1.2.0 h1:/ykfd7V/aTsWUYNFimgbdhUiEMnWzvNaCxtbM/LX5F8=
+github.com/ziflex/lecho v1.2.0/go.mod h1:oUdYNxzLC78HCV0lpVUZR8dF4fGtEV+EEwcMV7AAHBc=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -173,10 +242,18 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
 golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -185,11 +262,16 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -197,22 +279,38 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZAEaqMN/RW+OTZkBzmLE=
+golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74 h1:4cFkmztxtMslUX2SctSl+blCyXfpzhGOy9LhKAqSMA4=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 h1:XpToik3MpT5iW3iHgNwnh3a8QwugfomvxOlyDnaOils=
+golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -223,6 +321,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
+gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

ingester/ingester.go

@@ -1,9 +1,12 @@
 package ingester
 
-import "time"
+import (
-import "github.com/rs/zerolog/log"
+	"time"
-import "git.eeqj.de/sneak/feta/toot"
+
-import "git.eeqj.de/sneak/feta/storage"
+	"git.eeqj.de/sneak/feta/storage"
+	"git.eeqj.de/sneak/feta/toot"
+	"github.com/rs/zerolog/log"
+)
 
 // TootIngester is the data structure for the ingester process that is
 // responsible for storing the discovered toots
@@ -15,7 +18,7 @@ type TootIngester struct {
 
 type seenTootMemo struct {
 	lastSeen time.Time
-	tootHash toot.Hash
+	tootHash string
 }
 
 // NewTootIngester returns a fresh TootIngester for your use
@@ -55,5 +58,7 @@ func (ti *TootIngester) storeToot(t *toot.Toot) {
 	if ti.storageBackend == nil {
 		panic("no storage backend")
 	}
-	ti.storageBackend.StoreToot(*t)
+	if !ti.storageBackend.TootExists(t) {
+		ti.storageBackend.StoreToot(t)
+	}
 }

instance/instance.go

@@ -11,64 +11,63 @@ import (
 	"time"
 
 	"git.eeqj.de/sneak/feta/jsonapis"
-	"git.eeqj.de/sneak/feta/storage"
 	"git.eeqj.de/sneak/feta/toot"
 	"github.com/google/uuid"
 	"github.com/looplab/fsm"
 	"github.com/rs/zerolog/log"
 )
 
-//import "github.com/gin-gonic/gin"
-
 const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
-const instanceNodeinfoTimeout = time.Second * 50
+const instanceNodeinfoTimeout = time.Second * 60 * 2        // 2m
-const instanceHTTPTimeout = time.Second * 120
+const instanceHTTPTimeout = time.Second * 60 * 2            // 2m
-const instanceSpiderInterval = time.Second * 120
+const instanceSpiderInterval = time.Second * 60 * 2         // 2m
-const instanceErrorInterval = time.Second * 60 * 30
+const instanceErrorInterval = time.Second * 60 * 60         // 1h
-
+const instancePersistentErrorInterval = time.Second * 86400 // 1d
-type instanceImplementation int
+const zeroInterval = time.Second * 0                        // 0s
-
-// Hostname is a special type for holding the hostname of an
-// instance (string)
-type Hostname string
-
-const (
-	implUnknown instanceImplementation = iota
-	implMastodon
-	implPleroma
-)
 
 // Instance stores all the information we know about an instance
 type Instance struct {
-	Identifier                 uuid.UUID
+	Disabled                   bool
-	structLock                 sync.Mutex
-	tootDestination            chan *toot.Toot
 	ErrorCount                 uint
-	SuccessCount               uint
+	ConsecutiveErrorCount      uint
-	highestID                  int
+	FSM                        *fsm.FSM
+	Fetching                   bool
+	HighestID                  uint
 	Hostname                   string
 	Identified                 bool
-	fetching                   bool
+	Implementation             string
-	disabled                   bool
+	InitialFSMState            string
-	implementation             instanceImplementation
-	storageBackend             *storage.TootStorageBackend
 	NextFetch                  time.Time
-	nodeInfoURL                string
+	LastError                  string
-	ServerVersionString        string
+	NodeInfoURL                string
 	ServerImplementationString string
+	ServerVersionString        string
+	SuccessCount               uint
+	UUID                       uuid.UUID
 	fetchingLock               sync.Mutex
-	fsm                        *fsm.FSM
 	fsmLock                    sync.Mutex
+	structLock                 sync.Mutex
+	tootDestination            chan *toot.Toot
 }
 
 // New returns a new instance, argument is a function that operates on the
 // new instance
 func New(options ...func(i *Instance)) *Instance {
 	i := new(Instance)
+	i.UUID = uuid.New()
 	i.setNextFetchAfter(1 * time.Second)
+	i.InitialFSMState = "STATUS_UNKNOWN"
 
-	i.fsm = fsm.NewFSM(
+	for _, opt := range options {
-		"STATUS_UNKNOWN",
+		opt(i)
+	}
+
+	if i.InitialFSMState == "FETCHING" {
+		i.InitialFSMState = "READY_FOR_TOOTFETCH"
+	}
+
+	i.FSM = fsm.NewFSM(
+		i.InitialFSMState,
 		fsm.Events{
 			{Name: "BEGIN_NODEINFO_URL_FETCH", Src: []string{"STATUS_UNKNOWN"}, Dst: "FETCHING_NODEINFO_URL"},
 			{Name: "GOT_NODEINFO_URL", Src: []string{"FETCHING_NODEINFO_URL"}, Dst: "PRE_NODEINFO_FETCH"},
@@ -80,15 +79,13 @@ func New(options ...func(i *Instance)) *Instance {
 			{Name: "EARLY_FETCH_ERROR", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "EARLY_ERROR"},
 			{Name: "TOOT_FETCH_ERROR", Src: []string{"FETCHING"}, Dst: "TOOT_FETCH_ERROR"},
 			{Name: "TOOTS_FETCHED", Src: []string{"FETCHING"}, Dst: "READY_FOR_TOOTFETCH"},
+			{Name: "DISABLEMENT", Src: []string{"WEIRD_NODE", "EARLY_ERROR", "TOOT_FETCH_ERROR"}, Dst: "DISABLED"},
 		},
 		fsm.Callbacks{
 			"enter_state": func(e *fsm.Event) { i.fsmEnterState(e) },
 		},
 	)
 
-	for _, opt := range options {
-		opt(i)
-	}
 	return i
 }
 
@@ -96,7 +93,7 @@ func New(options ...func(i *Instance)) *Instance {
 func (i *Instance) Status() string {
 	i.fsmLock.Lock()
 	defer i.fsmLock.Unlock()
-	return i.fsm.Current()
+	return i.FSM.Current()
 }
 
 // SetTootDestination takes a channel from the manager that all toots
@@ -111,7 +108,7 @@ func (i *Instance) SetTootDestination(d chan *toot.Toot) {
 func (i *Instance) Event(eventname string) {
 	i.fsmLock.Lock()
 	defer i.fsmLock.Unlock()
-	i.fsm.Event(eventname)
+	i.FSM.Event(eventname)
 }
 
 func (i *Instance) fsmEnterState(e *fsm.Event) {
@@ -131,10 +128,36 @@ func (i *Instance) Unlock() {
 	i.structLock.Unlock()
 }
 
-func (i *Instance) bumpFetch() {
+func (i *Instance) bumpFetchError() {
 	i.Lock()
-	defer i.Unlock()
+	probablyDead := i.ConsecutiveErrorCount > 3
-	i.NextFetch = time.Now().Add(120 * time.Second)
+	shouldDisable := i.ConsecutiveErrorCount > 6
+	i.Unlock()
+
+	if shouldDisable {
+		// auf wiedersehen, felicia
+		i.Lock()
+		i.Disabled = true
+		i.Unlock()
+		i.Event("DISABLEMENT")
+		return
+	}
+
+	if probablyDead {
+		// if three consecutive fetch errors happen, only try once per day:
+		i.setNextFetchAfter(instancePersistentErrorInterval)
+	} else {
+		// otherwise give them 1h
+		i.setNextFetchAfter(instanceErrorInterval)
+	}
+}
+
+func (i *Instance) bumpFetchSuccess() {
+	i.setNextFetchAfter(instanceSpiderInterval)
+}
+
+func (i *Instance) scheduleFetchImmediate() {
+	i.setNextFetchAfter(zeroInterval)
 }
 
 func (i *Instance) setNextFetchAfter(d time.Duration) {
@@ -149,8 +172,7 @@ func (i *Instance) Fetch() {
 	i.fetchingLock.Lock()
 	defer i.fetchingLock.Unlock()
 
-	i.setNextFetchAfter(instanceErrorInterval)
+	i.bumpFetchError()
-
 	err := i.DetectNodeTypeIfNecessary()
 	if err != nil {
 		log.Debug().
@@ -159,8 +181,7 @@ func (i *Instance) Fetch() {
 			Msg("unable to fetch instance metadata")
 		return
 	}
-
+	i.scheduleFetchImmediate()
-	i.setNextFetchAfter(instanceSpiderInterval)
 	log.Info().
 		Str("hostname", i.Hostname).
 		Msg("instance now ready for fetch")
@@ -198,7 +219,7 @@ func (i *Instance) Tick() {
 func (i *Instance) nodeIdentified() bool {
 	i.Lock()
 	defer i.Unlock()
-	if i.implementation > implUnknown {
+	if i.Implementation != "" {
 		return true
 	}
 	return false
@@ -217,12 +238,14 @@ func (i *Instance) registerError() {
 	i.Lock()
 	defer i.Unlock()
 	i.ErrorCount++
+	i.ConsecutiveErrorCount++
 }
 
 func (i *Instance) registerSuccess() {
 	i.Lock()
 	defer i.Unlock()
 	i.SuccessCount++
+	i.ConsecutiveErrorCount = 0
 }
 
 // Up returns true if the success count is >0
@@ -288,7 +311,7 @@ func (i *Instance) fetchNodeInfoURL() error {
 				Msg("success fetching url for nodeinfo")
 
 			i.Lock()
-			i.nodeInfoURL = item.Href
+			i.NodeInfoURL = item.Href
 			i.Unlock()
 			i.registerSuccess()
 			i.Event("GOT_NODEINFO_URL")
@@ -323,7 +346,7 @@ func (i *Instance) fetchNodeInfo() error {
 	//FIXME make sure the nodeinfourl is on the same domain as the instance
 	//hostname
 	i.Lock()
-	url := i.nodeInfoURL
+	url := i.NodeInfoURL
 	i.Unlock()
 
 	i.Event("BEGIN_NODEINFO_FETCH")
@@ -368,7 +391,7 @@ func (i *Instance) fetchNodeInfo() error {
 		Str("serverVersion", ni.Software.Version).
 		Str("software", ni.Software.Name).
 		Str("hostname", i.Hostname).
-		Str("nodeInfoURL", i.nodeInfoURL).
+		Str("nodeInfoURL", i.NodeInfoURL).
 		Msg("received nodeinfo from instance")
 
 	i.Lock()
@@ -382,7 +405,7 @@ func (i *Instance) fetchNodeInfo() error {
 			Str("software", ni.Software.Name).
 			Msg("detected server software")
 		i.Identified = true
-		i.implementation = implPleroma
+		i.Implementation = "pleroma"
 		i.Unlock()
 		i.registerSuccess()
 		i.Event("GOT_NODEINFO")
@@ -393,7 +416,7 @@ func (i *Instance) fetchNodeInfo() error {
 			Str("software", ni.Software.Name).
 			Msg("detected server software")
 		i.Identified = true
-		i.implementation = implMastodon
+		i.Implementation = "mastodon"
 		i.Unlock()
 		i.registerSuccess()
 		i.Event("GOT_NODEINFO")
@@ -415,9 +438,12 @@ func (i *Instance) fetchRecentToots() error {
 
 	// it turns out pleroma supports the mastodon api so we'll just use that
 	// for everything for now
+
+	// FIXME would be nice to support non-https
 	url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true",
 		i.Hostname)
 
+	// FIXME support broken/expired certs
 	var c = &http.Client{
 		Timeout: instanceHTTPTimeout,
 	}
@@ -471,7 +497,7 @@ func (i *Instance) fetchRecentToots() error {
 		Msgf("got and parsed toots")
 	i.registerSuccess()
 	i.Event("TOOTS_FETCHED")
-	i.setNextFetchAfter(instanceSpiderInterval)
+	i.bumpFetchSuccess()
 
 	// this should go fast as either the channel is buffered bigly or the
 	// ingester receives fast and does its own buffering, but run it in its

locator/locator.go

@@ -7,15 +7,12 @@ import (
 	"sync"
 	"time"
 
-	"git.eeqj.de/sneak/feta/instance"
 	"git.eeqj.de/sneak/feta/jsonapis"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 	"golang.org/x/sync/semaphore"
 )
 
-//import "git.eeqj.de/sneak/feta"
-
 // IndexAPITimeout is the timeout for fetching json instance lists
 // from the listing servers
 const IndexAPITimeout = time.Second * 60 * 3
@@ -39,7 +36,7 @@ const pleromaIndexURL = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api
 type InstanceLocator struct {
 	pleromaIndexNextRefresh  *time.Time
 	mastodonIndexNextRefresh *time.Time
-	reportInstanceVia        chan instance.Hostname
+	reportInstanceVia        chan string
 	mu                       sync.Mutex
 }
 
@@ -62,13 +59,13 @@ func (il *InstanceLocator) unlock() {
 
 // SetInstanceNotificationChannel is the way the instanceLocator returns
 // newly discovered instances back to the manager for query/addition
-func (il *InstanceLocator) SetInstanceNotificationChannel(via chan instance.Hostname) {
+func (il *InstanceLocator) SetInstanceNotificationChannel(via chan string) {
 	il.lock()
 	defer il.unlock()
 	il.reportInstanceVia = via
 }
 
-func (il *InstanceLocator) addInstance(hostname instance.Hostname) {
+func (il *InstanceLocator) addInstance(hostname string) {
 	// receiver (InstanceManager) is responsible for de-duping against its
 	// map, we just locate and spray, it manages
 	il.reportInstanceVia <- hostname
@@ -201,7 +198,7 @@ func (il *InstanceLocator) locateMastodon() {
 		Msg("received hosts from mastodon index")
 
 	for k := range hosts {
-		il.addInstance(instance.Hostname(k))
+		il.addInstance(k)
 	}
 
 }
@@ -269,7 +266,7 @@ func (il *InstanceLocator) locatePleroma() {
 		Msg("received hosts from pleroma index")
 
 	for k := range hosts {
-		il.addInstance(instance.Hostname(k))
+		il.addInstance(k)
 	}
 
 }

manager/manager.go

@@ -11,31 +11,65 @@ import (
 	"github.com/spf13/viper"
 )
 
-//import "github.com/gin-gonic/gin"
+// conform for storing toots
-
+type DatabaseStorage interface {
-// LogReportInterval defines how long between logging internal
+	ListInstances() ([]*instance.Instance, error)
-// stats/reporting for user supervision
+	//StoreInstances([]*instance.Instance) error
-var LogReportInterval = time.Second * 10
+	SaveInstance(*instance.Instance) error
+}
 
 // InstanceManager is the main data structure for the goroutine that manages
 // the list of all known instances, fed by the locator
 type InstanceManager struct {
 	mu                       sync.Mutex
-	instances                map[instance.Hostname]*instance.Instance
+	db                       DatabaseStorage
-	newInstanceNotifications chan instance.Hostname
+	instances                map[string]*instance.Instance
+	newInstanceNotifications chan string
 	tootDestination          chan *toot.Toot
 	startup                  time.Time
-	hostDiscoveryParallelism int
 	hostAdderSemaphore       chan bool
+	nextDBSave               time.Time
 }
 
 // New returns a new InstanceManager for use by the Process
-func New() *InstanceManager {
+func New(db DatabaseStorage) *InstanceManager {
-	i := new(InstanceManager)
+	im := new(InstanceManager)
-	i.hostDiscoveryParallelism = viper.GetInt("HostDiscoveryParallelism")
+	im.db = db
-	i.hostAdderSemaphore = make(chan bool, i.hostDiscoveryParallelism)
+	im.hostAdderSemaphore = make(chan bool, viper.GetInt("HostDiscoveryParallelism"))
-	i.instances = make(map[instance.Hostname]*instance.Instance)
+	im.instances = make(map[string]*instance.Instance)
-	return i
+	im.RestoreFromDB()
+	return im
+}
+
+func (im *InstanceManager) RestoreFromDB() {
+	newil, err := im.db.ListInstances()
+	if err != nil {
+		log.Panic().
+			Err(err).
+			Msg("cannot get instance list from db")
+	}
+	im.lock()
+	defer im.unlock()
+	count := 0
+	for _, x := range newil {
+		x.SetTootDestination(im.tootDestination)
+		im.instances[x.Hostname] = x
+		count = count + 1
+	}
+	log.Info().
+		Int("count", count).
+		Msg("restored instances from database")
+}
+
+func (im *InstanceManager) SaveToDB() {
+	for _, x := range im.ListInstances() {
+		err := im.db.SaveInstance(x)
+		if err != nil {
+			log.Panic().
+				Err(err).
+				Msg("cannot write to db")
+		}
+	}
 }
 
 // SetTootDestination provides the instancemanager with a channel to the
@@ -56,7 +90,7 @@ func (im *InstanceManager) unlock() {
 // InstanceManager about the channel from the InstanceLocator so that the
 // InstanceLocator can provide it/us (the InstanceManager) with new
 // instance.Hostnames.  We (the manager) deduplicate the list ourselves.
-func (im *InstanceManager) SetInstanceNotificationChannel(via chan instance.Hostname) {
+func (im *InstanceManager) SetInstanceNotificationChannel(via chan string) {
 	im.lock()
 	defer im.unlock()
 	im.newInstanceNotifications = via
@@ -64,9 +98,9 @@ func (im *InstanceManager) SetInstanceNotificationChannel(via chan instance.Host
 
 func (im *InstanceManager) receiveSeedInstanceHostnames() {
 	for _, x := range seeds.SeedInstances {
-		go func(tmp instance.Hostname) {
+		go func(tmp string) {
 			im.addInstanceByHostname(tmp)
-		}(instance.Hostname(x))
+		}(x)
 	}
 }
 
@@ -89,10 +123,15 @@ func (im *InstanceManager) Manage() {
 		log.Info().Msg("InstanceManager tick")
 		im.managerLoop()
 		time.Sleep(1 * time.Second)
-		if time.Now().After(x.Add(LogReportInterval)) {
+		if time.Now().After(x.Add(viper.GetDuration("LogReportInterval"))) {
 			x = time.Now()
 			im.logInstanceReport()
 		}
+
+		if im.nextDBSave.Before(time.Now()) {
+			im.nextDBSave = time.Now().Add(time.Second * 60)
+			im.SaveToDB()
+		}
 	}
 }
 
@@ -112,7 +151,7 @@ func (im *InstanceManager) managerLoop() {
 	}
 }
 
-func (im *InstanceManager) hostnameExists(newhn instance.Hostname) bool {
+func (im *InstanceManager) hostnameExists(newhn string) bool {
 	im.lock()
 	defer im.unlock()
 	for k := range im.instances {
@@ -123,7 +162,7 @@ func (im *InstanceManager) hostnameExists(newhn instance.Hostname) bool {
 	return false
 }
 
-func (im *InstanceManager) addInstanceByHostname(newhn instance.Hostname) {
+func (im *InstanceManager) addInstanceByHostname(newhn string) {
 	if im.hostnameExists(newhn) {
 		// ignore adding new if we already know about it
 		return
@@ -151,7 +190,7 @@ func (im *InstanceManager) addInstanceByHostname(newhn instance.Hostname) {
 }
 
 func (im *InstanceManager) receiveNewInstanceHostnames() {
-	var newhn instance.Hostname
+	var newhn string
 	for {
 		newhn = <-im.newInstanceNotifications
 		// receive them fast out of the channel, let the adding function lock to add
@@ -177,7 +216,6 @@ func (im *InstanceManager) logInstanceReport() {
 // ListInstances dumps a slice of all Instances the InstanceManager knows
 // about
 func (im *InstanceManager) ListInstances() []*instance.Instance {
-	// FIXME make this pull from db
 	var out []*instance.Instance
 	im.lock()
 	defer im.unlock()
@@ -188,7 +226,6 @@ func (im *InstanceManager) ListInstances() []*instance.Instance {
 }
 
 func (im *InstanceManager) instanceSummaryReport() map[string]uint {
-	// FIXME make this pull from db
 	r := make(map[string]uint)
 	for _, v := range im.ListInstances() {
 		v.Lock()

process/feta.go

@@ -1,15 +1,16 @@
 package process
 
 import (
+	"fmt"
 	"os"
 	"time"
 
 	"git.eeqj.de/sneak/feta/database"
 	"git.eeqj.de/sneak/feta/ingester"
-	"git.eeqj.de/sneak/feta/instance"
 	"git.eeqj.de/sneak/feta/locator"
 	"git.eeqj.de/sneak/feta/manager"
 	"git.eeqj.de/sneak/feta/storage"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/k0kubun/pp"
 	"github.com/mattn/go-isatty"
@@ -53,9 +54,11 @@ func (f *Feta) configure() {
 	viper.AutomaticEnv()
 
 	viper.SetDefault("Debug", false)
+	viper.SetDefault("TootsToDisk", false)
+	viper.SetDefault("TootsToDB", true)
 	viper.SetDefault("HostDiscoveryParallelism", 5)
 	viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d"))
-	viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))
+	viper.SetDefault("DBURL", fmt.Sprintf("sqlite://%s", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db")))
 	viper.SetDefault("LogReportInterval", time.Second*10)
 
 	if err := viper.ReadInConfig(); err != nil {
@@ -118,31 +121,17 @@ func (f *Feta) uptime() time.Duration {
 	return time.Since(f.startup)
 }
 
-/*
-func (f *Feta) setupDatabase() {
-	var err error
-	f.db, err = gorm.Open("sqlite3", "feta.sqlite")
-
-
-	if err != nil {
-		panic(err)
-	}
-
-	//f.databaseMigrations()
-}
-*/
-
 func (f *Feta) runForever() int {
 	f.startup = time.Now()
 
-	//f.setupDatabase()
-
 	// FIXME move this channel creation into the manager's constructor
 	// and add getters/setters on the manager/locator
-	newInstanceHostnameNotifications := make(chan instance.Hostname)
+	newInstanceHostnameNotifications := make(chan string)
 
 	f.locator = locator.New()
-	f.manager = manager.New()
+
+	f.manager = manager.New(f.dbm)
+
 	f.ingester = ingester.NewTootIngester()
 
 	home := os.Getenv("HOME")
@@ -150,8 +139,15 @@ func (f *Feta) runForever() int {
 		panic("can't find home directory")
 	}
 
-	diskBackend := storage.NewTootFSStorage(home + "/.local/feta")
+	// TODO make the ingester support multiple storage backends simultaneously
-	f.ingester.SetStorageBackend(diskBackend)
+	if viper.GetBool("TootsToDB") {
+		f.ingester.SetStorageBackend(f.dbm)
+	} else if viper.GetBool("TootsToDisk") {
+		diskBackend := storage.NewTootFSStorage(viper.GetString("FSStorageLocation"))
+		f.ingester.SetStorageBackend(diskBackend)
+	} else {
+		log.Info().Msg("toots will not be saved to disk")
+	}
 
 	f.api = new(Server)
 	f.api.SetFeta(f) // api needs to get to us to access data

process/handlers.go

@@ -1,14 +1,16 @@
 package process
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"runtime"
 	"strings"
 	"time"
 
-	"github.com/gin-gonic/gin"
+	u "git.eeqj.de/sneak/goutil"
+	"github.com/flosch/pongo2"
+	"github.com/google/uuid"
+	"github.com/labstack/echo"
 )
 
 type hash map[string]interface{}
@@ -18,15 +20,23 @@ func (a *Server) instances() []hash {
 	now := time.Now()
 	for _, v := range a.feta.manager.ListInstances() {
 		i := make(hash)
-		// FIXME figure out why a very short lock here deadlocks
+		// TODO move this locking onto a method on Instance that just
+		// returns a new hash
+
+		//this only locks the FSM, not the whole instance struct
+		i["status"] = v.Status()
+
+		// now do a quick lock of the whole instance just to copy out the
+		// attrs
 		v.Lock()
 		i["hostname"] = v.Hostname
+		i["uuid"] = v.UUID.String()
 		i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
 		i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
 		i["successCount"] = v.SuccessCount
 		i["errorCount"] = v.ErrorCount
+		i["consecutiveErrorCount"] = v.ConsecutiveErrorCount
 		i["identified"] = v.Identified
-		i["status"] = v.Status()
 		i["software"] = "unknown"
 		i["version"] = "unknown"
 		if v.Identified {
@@ -34,82 +44,109 @@ func (a *Server) instances() []hash {
 			i["version"] = v.ServerVersionString
 		}
 		v.Unlock()
+
 		resp = append(resp, i)
 	}
+
+	for _, item := range resp {
+		count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
+		item["tootCount"] = 0
+		if err != nil {
+			item["tootCount"] = count
+		}
+	}
 	return resp
 }
 
-func (a *Server) instanceSummary() map[string]int {
+func (a *Server) instanceStatusSummary() map[string]int {
 	resp := make(map[string]int)
 	for _, v := range a.feta.manager.ListInstances() {
 		v.Lock()
 		resp[fmt.Sprintf("STATUS_%s", v.Status())]++
 		if v.ServerImplementationString != "" {
-			//FIXME(sneak) sanitize this to a-z0-9, it is server-provided
+			impl := strings.ToUpper(u.FilterToAlnum(v.ServerImplementationString))
-			resp[fmt.Sprintf("SOFTWARE_%s", strings.ToUpper(v.ServerImplementationString))]++
+			resp[fmt.Sprintf("SOFTWARE_%s", impl)]++
 		}
 		v.Unlock()
 	}
 	return resp
 }
 
-func (a *Server) getInstanceListHandler() http.HandlerFunc {
+func (a *Server) notFoundHandler(c echo.Context) error {
-	return func(w http.ResponseWriter, r *http.Request) {
+	return c.String(http.StatusNotFound, "404 not found")
-
-		result := &gin.H{
-			"instances": a.instances(),
-		}
-
-		json, err := json.Marshal(result)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		w.Write(json)
-	}
 }
 
-func (a *Server) getIndexHandler() http.HandlerFunc {
+func (a *Server) instanceHandler(c echo.Context) error {
-	return func(w http.ResponseWriter, r *http.Request) {
+	tu := c.Param("uuid")
-		index := &gin.H{
+	u, err := uuid.Parse(tu)
-			"server": &gin.H{
+	if err != nil {
-				"now":        time.Now().UTC().Format(time.RFC3339),
+		return a.notFoundHandler(c)
-				"uptime":     a.feta.uptime().String(),
-				"goroutines": runtime.NumGoroutine(),
-				"goversion":  runtime.Version(),
-				"version":    a.feta.version,
-				"buildarch":  a.feta.buildarch,
-			},
-			"instanceSummary": a.instanceSummary(),
-		}
-
-		json, err := json.Marshal(index)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		w.Write(json)
 	}
+
+	tc := pongo2.Context{}
+	instances := a.feta.manager.ListInstances()
+	found := false
+	for _, item := range instances {
+		if item.UUID == u {
+			tc["instance"] = item
+			found = true
+		}
+	}
+
+	if !found {
+		return a.notFoundHandler(c)
+	}
+
+	return c.Render(http.StatusOK, "instance.html", tc)
 }
 
-func (a *Server) getHealthCheckHandler() http.HandlerFunc {
+func (a *Server) indexHandler(c echo.Context) error {
-	return func(w http.ResponseWriter, r *http.Request) {
+	count, err := a.feta.dbm.TotalTootCount()
-		resp := &gin.H{
+	if err != nil {
-			"status": "ok",
+		count = 0
-			"now":    time.Now().UTC().Format(time.RFC3339),
-			"uptime": a.feta.uptime().String(),
-		}
-
-		json, err := json.Marshal(resp)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.Write(json)
 	}
+	tc := pongo2.Context{
+		"time":                  time.Now().UTC().Format(time.RFC3339Nano),
+		"gitrev":                a.feta.version,
+		"tootCount":             count,
+		"instances":             a.instances(),
+		"instanceStatusSummary": a.instanceStatusSummary(),
+	}
+	return c.Render(http.StatusOK, "index.html", tc)
+}
+
+func (a *Server) instanceListHandler(c echo.Context) error {
+	il := a.instances()
+	tc := pongo2.Context{
+		"time":      time.Now().UTC().Format(time.RFC3339Nano),
+		"gitrev":    a.feta.version,
+		"instances": il,
+	}
+	return c.Render(http.StatusOK, "instancelist.html", tc)
+}
+
+func (a *Server) statsHandler(c echo.Context) error {
+	index := &hash{
+		"server": &hash{
+			"now":        time.Now().UTC().Format(time.RFC3339),
+			"uptime":     a.feta.uptime().String(),
+			"goroutines": runtime.NumGoroutine(),
+			"goversion":  runtime.Version(),
+			"version":    a.feta.version,
+			"buildarch":  a.feta.buildarch,
+		},
+		"instanceStatusSummary": a.instanceStatusSummary(),
+	}
+
+	return c.JSONPretty(http.StatusOK, index, "    ")
+}
+
+func (a *Server) healthCheckHandler(c echo.Context) error {
+	resp := &hash{
+		"status": "ok",
+		"now":    time.Now().UTC().Format(time.RFC3339),
+		"uptime": a.feta.uptime().String(),
+	}
+	return c.JSONPretty(http.StatusOK, resp, "    ")
+
 }

process/server.go

@@ -1,22 +1,33 @@
 package process
 
-import "fmt"
+import (
-import "net/http"
+	"fmt"
-import "os"
+	"net/http"
-import "strconv"
+	"os"
-import "time"
+	"strconv"
+	"time"
 
-import "github.com/rs/zerolog/log"
+	"github.com/gin-gonic/gin"
-import "github.com/gin-gonic/gin"
+	"github.com/labstack/echo"
-import "github.com/dn365/gin-zerolog"
+	"github.com/labstack/echo/middleware"
+	gl "github.com/labstack/gommon/log"
+	ep2 "github.com/mayowa/echo-pongo2"
+	"github.com/ziflex/lecho"
+
+	//"github.com/gin-gonic/gin"
+	"github.com/jinzhu/gorm"
+	"github.com/rs/zerolog/log"
+)
 
 // Server is the HTTP webserver object
 type Server struct {
-	feta   *Feta
+	feta       *Feta
-	port   uint
+	port       uint
-	router *gin.Engine
+	e          *echo.Echo
-	server *http.Server
+	router     *gin.Engine
-	debug  bool
+	httpserver *http.Server
+	debug      bool
+	db         *gorm.DB
 }
 
 // SetFeta tells the http Server where to find the Process object so that it
@@ -28,6 +39,7 @@ func (a *Server) SetFeta(feta *Feta) {
 // Serve is the entrypoint for the Server, which should run in its own
 // goroutine (started by the Process)
 func (a *Server) Serve() {
+
 	if a.feta == nil {
 		panic("must have feta app from which to serve stats")
 	}
@@ -47,43 +59,55 @@ func (a *Server) Serve() {
 
 	a.initRouter()
 	a.initServer()
-	err := a.server.ListenAndServe()
+	a.e.Logger.Fatal(a.e.StartServer(a.httpserver))
+}
 
+func (s *Server) initRouter() {
+
+	// Echo instance
+	s.e = echo.New()
+
+	s.e.HideBanner = true
+
+	lev := gl.INFO
+	if os.Getenv("DEBUG") != "" {
+		lev = gl.DEBUG
+	}
+
+	logger := lecho.New(
+		os.Stdout,
+		lecho.WithLevel(lev),
+		lecho.WithTimestamp(),
+		lecho.WithCaller(),
+	)
+	s.e.Logger = logger
+	s.e.Use(middleware.RequestID())
+
+	// Middleware
+	s.e.Use(middleware.Logger())
+	s.e.Use(middleware.Recover())
+
+	r, err := ep2.NewRenderer("view")
 	if err != nil {
-		log.Fatal().Err(err).Msg("webserver failure")
+		s.e.Logger.Fatal(err)
-		return
 	}
+	s.e.Renderer = r
+
+	// Routes
+	s.e.GET("/", s.indexHandler)
+	s.e.GET("/instance/:uuid", s.instanceHandler)
+	s.e.GET("/instances", s.instanceListHandler)
+	s.e.GET("/stats.json", s.statsHandler)
+	s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
+	//a.e.GET("/about", s.aboutHandler)
+
 }
 
-func (a *Server) initRouter() {
+func (s *Server) initServer() {
+	log.Info().Uint("port", s.port).Msg("starting webserver")
 
-	// empty router
+	s.httpserver = &http.Server{
-	r := gin.New()
+		Addr:           fmt.Sprintf(":%d", s.port),
-
-	// wrap panics:
-	r.Use(gin.Recovery())
-
-	// attach logger middleware
-	r.Use(ginzerolog.Logger("gin"))
-
-	r.GET("/.well-known/healthcheck.json", gin.WrapF(a.getHealthCheckHandler()))
-	r.GET("/", func(c *gin.Context) { c.Redirect(http.StatusMovedPermanently, "/feta") })
-	r.GET("/feta", gin.WrapF(a.getIndexHandler()))
-	r.GET("/feta/list/instances", gin.WrapF(a.getInstanceListHandler()))
-
-	a.router = r
-}
-
-func (a *Server) initServer() {
-	if !a.debug {
-		gin.SetMode(gin.ReleaseMode)
-	}
-
-	log.Info().Uint("port", a.port).Msg("starting webserver")
-
-	a.server = &http.Server{
-		Addr:           fmt.Sprintf(":%d", a.port),
-		Handler:        a.router,
 		ReadTimeout:    10 * time.Second,
 		WriteTimeout:   10 * time.Second,
 		MaxHeaderBytes: 1 << 20,

seeds/seeds.go

@@ -315,6 +315,7 @@ var SeedInstances = [...]string{
 	"snuskete.net",
 	"social.1in9.net",
 	"social.adlerweb.info",
+	"social.art-software.fr",
 	"social.au2pb.net",
 	"social.avareborn.de",
 	"social.azkware.net",

storage/fs.go

@@ -11,18 +11,6 @@ import (
 	"git.eeqj.de/sneak/feta/toot"
 )
 
-// TootStorageBackend is the interface to which storage backends must
-// conform for storing toots
-type TootStorageBackend interface {
-	TootExists(t toot.Toot) bool
-	StoreToot(t toot.Toot) error
-	StoreToots(tc []*toot.Toot) error
-}
-
-type TootDBStorage struct {
-	db string
-}
-
 // TootFSStorage is a TootStorageBackend that writes to the local
 // filesystem.
 type TootFSStorage struct {
@@ -41,7 +29,7 @@ func NewTootFSStorage(root string) *TootFSStorage {
 func (ts *TootFSStorage) StoreToots(tc []*toot.Toot) error {
 	var returnErrors []string
 	for _, item := range tc {
-		err := ts.StoreToot(*item)
+		err := ts.StoreToot(item)
 		if err != nil {
 			returnErrors = append(returnErrors, err.Error())
 			continue
@@ -56,7 +44,7 @@ func (ts *TootFSStorage) StoreToots(tc []*toot.Toot) error {
 // TootExists checks to see if we have already written a toot to disk or
 // not.  Note that the ingester de-dupes with a table in memory so that this
 // will only really get used on app restarts
-func (ts *TootFSStorage) TootExists(t toot.Toot) bool {
+func (ts *TootFSStorage) TootExists(t *toot.Toot) bool {
 	path := t.DiskStoragePath()
 	full := ts.root + "/" + path
 	_, err := os.Stat(full)
@@ -67,7 +55,7 @@ func (ts *TootFSStorage) TootExists(t toot.Toot) bool {
 }
 
 // StoreToot writes a single toot to disk
-func (ts *TootFSStorage) StoreToot(t toot.Toot) error {
+func (ts *TootFSStorage) StoreToot(t *toot.Toot) error {
 	path := t.DiskStoragePath()
 	full := ts.root + "/" + path
 	dir := filepath.Dir(full)
@@ -82,7 +70,7 @@ func (ts *TootFSStorage) StoreToot(t toot.Toot) error {
 // toots in ram forever until the computer fills up and catches fire and explodes
 type TootMemoryStorage struct {
 	sync.Mutex
-	toots map[toot.Hash]toot.Toot
+	toots map[string]*toot.Toot
 	//maxSize uint   // FIXME support eviction
 }
 
@@ -90,12 +78,12 @@ type TootMemoryStorage struct {
 // ram forever
 func NewTootMemoryStorage() *TootMemoryStorage {
 	ts := new(TootMemoryStorage)
-	ts.toots = make(map[toot.Hash]toot.Toot)
+	ts.toots = make(map[string]*toot.Toot)
 	return ts
 }
 
 // StoreToot saves a single toot into an in-memory hashtable
-func (ts *TootMemoryStorage) StoreToot(t toot.Toot) {
+func (ts *TootMemoryStorage) StoreToot(t *toot.Toot) {
 	if ts.TootExists(t) {
 		return
 	}
@@ -106,7 +94,7 @@ func (ts *TootMemoryStorage) StoreToot(t toot.Toot) {
 }
 
 // TootExists checks to see if we have a toot in memory already
-func (ts *TootMemoryStorage) TootExists(t toot.Toot) bool {
+func (ts *TootMemoryStorage) TootExists(t *toot.Toot) bool {
 	ts.Lock()
 	defer ts.Unlock()
 	if _, ok := ts.toots[t.Hash]; ok { //this syntax is so gross

storage/interface.go

@@ -0,0 +1,13 @@
+package storage
+
+import (
+	"git.eeqj.de/sneak/feta/toot"
+)
+
+// TootStorageBackend is the interface to which storage backends must
+// conform for storing toots
+type TootStorageBackend interface {
+	TootExists(t *toot.Toot) bool
+	StoreToot(t *toot.Toot) error
+	StoreToots(tc []*toot.Toot) error
+}

toot/toot.go

@@ -1,27 +1,30 @@
 package toot
 
-import "fmt"
+import (
-import "encoding/json"
+	"encoding/json"
-import "errors"
+	"errors"
-import "strings"
+	"fmt"
-import "git.eeqj.de/sneak/feta/jsonapis"
+	"strings"
 
-//import "github.com/davecgh/go-spew/spew"
+	"git.eeqj.de/sneak/feta/jsonapis"
-import "github.com/rs/zerolog/log"
+	"github.com/rs/zerolog/log"
 
-//import "encoding/hex"
+	//import "github.com/davecgh/go-spew/spew"
-import mh "github.com/multiformats/go-multihash"
+
-import mhopts "github.com/multiformats/go-multihash/opts"
+	//import "encoding/hex"
+	mh "github.com/multiformats/go-multihash"
+
+	mhopts "github.com/multiformats/go-multihash/opts"
+)
 
 // Hash is a type for storing a string-based base58 multihash of a
 // toot's identity
-type Hash string
 
 // Toot is an object we use internally for storing a discovered toot
 type Toot struct {
 	Original []byte
 	Parsed   *jsonapis.APISerializedToot
-	Hash     Hash
+	Hash     string
 	FromHost string
 }
 
@@ -111,7 +114,14 @@ func (t *Toot) identityHashInput() string {
 	)
 }
 
+func (t *Toot) GetHash() string {
+	if t.Hash == "" {
+		t.calcHash()
+	}
+	return t.Hash
+}
+
 func (t *Toot) calcHash() {
 	hi := t.identityHashInput()
-	t.Hash = Hash(t.multiHash([]byte(hi)))
+	t.Hash = string(t.multiHash([]byte(hi)))
 }

view/about.html

@@ -0,0 +1,32 @@
+{% extends "page.html" %}
+
+{% block content %}
+
+<div class="col-lg-12">
+    <h1>About This Site</h1>
+
+    <h2>⚠️ Important!</h2>
+
+    <p>This is a third-party site, not affiliated in any way with HN,
+    provided for informational purposes only.  <b>Do not</b> contact or
+    hassle HN administrators about any information you learn here.
+    Their site, their rules.  I repeat: <b>Do not hassle or interrogate
+    the HN admins about their front page.</b> <small>(<a
+    href="https://news.ycombinator.com/item?id=22643777">more
+    info</a>)</small> </p>
+
+    <h2>Why?</h2>
+
+    <p>I like reading stuff on the internet that is of interest to HN voters,
+    even if the HN administrators think it's off topic and don't want it on
+    the frontpage, which is a decision I respect.  Again, their site, their
+    rules.</p>
+
+    <h2>How?</h2>
+
+    <p>Go and the
+    graciously provided <a href="https://github.com/HackerNews/API">HN
+        API</a>.</p>
+
+</div>
+{% endblock %}

view/base.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <!--
+      <meta name="description" content="">
+      <meta name="author" content="">
+    -->
+    <title>{{ htmltitle }}</title>
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
+  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+
+<style>
+{% include "style.css" %}
+</style>
+
+</head>
+
+<body>
+      {% block body %}
+      {% endblock %}
+<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+</body>
+
+</html>

view/bodyfooter.html

@@ -0,0 +1,10 @@
+<footer>
+    <small>
+    A project by <a href="https://sneak.berlin">@sneak</a>.
+   &nbsp;&nbsp;&nbsp;&nbsp;
+    <code>{{gitrev}}</code>
+   &nbsp;&nbsp;&nbsp;&nbsp;
+    <a href="http://git.eeqj.de/sneak/feta">source</a> (<a
+       href="https://en.wikipedia.org/wiki/WTFPL">WTFPL</a>)
+    </small>
+</footer>

view/index.html

@@ -0,0 +1,47 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+
+<h2>feta overview</h2>
+
+
+<div class="card m-5">
+  <h5 class="card-header">Instances</h5>
+  <div class="card-body">
+      <h5 class="card-title">Tracking {{ instances | length }} instances
+          across the Fediverse.</h5>
+    <!-- 
+        <p class="card-text">
+        </p> -->
+
+    <a href="/instances" class="btn btn-primary">View Instance List</a>
+  </div>
+</div>
+
+<div class="card m-5">
+  <h5 class="card-header">Toots</h5>
+  <div class="card-body">
+      <h5 class="card-title">I have {{ tootCount }} toots
+          in my database.</h5>
+    <a href="/toots" class="btn btn-primary">View Latest Toots</a>
+  </div>
+</div>
+
+<div class="card m-5">
+  <h5 class="card-header">Recent Events</h5>
+  <div class="card-body">
+      <h5 class="card-title">Last n System Events</h5>
+
+        <p class="card-text"> Discovered instance toot1.example.com </p>
+        <p class="card-text"> Discovered instance toot2.example.com </p>
+        <p class="card-text"> Discovered instance toot3.example.com </p>
+        <p class="card-text"> Discovered instance toot4.example.com </p>
+
+  </div>
+</div>
+
+
+</div>
+{% endblock %}

view/instance.html

@@ -0,0 +1,24 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+
+<h2>instance {{instance.hostname}}</h2>
+
+<div class="card m-5">
+  <div class="card-header">
+      <a href="/instance/{{instance.uuid}}">{{ instance.hostname }}</a>
+      ({{instance.tootCount}} toots)
+  </div>
+  <div class="card-body">
+      <h5 class="card-title">{{instance.status}}</h5>
+    <p class="card-text">First Stat</p>
+    <p class="card-text">Second Stat</p>
+    <p class="card-text">Third Stat</p>
+    <a href="https://{{instance.hostname}}" class="btn btn-primary">View Instance Website</a>
+  </div>
+</div>
+
+</div>
+{% endblock %}

view/instancelist.html

@@ -0,0 +1,36 @@
+{% extends "page.html" %}
+
+{% block content %}
+<div class="col-lg-12">
+
+<h2>instance list</h2>
+
+<table class="table table-striped table-hover">
+  <thead class="thead-dark">
+<tr>
+  <th scope="col">hostname</th>
+  <th scope="col">status</th>
+  <th scope="col">tootCount</th>
+  <th scope="col">nextFetch</th>
+  <th scope="col">Detail</th>
+</tr>
+</thead>
+<tbody>
+{% for instance in instances %}
+    <tr>
+        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
+        <td>{{instance.status}}</td>
+        <td>{{instance.tootCount}}</td>
+        <td>{{instance.nextFetch}}</td>
+        <td><a
+                href="/instance/{{instance.uuid}}"
+                class="btn btn-info">
+                <i class="fab fa-mastodon"></i>
+            </button></td>
+    </tr>
+{% endfor %}
+ </tbody>
+</table>
+
+</div>
+{% endblock %}

view/navbar.html

@@ -0,0 +1,16 @@
+<!-- Navigation -->
+  <nav class="navbar navbar-expand-lg navbar-dark bg-dark static-top">
+    <div class="container">
+      <a class="navbar-brand" href="/">Fediverse Index</a>
+      <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarResponsive" aria-controls="navbarResponsive" aria-expanded="false" aria-label="Toggle navigation">
+        <span class="navbar-toggler-icon"></span>
+      </button>
+      <div class="collapse navbar-collapse" id="navbarResponsive">
+        <ul class="navbar-nav ml-auto">
+          <li class="nav-item">
+            <a class="nav-link" href="https://sneak.berlin">Author</a>
+          </li>
+        </ul>
+      </div>
+    </div>
+  </nav>

view/page.html

@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+
+{% block body %}
+    {% include "navbar.html" %}
+    <!-- Page Content -->
+    <div class="container" id="pagebody">
+        <div class="row">
+            {% block content %}
+            <div class="col-lg-12 text-center">
+                <h1 class="mt-5">View Not Found</h1>
+            </div>
+            {% endblock %}
+        </div>
+    </div>
+    {% include "bodyfooter.html" %}
+{% endblock %}

view/style.css

@@ -0,0 +1,17 @@
+body {
+}
+
+#pagebody {
+    margin-top: 4em;
+}
+
+footer {
+    padding-left: 1em;
+    padding-top: 2em;
+    padding-bottom: 0.5em;
+}
+
+h1,h2,h3,h4 {
+    padding-top: 1em;
+    padding-bottom: 0.25em;
+}