update sums

database/imconnector.go

@@ -27,6 +27,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			Identified:                 i.Identified,
 			Implementation:             i.Implementation,
 			NextFetch:                  i.NextFetch,
+			LastError:                  i.LastError,
 			NodeInfoURL:                i.NodeInfoURL,
 			ServerImplementationString: i.ServerImplementationString,
 			ServerVersionString:        i.ServerVersionString,
@@ -49,6 +50,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		ei.Fetching = i.Fetching
 		ei.HighestID = i.HighestID
 		ei.Hostname = i.Hostname
+		ei.LastError = i.LastError
 		ei.Identified = i.Identified
 		ei.Implementation = string(i.Implementation)
 		ei.NextFetch = i.NextFetch
@@ -56,7 +58,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		ei.ServerImplementationString = i.ServerImplementationString
 		ei.ServerVersionString = i.ServerVersionString
 		ei.SuccessCount = i.SuccessCount
-
 		r := m.db.Save(&ei)
 		return r.Error
 	}
@@ -77,6 +78,7 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
 			x.Fetching = i.Fetching
 			x.HighestID = i.HighestID
 			x.Hostname = i.Hostname
+			x.LastError = i.LastError
 			x.Identified = i.Identified
 			x.Implementation = i.Implementation
 			x.NextFetch = i.NextFetch

database/manager.go

@@ -3,21 +3,24 @@ package database
 import (
 	"path/filepath"
 	"sync"
-
 	"github.com/jinzhu/gorm"
-
 	u "git.eeqj.de/sneak/goutil"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
+	"github.com/golang/groupcache/lru"
 )
 
+const cacheEntries = 1000000
+
+// Manager coordinates database and cache reads and writes
 type Manager struct {
 	db                            *gorm.DB
 	cachelock                     sync.Mutex
-	recentlyInsertedTootHashCache *map[string]bool
+	recentlyInsertedTootHashCache *lru.Cache
 }
 
+// New creates new Manager
 func New() *Manager {
 	m := new(Manager)
 	m.init()
@@ -32,9 +35,7 @@ func (m *Manager) init() {
 	if viper.GetBool("Debug") {
 		m.db.LogMode(true)
 	}
-
+	m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
-	h := make(map[string]bool)
-	m.recentlyInsertedTootHashCache = &h
 }
 
 func (m *Manager) open() {

database/model.go

@@ -21,7 +21,7 @@ type StoredToot struct {
 	Content       []byte
 	TextContent   []byte
 	URL           string
-	Hostname      string
+	Hostname      string `gorm:"index:hostnameindex"`
 }
 
 type APInstance struct {
@@ -34,6 +34,7 @@ type APInstance struct {
 	Identified                 bool
 	Fetching                   bool
 	Disabled                   bool
+	LastError                  string
 	Implementation             string
 	NextFetch                  time.Time
 	NodeInfoURL                string

database/readshortcuts.go

@@ -0,0 +1,24 @@
+package database
+
+import (
+	"github.com/google/uuid"
+)
+
+func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
+	var c uint
+	e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
+	if e.Error != nil {
+		return 0, e.Error
+	} else {
+		return c, nil
+	}
+}
+
+func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
+	var i APInstance
+	e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
+	if e.Error != nil {
+		return nil, e.Error
+	}
+	return &i, nil
+}

database/storageconnector.go

@@ -4,9 +4,7 @@ import (
 	"fmt"
 	"html"
 	"strings"
-
 	"git.eeqj.de/sneak/feta/toot"
-
 	"github.com/google/uuid"
 	hstg "github.com/grokify/html-strip-tags-go"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
@@ -15,24 +13,15 @@ import (
 func (m *Manager) TootInsertHashCacheSize() uint {
 	m.cachelock.Lock()
 	defer m.cachelock.Unlock()
-	return uint(len(*m.recentlyInsertedTootHashCache))
+	return uint(m.recentlyInsertedTootHashCache.Len())
 }
 
 func (m *Manager) TootExists(t *toot.Toot) bool {
 	var try StoredToot
 
-	// first, nuke cache if it gets too big to prevent
+	// check cache
-	// unlimited memory usage
 	m.cachelock.Lock()
-	if len(*m.recentlyInsertedTootHashCache) > 1000000 {
+	if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
-		emptycache := make(map[string]bool)
-		m.recentlyInsertedTootHashCache = &emptycache
-	}
-	m.cachelock.Unlock()
-
-	m.cachelock.Lock()
-	// check map
-	if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
 		m.cachelock.Unlock()
 		return true
 	}
@@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
 
 	// put it in the cache to avoid relying on db for dedupe:
 	m.cachelock.Lock()
-	(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
+	m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
 	m.cachelock.Unlock()
 
 	//panic(fmt.Sprintf("%+v", t))

go.mod

@@ -4,8 +4,6 @@ go 1.14
 
 require (
 	git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
-	github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
-	github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
 	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gin-gonic/gin v1.6.2
@@ -37,10 +35,9 @@ require (
 	github.com/valyala/fasttemplate v1.1.0 // indirect
 	github.com/ziflex/lecho v1.2.0
 	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
-	golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 // indirect
 	gopkg.in/ini.v1 v1.55.0 // indirect
+	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
 )

go.sum

@@ -56,6 +56,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

instance/instance.go

@@ -37,6 +37,7 @@ type Instance struct {
 	Implementation             string
 	InitialFSMState            string
 	NextFetch                  time.Time
+	LastError                  string
 	NodeInfoURL                string
 	ServerImplementationString string
 	ServerVersionString        string

process/handlers.go

@@ -20,15 +20,19 @@ func (a *Server) instances() []hash {
 	now := time.Now()
 	for _, v := range a.feta.manager.ListInstances() {
 		i := make(hash)
+		// TODO move this locking onto a method on Instance that just
+		// returns a new hash
 		// FIXME figure out why a very short lock here deadlocks
 		v.Lock()
 		i["hostname"] = v.Hostname
+		i["uuid"] = v.UUID.String()
 		i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
 		i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
 		i["successCount"] = v.SuccessCount
 		i["errorCount"] = v.ErrorCount
 		i["identified"] = v.Identified
-		i["status"] = v.Status() //FIXME maybe this is why
+		//this only locks the FSM, not the whole instance struct
+		i["status"] = v.Status()
 		i["software"] = "unknown"
 		i["version"] = "unknown"
 		if v.Identified {
@@ -38,6 +42,14 @@ func (a *Server) instances() []hash {
 		v.Unlock()
 		resp = append(resp, i)
 	}
+
+	for _, item := range resp {
+		count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
+		item["tootCount"] = 0
+		if err != nil {
+			item["tootCount"] = count
+		}
+	}
 	return resp
 }
 
@@ -75,10 +87,27 @@ func (a *Server) getInstanceListHandler() http.HandlerFunc {
 }
 */
 
+func (a *Server) instanceHandler(c echo.Context) error {
+	tu := c.Param("uuid")
+	u, err := uuid.Parse(tu)
+	if err != nil {
+		return c.String(http.StatusNotFound, "404 not found")
+	}
+
+
+	i := a.feta.manager.ListInstances() {
+	tc := pongo2.Context{
+		"instance": i,
+	}
+
+	return c.Render(http.StatusOK, "instance.html", tc)
+}
+
 func (a *Server) indexHandler(c echo.Context) error {
 	tc := pongo2.Context{
 		"time":      time.Now().UTC().Format(time.RFC3339Nano),
 		"gitrev":    a.feta.version,
+		"instances": a.instances(),
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }

process/server.go

@@ -95,6 +95,7 @@ func (s *Server) initRouter() {
 
 	// Routes
 	s.e.GET("/", s.indexHandler)
+	s.e.GET("/instance/:uuid", s.instanceHandler)
 	s.e.GET("/stats.json", s.statsHandler)
 	s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
 	//a.e.GET("/about", s.aboutHandler)

seeds/seeds.go

@@ -315,6 +315,7 @@ var SeedInstances = [...]string{
 	"snuskete.net",
 	"social.1in9.net",
 	"social.adlerweb.info",
+	"social.art-software.fr",
 	"social.au2pb.net",
 	"social.avareborn.de",
 	"social.azkware.net",

view/base.html

@@ -10,6 +10,7 @@
       <meta name="author" content="">
     -->
     <title>{{ htmltitle }}</title>
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
   <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
 
 <style>

view/index.html

@@ -9,17 +9,25 @@
 <table class="table table-striped table-hover">
   <thead class="thead-dark">
 <tr>
-  <th scope="col">Key</th>
+  <th scope="col">instance id</th>
-  <th scope="col">Value</th>
+  <th scope="col">hostname</th>
+  <th scope="col">status</th>
+  <th scope="col">tootCount</th>
+  <th scope="col">Detail</th>
 </tr>
 </thead>
 <tbody>
-{% for stat in stats %}
+{% for instance in instances %}
     <tr>
-        {{stat.key}}
+        <td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
-    </tr>
+        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
-    <tr>
+        <td>{{instance.status}}</td>
-        {{stat.value}}
+        <td>{{instance.tootCount}}</td>
+        <td><a
+                href="/instance/{{instance.uuid}}"
+                class="btn btn-info">
+                <i class="fab fa-mastodon"></i>
+            </button></td>
     </tr>
 {% endfor %}
  </tbody>