database/imconnector.go

@@ -27,7 +27,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 			Identified:                 i.Identified,
 			Implementation:             i.Implementation,
 			NextFetch:                  i.NextFetch,
-			LastError:                  i.LastError,
 			NodeInfoURL:                i.NodeInfoURL,
 			ServerImplementationString: i.ServerImplementationString,
 			ServerVersionString:        i.ServerVersionString,
@@ -50,7 +49,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		ei.Fetching = i.Fetching
 		ei.HighestID = i.HighestID
 		ei.Hostname = i.Hostname
-		ei.LastError = i.LastError
 		ei.Identified = i.Identified
 		ei.Implementation = string(i.Implementation)
 		ei.NextFetch = i.NextFetch
@@ -58,6 +56,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
 		ei.ServerImplementationString = i.ServerImplementationString
 		ei.ServerVersionString = i.ServerVersionString
 		ei.SuccessCount = i.SuccessCount
+
 		r := m.db.Save(&ei)
 		return r.Error
 	}
@@ -78,7 +77,6 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
 			x.Fetching = i.Fetching
 			x.HighestID = i.HighestID
 			x.Hostname = i.Hostname
-			x.LastError = i.LastError
 			x.Identified = i.Identified
 			x.Implementation = i.Implementation
 			x.NextFetch = i.NextFetch

database/manager.go

@@ -3,24 +3,21 @@ package database
 import (
 	"path/filepath"
 	"sync"
+
 	"github.com/jinzhu/gorm"
+
 	u "git.eeqj.de/sneak/goutil"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"github.com/golang/groupcache/lru"
 )
 
-const cacheEntries = 1000000
-
-// Manager coordinates database and cache reads and writes
 type Manager struct {
 	db                            *gorm.DB
 	cachelock                     sync.Mutex
-	recentlyInsertedTootHashCache *lru.Cache
+	recentlyInsertedTootHashCache *map[string]bool
 }
 
-// New creates new Manager
 func New() *Manager {
 	m := new(Manager)
 	m.init()
@@ -35,7 +32,9 @@ func (m *Manager) init() {
 	if viper.GetBool("Debug") {
 		m.db.LogMode(true)
 	}
-	m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
+
+	h := make(map[string]bool)
+	m.recentlyInsertedTootHashCache = &h
 }
 
 func (m *Manager) open() {

database/model.go

@@ -21,7 +21,7 @@ type StoredToot struct {
 	Content       []byte
 	TextContent   []byte
 	URL           string
-	Hostname      string `gorm:"index:hostnameindex"`
+	Hostname      string
 }
 
 type APInstance struct {
@@ -34,7 +34,6 @@ type APInstance struct {
 	Identified                 bool
 	Fetching                   bool
 	Disabled                   bool
-	LastError                  string
 	Implementation             string
 	NextFetch                  time.Time
 	NodeInfoURL                string

database/readshortcuts.go

@@ -1,24 +0,0 @@
-package database
-
-import (
-	"github.com/google/uuid"
-)
-
-func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
-	var c uint
-	e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
-	if e.Error != nil {
-		return 0, e.Error
-	} else {
-		return c, nil
-	}
-}
-
-func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
-	var i APInstance
-	e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
-	if e.Error != nil {
-		return nil, e.Error
-	}
-	return &i, nil
-}

database/storageconnector.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"html"
 	"strings"
+
 	"git.eeqj.de/sneak/feta/toot"
+
 	"github.com/google/uuid"
 	hstg "github.com/grokify/html-strip-tags-go"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
@@ -13,15 +15,24 @@ import (
 func (m *Manager) TootInsertHashCacheSize() uint {
 	m.cachelock.Lock()
 	defer m.cachelock.Unlock()
-	return uint(m.recentlyInsertedTootHashCache.Len())
+	return uint(len(*m.recentlyInsertedTootHashCache))
 }
 
 func (m *Manager) TootExists(t *toot.Toot) bool {
 	var try StoredToot
 
-	// check cache
+	// first, nuke cache if it gets too big to prevent
+	// unlimited memory usage
 	m.cachelock.Lock()
-	if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
+	if len(*m.recentlyInsertedTootHashCache) > 1000000 {
+		emptycache := make(map[string]bool)
+		m.recentlyInsertedTootHashCache = &emptycache
+	}
+	m.cachelock.Unlock()
+
+	m.cachelock.Lock()
+	// check map
+	if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
 		m.cachelock.Unlock()
 		return true
 	}
@@ -56,7 +67,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
 
 	// put it in the cache to avoid relying on db for dedupe:
 	m.cachelock.Lock()
-	m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
+	(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
 	m.cachelock.Unlock()
 
 	//panic(fmt.Sprintf("%+v", t))

go.mod

@@ -4,6 +4,8 @@ go 1.14
 
 require (
 	git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
+	github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
+	github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
 	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gin-gonic/gin v1.6.2
@@ -35,9 +37,10 @@ require (
 	github.com/valyala/fasttemplate v1.1.0 // indirect
 	github.com/ziflex/lecho v1.2.0
 	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
+	golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 // indirect
 	gopkg.in/ini.v1 v1.55.0 // indirect
-	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
 )

go.sum

@@ -56,7 +56,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

instance/instance.go

@@ -37,7 +37,6 @@ type Instance struct {
 	Implementation             string
 	InitialFSMState            string
 	NextFetch                  time.Time
-	LastError                  string
 	NodeInfoURL                string
 	ServerImplementationString string
 	ServerVersionString        string

process/handlers.go

@@ -20,19 +20,15 @@ func (a *Server) instances() []hash {
 	now := time.Now()
 	for _, v := range a.feta.manager.ListInstances() {
 		i := make(hash)
-		// TODO move this locking onto a method on Instance that just
-		// returns a new hash
 		// FIXME figure out why a very short lock here deadlocks
 		v.Lock()
 		i["hostname"] = v.Hostname
-		i["uuid"] = v.UUID.String()
 		i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
 		i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
 		i["successCount"] = v.SuccessCount
 		i["errorCount"] = v.ErrorCount
 		i["identified"] = v.Identified
-		//this only locks the FSM, not the whole instance struct
-		i["status"] = v.Status()
+		i["status"] = v.Status() //FIXME maybe this is why
 		i["software"] = "unknown"
 		i["version"] = "unknown"
 		if v.Identified {
@@ -42,14 +38,6 @@ func (a *Server) instances() []hash {
 		v.Unlock()
 		resp = append(resp, i)
 	}
-
-	for _, item := range resp {
-		count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
-		item["tootCount"] = 0
-		if err != nil {
-			item["tootCount"] = count
-		}
-	}
 	return resp
 }
 
@@ -87,27 +75,10 @@ func (a *Server) getInstanceListHandler() http.HandlerFunc {
 }
 */
 
-func (a *Server) instanceHandler(c echo.Context) error {
-	tu := c.Param("uuid")
-	u, err := uuid.Parse(tu)
-	if err != nil {
-		return c.String(http.StatusNotFound, "404 not found")
-	}
-
-
-	i := a.feta.manager.ListInstances() {
-	tc := pongo2.Context{
-		"instance": i,
-	}
-
-	return c.Render(http.StatusOK, "instance.html", tc)
-}
-
 func (a *Server) indexHandler(c echo.Context) error {
 	tc := pongo2.Context{
-		"time":      time.Now().UTC().Format(time.RFC3339Nano),
-		"gitrev":    a.feta.version,
-		"instances": a.instances(),
+		"time":   time.Now().UTC().Format(time.RFC3339Nano),
+		"gitrev": a.feta.version,
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }

process/server.go

@@ -95,7 +95,6 @@ func (s *Server) initRouter() {
 
 	// Routes
 	s.e.GET("/", s.indexHandler)
-	s.e.GET("/instance/:uuid", s.instanceHandler)
 	s.e.GET("/stats.json", s.statsHandler)
 	s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
 	//a.e.GET("/about", s.aboutHandler)

seeds/seeds.go

@@ -315,7 +315,6 @@ var SeedInstances = [...]string{
 	"snuskete.net",
 	"social.1in9.net",
 	"social.adlerweb.info",
-	"social.art-software.fr",
 	"social.au2pb.net",
 	"social.avareborn.de",
 	"social.azkware.net",

view/base.html

@@ -10,7 +10,6 @@
       <meta name="author" content="">
     -->
     <title>{{ htmltitle }}</title>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
   <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
 
 <style>

view/index.html

@@ -9,25 +9,17 @@
 <table class="table table-striped table-hover">
   <thead class="thead-dark">
 <tr>
-  <th scope="col">instance id</th>
-  <th scope="col">hostname</th>
-  <th scope="col">status</th>
-  <th scope="col">tootCount</th>
-  <th scope="col">Detail</th>
+  <th scope="col">Key</th>
+  <th scope="col">Value</th>
 </tr>
 </thead>
 <tbody>
-{% for instance in instances %}
+{% for stat in stats %}
     <tr>
-        <td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
-        <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
-        <td>{{instance.status}}</td>
-        <td>{{instance.tootCount}}</td>
-        <td><a
-                href="/instance/{{instance.uuid}}"
-                class="btn btn-info">
-                <i class="fab fa-mastodon"></i>
-            </button></td>
+        {{stat.key}}
+    </tr>
+    <tr>
+        {{stat.value}}
     </tr>
 {% endfor %}
  </tbody>