Compare commits

...

4 Commits

Author SHA1 Message Date
Jeffrey Paul 783086bb0d update sums
continuous-integration/drone/push Build is failing Details
2020-04-04 18:26:55 -07:00
Jeffrey Paul 45848dbbdf add LRU caching 2020-04-04 18:17:35 -07:00
Jeffrey Paul 70faf517f3 commit before applying 2p patch 2020-04-04 18:16:37 -07:00
Jeffrey Paul 4c33b5dd0e make sure we spider the instance that sent me death threats 2020-03-30 21:57:26 -07:00
13 changed files with 93 additions and 37 deletions

View File

@ -27,6 +27,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
Identified: i.Identified, Identified: i.Identified,
Implementation: i.Implementation, Implementation: i.Implementation,
NextFetch: i.NextFetch, NextFetch: i.NextFetch,
LastError: i.LastError,
NodeInfoURL: i.NodeInfoURL, NodeInfoURL: i.NodeInfoURL,
ServerImplementationString: i.ServerImplementationString, ServerImplementationString: i.ServerImplementationString,
ServerVersionString: i.ServerVersionString, ServerVersionString: i.ServerVersionString,
@ -49,6 +50,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
ei.Fetching = i.Fetching ei.Fetching = i.Fetching
ei.HighestID = i.HighestID ei.HighestID = i.HighestID
ei.Hostname = i.Hostname ei.Hostname = i.Hostname
ei.LastError = i.LastError
ei.Identified = i.Identified ei.Identified = i.Identified
ei.Implementation = string(i.Implementation) ei.Implementation = string(i.Implementation)
ei.NextFetch = i.NextFetch ei.NextFetch = i.NextFetch
@ -56,7 +58,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
ei.ServerImplementationString = i.ServerImplementationString ei.ServerImplementationString = i.ServerImplementationString
ei.ServerVersionString = i.ServerVersionString ei.ServerVersionString = i.ServerVersionString
ei.SuccessCount = i.SuccessCount ei.SuccessCount = i.SuccessCount
r := m.db.Save(&ei) r := m.db.Save(&ei)
return r.Error return r.Error
} }
@ -77,6 +78,7 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
x.Fetching = i.Fetching x.Fetching = i.Fetching
x.HighestID = i.HighestID x.HighestID = i.HighestID
x.Hostname = i.Hostname x.Hostname = i.Hostname
x.LastError = i.LastError
x.Identified = i.Identified x.Identified = i.Identified
x.Implementation = i.Implementation x.Implementation = i.Implementation
x.NextFetch = i.NextFetch x.NextFetch = i.NextFetch

View File

@ -3,21 +3,24 @@ package database
import ( import (
"path/filepath" "path/filepath"
"sync" "sync"
"github.com/jinzhu/gorm" "github.com/jinzhu/gorm"
u "git.eeqj.de/sneak/goutil" u "git.eeqj.de/sneak/goutil"
_ "github.com/jinzhu/gorm/dialects/sqlite" _ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/golang/groupcache/lru"
) )
const cacheEntries = 1000000
// Manager coordinates database and cache reads and writes
type Manager struct { type Manager struct {
db *gorm.DB db *gorm.DB
cachelock sync.Mutex cachelock sync.Mutex
recentlyInsertedTootHashCache *map[string]bool recentlyInsertedTootHashCache *lru.Cache
} }
// New creates new Manager
func New() *Manager { func New() *Manager {
m := new(Manager) m := new(Manager)
m.init() m.init()
@ -32,9 +35,7 @@ func (m *Manager) init() {
if viper.GetBool("Debug") { if viper.GetBool("Debug") {
m.db.LogMode(true) m.db.LogMode(true)
} }
m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
h := make(map[string]bool)
m.recentlyInsertedTootHashCache = &h
} }
func (m *Manager) open() { func (m *Manager) open() {

View File

@ -21,7 +21,7 @@ type StoredToot struct {
Content []byte Content []byte
TextContent []byte TextContent []byte
URL string URL string
Hostname string Hostname string `gorm:"index:hostnameindex"`
} }
type APInstance struct { type APInstance struct {
@ -34,6 +34,7 @@ type APInstance struct {
Identified bool Identified bool
Fetching bool Fetching bool
Disabled bool Disabled bool
LastError string
Implementation string Implementation string
NextFetch time.Time NextFetch time.Time
NodeInfoURL string NodeInfoURL string

24
database/readshortcuts.go Normal file
View File

@ -0,0 +1,24 @@
package database
import (
"github.com/google/uuid"
)
func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
var c uint
e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
if e.Error != nil {
return 0, e.Error
} else {
return c, nil
}
}
func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
var i APInstance
e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
if e.Error != nil {
return nil, e.Error
}
return &i, nil
}

View File

@ -4,9 +4,7 @@ import (
"fmt" "fmt"
"html" "html"
"strings" "strings"
"git.eeqj.de/sneak/feta/toot" "git.eeqj.de/sneak/feta/toot"
"github.com/google/uuid" "github.com/google/uuid"
hstg "github.com/grokify/html-strip-tags-go" hstg "github.com/grokify/html-strip-tags-go"
_ "github.com/jinzhu/gorm/dialects/sqlite" _ "github.com/jinzhu/gorm/dialects/sqlite"
@ -15,24 +13,15 @@ import (
func (m *Manager) TootInsertHashCacheSize() uint { func (m *Manager) TootInsertHashCacheSize() uint {
m.cachelock.Lock() m.cachelock.Lock()
defer m.cachelock.Unlock() defer m.cachelock.Unlock()
return uint(len(*m.recentlyInsertedTootHashCache)) return uint(m.recentlyInsertedTootHashCache.Len())
} }
func (m *Manager) TootExists(t *toot.Toot) bool { func (m *Manager) TootExists(t *toot.Toot) bool {
var try StoredToot var try StoredToot
// first, nuke cache if it gets too big to prevent // check cache
// unlimited memory usage
m.cachelock.Lock() m.cachelock.Lock()
if len(*m.recentlyInsertedTootHashCache) > 1000000 { if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
emptycache := make(map[string]bool)
m.recentlyInsertedTootHashCache = &emptycache
}
m.cachelock.Unlock()
m.cachelock.Lock()
// check map
if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
m.cachelock.Unlock() m.cachelock.Unlock()
return true return true
} }
@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
// put it in the cache to avoid relying on db for dedupe: // put it in the cache to avoid relying on db for dedupe:
m.cachelock.Lock() m.cachelock.Lock()
(*m.recentlyInsertedTootHashCache)[nt.Hash] = true m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
m.cachelock.Unlock() m.cachelock.Unlock()
//panic(fmt.Sprintf("%+v", t)) //panic(fmt.Sprintf("%+v", t))

5
go.mod
View File

@ -4,8 +4,6 @@ go 1.14
require ( require (
git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5 git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/gin-gonic/gin v1.6.2 github.com/gin-gonic/gin v1.6.2
@ -37,10 +35,9 @@ require (
github.com/valyala/fasttemplate v1.1.0 // indirect github.com/valyala/fasttemplate v1.1.0 // indirect
github.com/ziflex/lecho v1.2.0 github.com/ziflex/lecho v1.2.0
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 // indirect
gopkg.in/ini.v1 v1.55.0 // indirect gopkg.in/ini.v1 v1.55.0 // indirect
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
) )

1
go.sum
View File

@ -56,6 +56,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

View File

@ -37,6 +37,7 @@ type Instance struct {
Implementation string Implementation string
InitialFSMState string InitialFSMState string
NextFetch time.Time NextFetch time.Time
LastError string
NodeInfoURL string NodeInfoURL string
ServerImplementationString string ServerImplementationString string
ServerVersionString string ServerVersionString string

View File

@ -20,15 +20,19 @@ func (a *Server) instances() []hash {
now := time.Now() now := time.Now()
for _, v := range a.feta.manager.ListInstances() { for _, v := range a.feta.manager.ListInstances() {
i := make(hash) i := make(hash)
// TODO move this locking onto a method on Instance that just
// returns a new hash
// FIXME figure out why a very short lock here deadlocks // FIXME figure out why a very short lock here deadlocks
v.Lock() v.Lock()
i["hostname"] = v.Hostname i["hostname"] = v.Hostname
i["uuid"] = v.UUID.String()
i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339) i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String() i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
i["successCount"] = v.SuccessCount i["successCount"] = v.SuccessCount
i["errorCount"] = v.ErrorCount i["errorCount"] = v.ErrorCount
i["identified"] = v.Identified i["identified"] = v.Identified
i["status"] = v.Status() //FIXME maybe this is why //this only locks the FSM, not the whole instance struct
i["status"] = v.Status()
i["software"] = "unknown" i["software"] = "unknown"
i["version"] = "unknown" i["version"] = "unknown"
if v.Identified { if v.Identified {
@ -38,6 +42,14 @@ func (a *Server) instances() []hash {
v.Unlock() v.Unlock()
resp = append(resp, i) resp = append(resp, i)
} }
for _, item := range resp {
count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
item["tootCount"] = 0
if err != nil {
item["tootCount"] = count
}
}
return resp return resp
} }
@ -75,10 +87,27 @@ func (a *Server) getInstanceListHandler() http.HandlerFunc {
} }
*/ */
func (a *Server) instanceHandler(c echo.Context) error {
tu := c.Param("uuid")
u, err := uuid.Parse(tu)
if err != nil {
return c.String(http.StatusNotFound, "404 not found")
}
i := a.feta.manager.ListInstances() {
tc := pongo2.Context{
"instance": i,
}
return c.Render(http.StatusOK, "instance.html", tc)
}
func (a *Server) indexHandler(c echo.Context) error { func (a *Server) indexHandler(c echo.Context) error {
tc := pongo2.Context{ tc := pongo2.Context{
"time": time.Now().UTC().Format(time.RFC3339Nano), "time": time.Now().UTC().Format(time.RFC3339Nano),
"gitrev": a.feta.version, "gitrev": a.feta.version,
"instances": a.instances(),
} }
return c.Render(http.StatusOK, "index.html", tc) return c.Render(http.StatusOK, "index.html", tc)
} }

View File

@ -95,6 +95,7 @@ func (s *Server) initRouter() {
// Routes // Routes
s.e.GET("/", s.indexHandler) s.e.GET("/", s.indexHandler)
s.e.GET("/instance/:uuid", s.instanceHandler)
s.e.GET("/stats.json", s.statsHandler) s.e.GET("/stats.json", s.statsHandler)
s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler) s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
//a.e.GET("/about", s.aboutHandler) //a.e.GET("/about", s.aboutHandler)

View File

@ -315,6 +315,7 @@ var SeedInstances = [...]string{
"snuskete.net", "snuskete.net",
"social.1in9.net", "social.1in9.net",
"social.adlerweb.info", "social.adlerweb.info",
"social.art-software.fr",
"social.au2pb.net", "social.au2pb.net",
"social.avareborn.de", "social.avareborn.de",
"social.azkware.net", "social.azkware.net",

View File

@ -10,6 +10,7 @@
<meta name="author" content=""> <meta name="author" content="">
--> -->
<title>{{ htmltitle }}</title> <title>{{ htmltitle }}</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous"> <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<style> <style>

View File

@ -9,17 +9,25 @@
<table class="table table-striped table-hover"> <table class="table table-striped table-hover">
<thead class="thead-dark"> <thead class="thead-dark">
<tr> <tr>
<th scope="col">Key</th> <th scope="col">instance id</th>
<th scope="col">Value</th> <th scope="col">hostname</th>
<th scope="col">status</th>
<th scope="col">tootCount</th>
<th scope="col">Detail</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{% for stat in stats %} {% for instance in instances %}
<tr> <tr>
{{stat.key}} <td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
</tr> <td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
<tr> <td>{{instance.status}}</td>
{{stat.value}} <td>{{instance.tootCount}}</td>
<td><a
href="/instance/{{instance.uuid}}"
class="btn btn-info">
<i class="fab fa-mastodon"></i>
</button></td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>