Compare commits
4 Commits
bf404db2af
...
783086bb0d
Author | SHA1 | Date |
---|---|---|
Jeffrey Paul | 783086bb0d | |
Jeffrey Paul | 45848dbbdf | |
Jeffrey Paul | 70faf517f3 | |
Jeffrey Paul | 4c33b5dd0e |
|
@ -27,6 +27,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
|
|||
Identified: i.Identified,
|
||||
Implementation: i.Implementation,
|
||||
NextFetch: i.NextFetch,
|
||||
LastError: i.LastError,
|
||||
NodeInfoURL: i.NodeInfoURL,
|
||||
ServerImplementationString: i.ServerImplementationString,
|
||||
ServerVersionString: i.ServerVersionString,
|
||||
|
@ -49,6 +50,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
|
|||
ei.Fetching = i.Fetching
|
||||
ei.HighestID = i.HighestID
|
||||
ei.Hostname = i.Hostname
|
||||
ei.LastError = i.LastError
|
||||
ei.Identified = i.Identified
|
||||
ei.Implementation = string(i.Implementation)
|
||||
ei.NextFetch = i.NextFetch
|
||||
|
@ -56,7 +58,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
|
|||
ei.ServerImplementationString = i.ServerImplementationString
|
||||
ei.ServerVersionString = i.ServerVersionString
|
||||
ei.SuccessCount = i.SuccessCount
|
||||
|
||||
r := m.db.Save(&ei)
|
||||
return r.Error
|
||||
}
|
||||
|
@ -77,6 +78,7 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
|
|||
x.Fetching = i.Fetching
|
||||
x.HighestID = i.HighestID
|
||||
x.Hostname = i.Hostname
|
||||
x.LastError = i.LastError
|
||||
x.Identified = i.Identified
|
||||
x.Implementation = i.Implementation
|
||||
x.NextFetch = i.NextFetch
|
||||
|
|
|
@ -3,21 +3,24 @@ package database
|
|||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
|
||||
u "git.eeqj.de/sneak/goutil"
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/golang/groupcache/lru"
|
||||
)
|
||||
|
||||
const cacheEntries = 1000000
|
||||
|
||||
// Manager coordinates database and cache reads and writes
|
||||
type Manager struct {
|
||||
db *gorm.DB
|
||||
cachelock sync.Mutex
|
||||
recentlyInsertedTootHashCache *map[string]bool
|
||||
recentlyInsertedTootHashCache *lru.Cache
|
||||
}
|
||||
|
||||
// New creates new Manager
|
||||
func New() *Manager {
|
||||
m := new(Manager)
|
||||
m.init()
|
||||
|
@ -32,9 +35,7 @@ func (m *Manager) init() {
|
|||
if viper.GetBool("Debug") {
|
||||
m.db.LogMode(true)
|
||||
}
|
||||
|
||||
h := make(map[string]bool)
|
||||
m.recentlyInsertedTootHashCache = &h
|
||||
m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
|
||||
}
|
||||
|
||||
func (m *Manager) open() {
|
||||
|
|
|
@ -21,7 +21,7 @@ type StoredToot struct {
|
|||
Content []byte
|
||||
TextContent []byte
|
||||
URL string
|
||||
Hostname string
|
||||
Hostname string `gorm:"index:hostnameindex"`
|
||||
}
|
||||
|
||||
type APInstance struct {
|
||||
|
@ -34,6 +34,7 @@ type APInstance struct {
|
|||
Identified bool
|
||||
Fetching bool
|
||||
Disabled bool
|
||||
LastError string
|
||||
Implementation string
|
||||
NextFetch time.Time
|
||||
NodeInfoURL string
|
||||
|
|
|
@ -0,0 +1,24 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
|
||||
var c uint
|
||||
e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
|
||||
if e.Error != nil {
|
||||
return 0, e.Error
|
||||
} else {
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
|
||||
var i APInstance
|
||||
e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
|
||||
if e.Error != nil {
|
||||
return nil, e.Error
|
||||
}
|
||||
return &i, nil
|
||||
}
|
|
@ -4,9 +4,7 @@ import (
|
|||
"fmt"
|
||||
"html"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
|
||||
"github.com/google/uuid"
|
||||
hstg "github.com/grokify/html-strip-tags-go"
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
|
@ -15,24 +13,15 @@ import (
|
|||
func (m *Manager) TootInsertHashCacheSize() uint {
|
||||
m.cachelock.Lock()
|
||||
defer m.cachelock.Unlock()
|
||||
return uint(len(*m.recentlyInsertedTootHashCache))
|
||||
return uint(m.recentlyInsertedTootHashCache.Len())
|
||||
}
|
||||
|
||||
func (m *Manager) TootExists(t *toot.Toot) bool {
|
||||
var try StoredToot
|
||||
|
||||
// first, nuke cache if it gets too big to prevent
|
||||
// unlimited memory usage
|
||||
// check cache
|
||||
m.cachelock.Lock()
|
||||
if len(*m.recentlyInsertedTootHashCache) > 1000000 {
|
||||
emptycache := make(map[string]bool)
|
||||
m.recentlyInsertedTootHashCache = &emptycache
|
||||
}
|
||||
m.cachelock.Unlock()
|
||||
|
||||
m.cachelock.Lock()
|
||||
// check map
|
||||
if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
|
||||
if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
|
||||
m.cachelock.Unlock()
|
||||
return true
|
||||
}
|
||||
|
@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
|
|||
|
||||
// put it in the cache to avoid relying on db for dedupe:
|
||||
m.cachelock.Lock()
|
||||
(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
|
||||
m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
|
||||
m.cachelock.Unlock()
|
||||
|
||||
//panic(fmt.Sprintf("%+v", t))
|
||||
|
|
5
go.mod
5
go.mod
|
@ -4,8 +4,6 @@ go 1.14
|
|||
|
||||
require (
|
||||
git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
|
||||
github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
|
||||
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
|
||||
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/gin-gonic/gin v1.6.2
|
||||
|
@ -37,10 +35,9 @@ require (
|
|||
github.com/valyala/fasttemplate v1.1.0 // indirect
|
||||
github.com/ziflex/lecho v1.2.0
|
||||
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
|
||||
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
|
||||
golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 // indirect
|
||||
gopkg.in/ini.v1 v1.55.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
|
||||
)
|
||||
|
|
1
go.sum
1
go.sum
|
@ -56,6 +56,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
|
|||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
|
|
|
@ -37,6 +37,7 @@ type Instance struct {
|
|||
Implementation string
|
||||
InitialFSMState string
|
||||
NextFetch time.Time
|
||||
LastError string
|
||||
NodeInfoURL string
|
||||
ServerImplementationString string
|
||||
ServerVersionString string
|
||||
|
|
|
@ -20,15 +20,19 @@ func (a *Server) instances() []hash {
|
|||
now := time.Now()
|
||||
for _, v := range a.feta.manager.ListInstances() {
|
||||
i := make(hash)
|
||||
// TODO move this locking onto a method on Instance that just
|
||||
// returns a new hash
|
||||
// FIXME figure out why a very short lock here deadlocks
|
||||
v.Lock()
|
||||
i["hostname"] = v.Hostname
|
||||
i["uuid"] = v.UUID.String()
|
||||
i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
|
||||
i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
|
||||
i["successCount"] = v.SuccessCount
|
||||
i["errorCount"] = v.ErrorCount
|
||||
i["identified"] = v.Identified
|
||||
i["status"] = v.Status() //FIXME maybe this is why
|
||||
//this only locks the FSM, not the whole instance struct
|
||||
i["status"] = v.Status()
|
||||
i["software"] = "unknown"
|
||||
i["version"] = "unknown"
|
||||
if v.Identified {
|
||||
|
@ -38,6 +42,14 @@ func (a *Server) instances() []hash {
|
|||
v.Unlock()
|
||||
resp = append(resp, i)
|
||||
}
|
||||
|
||||
for _, item := range resp {
|
||||
count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
|
||||
item["tootCount"] = 0
|
||||
if err != nil {
|
||||
item["tootCount"] = count
|
||||
}
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
|
@ -75,10 +87,27 @@ func (a *Server) getInstanceListHandler() http.HandlerFunc {
|
|||
}
|
||||
*/
|
||||
|
||||
func (a *Server) instanceHandler(c echo.Context) error {
|
||||
tu := c.Param("uuid")
|
||||
u, err := uuid.Parse(tu)
|
||||
if err != nil {
|
||||
return c.String(http.StatusNotFound, "404 not found")
|
||||
}
|
||||
|
||||
|
||||
i := a.feta.manager.ListInstances() {
|
||||
tc := pongo2.Context{
|
||||
"instance": i,
|
||||
}
|
||||
|
||||
return c.Render(http.StatusOK, "instance.html", tc)
|
||||
}
|
||||
|
||||
func (a *Server) indexHandler(c echo.Context) error {
|
||||
tc := pongo2.Context{
|
||||
"time": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"gitrev": a.feta.version,
|
||||
"time": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"gitrev": a.feta.version,
|
||||
"instances": a.instances(),
|
||||
}
|
||||
return c.Render(http.StatusOK, "index.html", tc)
|
||||
}
|
||||
|
|
|
@ -95,6 +95,7 @@ func (s *Server) initRouter() {
|
|||
|
||||
// Routes
|
||||
s.e.GET("/", s.indexHandler)
|
||||
s.e.GET("/instance/:uuid", s.instanceHandler)
|
||||
s.e.GET("/stats.json", s.statsHandler)
|
||||
s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
|
||||
//a.e.GET("/about", s.aboutHandler)
|
||||
|
|
|
@ -315,6 +315,7 @@ var SeedInstances = [...]string{
|
|||
"snuskete.net",
|
||||
"social.1in9.net",
|
||||
"social.adlerweb.info",
|
||||
"social.art-software.fr",
|
||||
"social.au2pb.net",
|
||||
"social.avareborn.de",
|
||||
"social.azkware.net",
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
<meta name="author" content="">
|
||||
-->
|
||||
<title>{{ htmltitle }}</title>
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
|
||||
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
|
||||
|
||||
<style>
|
||||
|
|
|
@ -9,17 +9,25 @@
|
|||
<table class="table table-striped table-hover">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th scope="col">Key</th>
|
||||
<th scope="col">Value</th>
|
||||
<th scope="col">instance id</th>
|
||||
<th scope="col">hostname</th>
|
||||
<th scope="col">status</th>
|
||||
<th scope="col">tootCount</th>
|
||||
<th scope="col">Detail</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for stat in stats %}
|
||||
{% for instance in instances %}
|
||||
<tr>
|
||||
{{stat.key}}
|
||||
</tr>
|
||||
<tr>
|
||||
{{stat.value}}
|
||||
<td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
|
||||
<td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
|
||||
<td>{{instance.status}}</td>
|
||||
<td>{{instance.tootCount}}</td>
|
||||
<td><a
|
||||
href="/instance/{{instance.uuid}}"
|
||||
class="btn btn-info">
|
||||
<i class="fab fa-mastodon"></i>
|
||||
</button></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
|
|
Loading…
Reference in New Issue