sneak
/
feta
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Compare commits

base: sneak:783086bb0dc717e06f70d536809532f0779a6431
Branches Tags
sneak:master
sneak:next
..
compare: sneak:bf404db2af7ae71c612a330066039c7ac95cc1c4
Branches Tags
sneak:next
sneak:master

No commits in common. "783086bb0dc717e06f70d536809532f0779a6431" and "bf404db2af7ae71c612a330066039c7ac95cc1c4" have entirely different histories.
783086bb0d ... bf404db2af

13 changed files with 37 additions and 93 deletions
Show Stats Expand all files Collapse all files

4
database/imconnector.go
View File

@ -27,7 +27,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
Identified: i.Identified,
Implementation: i.Implementation,
NextFetch: i.NextFetch,
LastError: i.LastError,
NodeInfoURL: i.NodeInfoURL,
ServerImplementationString: i.ServerImplementationString,
ServerVersionString: i.ServerVersionString,
@ -50,7 +49,6 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
ei.Fetching = i.Fetching
ei.HighestID = i.HighestID
ei.Hostname = i.Hostname
ei.LastError = i.LastError
ei.Identified = i.Identified
ei.Implementation = string(i.Implementation)
ei.NextFetch = i.NextFetch
@ -58,6 +56,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error {
ei.ServerImplementationString = i.ServerImplementationString
ei.ServerVersionString = i.ServerVersionString
ei.SuccessCount = i.SuccessCount
r := m.db.Save(&ei)
return r.Error
}
@ -78,7 +77,6 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) {
x.Fetching = i.Fetching
x.HighestID = i.HighestID
x.Hostname = i.Hostname
x.LastError = i.LastError
x.Identified = i.Identified
x.Implementation = i.Implementation
x.NextFetch = i.NextFetch

13
database/manager.go
View File

@ -3,24 +3,21 @@ package database
import (
"path/filepath"
"sync"
"github.com/jinzhu/gorm"
u "git.eeqj.de/sneak/goutil"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
"github.com/golang/groupcache/lru"
)
const cacheEntries = 1000000
// Manager coordinates database and cache reads and writes
type Manager struct {
db *gorm.DB
cachelock sync.Mutex
recentlyInsertedTootHashCache *lru.Cache
recentlyInsertedTootHashCache *map[string]bool
}
// New creates new Manager
func New() *Manager {
m := new(Manager)
m.init()
@ -35,7 +32,9 @@ func (m *Manager) init() {
if viper.GetBool("Debug") {
m.db.LogMode(true)
}
m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
h := make(map[string]bool)
m.recentlyInsertedTootHashCache = &h
}
func (m *Manager) open() {

3
database/model.go
View File

@ -21,7 +21,7 @@ type StoredToot struct {
Content []byte
TextContent []byte
URL string
Hostname string `gorm:"index:hostnameindex"`
Hostname string
}
type APInstance struct {
@ -34,7 +34,6 @@ type APInstance struct {
Identified bool
Fetching bool
Disabled bool
LastError string
Implementation string
NextFetch time.Time
NodeInfoURL string

24
database/readshortcuts.go
View File

@ -1,24 +0,0 @@
package database
import (
"github.com/google/uuid"
)
func (m *Manager) TootCountForHostname(hostname string) (uint, error) {
var c uint
e := m.db.Model(&StoredToot{}).Where("hostname = ?", hostname).Count(&c)
if e.Error != nil {
return 0, e.Error
} else {
return c, nil
}
}
func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) {
var i APInstance
e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i)
if e.Error != nil {
return nil, e.Error
}
return &i, nil
}

19
database/storageconnector.go
View File

@ -4,7 +4,9 @@ import (
"fmt"
"html"
"strings"
"git.eeqj.de/sneak/feta/toot"
"github.com/google/uuid"
hstg "github.com/grokify/html-strip-tags-go"
_ "github.com/jinzhu/gorm/dialects/sqlite"
@ -13,15 +15,24 @@ import (
func (m *Manager) TootInsertHashCacheSize() uint {
m.cachelock.Lock()
defer m.cachelock.Unlock()
return uint(m.recentlyInsertedTootHashCache.Len())
return uint(len(*m.recentlyInsertedTootHashCache))
}
func (m *Manager) TootExists(t *toot.Toot) bool {
var try StoredToot
// check cache
// first, nuke cache if it gets too big to prevent
// unlimited memory usage
m.cachelock.Lock()
if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
if len(*m.recentlyInsertedTootHashCache) > 1000000 {
emptycache := make(map[string]bool)
m.recentlyInsertedTootHashCache = &emptycache
}
m.cachelock.Unlock()
m.cachelock.Lock()
// check map
if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
m.cachelock.Unlock()
return true
}
@ -56,7 +67,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
// put it in the cache to avoid relying on db for dedupe:
m.cachelock.Lock()
m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
m.cachelock.Unlock()
//panic(fmt.Sprintf("%+v", t))

5
go.mod
View File

@ -4,6 +4,8 @@ go 1.14
require (
git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/gin-gonic/gin v1.6.2
@ -35,9 +37,10 @@ require (
github.com/valyala/fasttemplate v1.1.0 // indirect
github.com/ziflex/lecho v1.2.0
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect
golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03 // indirect
gopkg.in/ini.v1 v1.55.0 // indirect
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
)

1
go.sum
View File

@ -56,7 +56,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

1
instance/instance.go
View File

@ -37,7 +37,6 @@ type Instance struct {
Implementation string
InitialFSMState string
NextFetch time.Time
LastError string
NodeInfoURL string
ServerImplementationString string
ServerVersionString string

35
process/handlers.go
View File

@ -20,19 +20,15 @@ func (a *Server) instances() []hash {
now := time.Now()
for _, v := range a.feta.manager.ListInstances() {
i := make(hash)
// TODO move this locking onto a method on Instance that just
// returns a new hash
// FIXME figure out why a very short lock here deadlocks
v.Lock()
i["hostname"] = v.Hostname
i["uuid"] = v.UUID.String()
i["nextCheck"] = v.NextFetch.UTC().Format(time.RFC3339)
i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String()
i["successCount"] = v.SuccessCount
i["errorCount"] = v.ErrorCount
i["identified"] = v.Identified
//this only locks the FSM, not the whole instance struct
i["status"] = v.Status()
i["status"] = v.Status() //FIXME maybe this is why
i["software"] = "unknown"
i["version"] = "unknown"
if v.Identified {
@ -42,14 +38,6 @@ func (a *Server) instances() []hash {
v.Unlock()
resp = append(resp, i)
}
for _, item := range resp {
count, err := a.feta.dbm.TootCountForHostname(item["hostname"].(string))
item["tootCount"] = 0
if err != nil {
item["tootCount"] = count
}
}
return resp
}
@ -87,27 +75,10 @@ func (a *Server) getInstanceListHandler() http.HandlerFunc {
}
*/
func (a *Server) instanceHandler(c echo.Context) error {
tu := c.Param("uuid")
u, err := uuid.Parse(tu)
if err != nil {
return c.String(http.StatusNotFound, "404 not found")
}
i := a.feta.manager.ListInstances() {
tc := pongo2.Context{
"instance": i,
}
return c.Render(http.StatusOK, "instance.html", tc)
}
func (a *Server) indexHandler(c echo.Context) error {
tc := pongo2.Context{
"time": time.Now().UTC().Format(time.RFC3339Nano),
"gitrev": a.feta.version,
"instances": a.instances(),
"time": time.Now().UTC().Format(time.RFC3339Nano),
"gitrev": a.feta.version,
}
return c.Render(http.StatusOK, "index.html", tc)
}

1
process/server.go
View File

@ -95,7 +95,6 @@ func (s *Server) initRouter() {
// Routes
s.e.GET("/", s.indexHandler)
s.e.GET("/instance/:uuid", s.instanceHandler)
s.e.GET("/stats.json", s.statsHandler)
s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler)
//a.e.GET("/about", s.aboutHandler)

1
seeds/seeds.go
View File

@ -315,7 +315,6 @@ var SeedInstances = [...]string{
"snuskete.net",
"social.1in9.net",
"social.adlerweb.info",
"social.art-software.fr",
"social.au2pb.net",
"social.avareborn.de",
"social.azkware.net",

1
view/base.html
View File

@ -10,7 +10,6 @@
<meta name="author" content="">
-->
<title>{{ htmltitle }}</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha384-Bfad6CLCknfcloXFOyFnlgtENryhrpZCe29RTifKEixXQZ38WheV+i/6YWSzkz3V" crossorigin="anonymous">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<style>

22
view/index.html
View File

@ -9,25 +9,17 @@
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">instance id</th>
<th scope="col">hostname</th>
<th scope="col">status</th>
<th scope="col">tootCount</th>
<th scope="col">Detail</th>
<th scope="col">Key</th>
<th scope="col">Value</th>
</tr>
</thead>
<tbody>
{% for instance in instances %}
{% for stat in stats %}
<tr>
<td><a href="/instance/{{instance.uuid}}">{{instance.uuid}}</a></td>
<td><a href="https://{{instance.hostname}}">{{instance.hostname}}</a></td>
<td>{{instance.status}}</td>
<td>{{instance.tootCount}}</td>
<td><a
href="/instance/{{instance.uuid}}"
class="btn btn-info">
<i class="fab fa-mastodon"></i>
</button></td>
{{stat.key}}
</tr>
<tr>
{{stat.value}}
</tr>
{% endfor %}
</tbody>