Compare commits
6 Commits
2476e2484e
...
2ecd833726
| Author | SHA1 | Date | |
|---|---|---|---|
| 2ecd833726 | |||
| b3f672b84a | |||
| 84b19fb14e | |||
| 0683cd7b32 | |||
| 1c2fdf6349 | |||
| 23c1b08798 |
@ -1,10 +0,0 @@
|
||||
version: 2
|
||||
jobs:
|
||||
build:
|
||||
machine: true
|
||||
steps:
|
||||
- checkout
|
||||
- run: |
|
||||
make dist
|
||||
- store_artifacts:
|
||||
path: output
|
||||
@ -6,11 +6,8 @@ steps:
|
||||
image: plugins/docker
|
||||
settings:
|
||||
repo: sneak/feta
|
||||
username:
|
||||
from_secret: docker_username
|
||||
password:
|
||||
from_secret: docker_password
|
||||
auto_tag: true
|
||||
dry_run: true
|
||||
#auto_tag: true
|
||||
tags:
|
||||
- ${DRONE_COMMIT_SHA}
|
||||
- ${DRONE_BRANCH}
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -2,3 +2,4 @@ feta
|
||||
output/
|
||||
feta.sqlite
|
||||
.lintsetup
|
||||
out
|
||||
|
||||
6
Makefile
6
Makefile
@ -23,10 +23,10 @@ ifneq ($(UNAME_S),Darwin)
|
||||
GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)"
|
||||
endif
|
||||
|
||||
default: run
|
||||
default: build
|
||||
|
||||
rundebug: build
|
||||
GOTRACEBACK=all DEBUG=1 ./$(FN)
|
||||
debug: build
|
||||
GOTRACEBACK=all FETA_DEBUG=1 ./$(FN)
|
||||
|
||||
run: build
|
||||
./$(FN)
|
||||
|
||||
13
README.md
13
README.md
@ -2,9 +2,20 @@
|
||||
|
||||
archives the fediverse
|
||||
|
||||
# todo
|
||||
|
||||
* scan toots for mentions and feed to locator
|
||||
* put toots in a separate db file
|
||||
* test with a real database
|
||||
* save instances to store more often
|
||||
* verify instances load properly on startup
|
||||
* do some simple in-memory dedupe for toot storage
|
||||
* make some templates using pongo2 and a simple website
|
||||
* update APIs
|
||||
|
||||
# status
|
||||
|
||||
[](https://circleci.com/gh/sneak/feta)
|
||||
[](https://drone.datavi.be/sneak/feta)
|
||||
|
||||
# ethics statement
|
||||
|
||||
|
||||
19
config.go
19
config.go
@ -1,19 +0,0 @@
|
||||
package feta
|
||||
|
||||
import "time"
|
||||
|
||||
// FIXME this should use viper or something
|
||||
|
||||
// Config stores the configuration for the feta process
|
||||
type Config struct {
|
||||
LogReportInterval time.Duration
|
||||
FSStorageLocation string
|
||||
}
|
||||
|
||||
// GetConfig returns the config
|
||||
func GetConfig() *Config {
|
||||
c := new(Config)
|
||||
c.LogReportInterval = time.Second * 10
|
||||
c.FSStorageLocation = "/home/sneak/Library/ApplicationSupport/feta/tootarchive"
|
||||
return c
|
||||
}
|
||||
92
database/imconnector.go
Normal file
92
database/imconnector.go
Normal file
@ -0,0 +1,92 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"git.eeqj.de/sneak/feta/instance"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
)
|
||||
|
||||
func (m *Manager) SaveInstance(i *instance.Instance) error {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
var x APInstance
|
||||
if m.db.Where("UUID = ?", i.UUID).First(&x).RecordNotFound() {
|
||||
log.Info().
|
||||
Str("hostname", i.Hostname).
|
||||
Msg("instance not in db, inserting")
|
||||
// item does not exist in db yet, must insert
|
||||
ni := APInstance{
|
||||
UUID: i.UUID,
|
||||
Disabled: i.Disabled,
|
||||
ErrorCount: i.ErrorCount,
|
||||
FSMState: i.Status(),
|
||||
Fetching: i.Fetching,
|
||||
HighestID: i.HighestID,
|
||||
Hostname: i.Hostname,
|
||||
Identified: i.Identified,
|
||||
Implementation: i.Implementation,
|
||||
NextFetch: i.NextFetch,
|
||||
NodeInfoURL: i.NodeInfoURL,
|
||||
ServerImplementationString: i.ServerImplementationString,
|
||||
ServerVersionString: i.ServerVersionString,
|
||||
SuccessCount: i.SuccessCount,
|
||||
}
|
||||
r := m.db.Create(&ni)
|
||||
return r.Error
|
||||
} else {
|
||||
log.Info().
|
||||
Str("hostname", i.Hostname).
|
||||
Str("id", i.UUID.String()).
|
||||
Msg("instance found in db, updating")
|
||||
// exists in db, update db
|
||||
var ei APInstance
|
||||
// EI EI uh-oh
|
||||
m.db.Where("UUID = ?", i.UUID).First(&ei)
|
||||
ei.Disabled = i.Disabled
|
||||
ei.ErrorCount = i.ErrorCount
|
||||
ei.FSMState = i.Status()
|
||||
ei.Fetching = i.Fetching
|
||||
ei.HighestID = i.HighestID
|
||||
ei.Hostname = i.Hostname
|
||||
ei.Identified = i.Identified
|
||||
ei.Implementation = string(i.Implementation)
|
||||
ei.NextFetch = i.NextFetch
|
||||
ei.NodeInfoURL = i.NodeInfoURL
|
||||
ei.ServerImplementationString = i.ServerImplementationString
|
||||
ei.ServerVersionString = i.ServerVersionString
|
||||
ei.SuccessCount = i.SuccessCount
|
||||
|
||||
r := m.db.Save(&ei)
|
||||
return r.Error
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) ListInstances() ([]*instance.Instance, error) {
|
||||
output := make([]*instance.Instance, 0)
|
||||
|
||||
var results []APInstance
|
||||
m.db.Find(&results)
|
||||
|
||||
for _, i := range results {
|
||||
newinst := instance.New(func(x *instance.Instance) {
|
||||
x.UUID = i.UUID
|
||||
x.Disabled = i.Disabled
|
||||
x.ErrorCount = i.ErrorCount
|
||||
x.InitialFSMState = i.FSMState
|
||||
x.Fetching = i.Fetching
|
||||
x.HighestID = i.HighestID
|
||||
x.Hostname = i.Hostname
|
||||
x.Identified = i.Identified
|
||||
x.Implementation = i.Implementation
|
||||
x.NextFetch = i.NextFetch
|
||||
x.NodeInfoURL = i.NodeInfoURL
|
||||
x.ServerImplementationString = i.ServerImplementationString
|
||||
x.ServerVersionString = i.ServerVersionString
|
||||
x.SuccessCount = i.SuccessCount
|
||||
})
|
||||
output = append(output, newinst)
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
66
database/manager.go
Normal file
66
database/manager.go
Normal file
@ -0,0 +1,66 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func New() *Manager {
|
||||
m := new(Manager)
|
||||
m.init()
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Manager) init() {
|
||||
m.open()
|
||||
m.db.LogMode(false)
|
||||
if viper.GetBool("Debug") {
|
||||
m.db.LogMode(true)
|
||||
}
|
||||
}
|
||||
|
||||
func mkdirp(p string) error {
|
||||
src, err := os.Stat(p)
|
||||
if os.IsNotExist(err) {
|
||||
errDir := os.MkdirAll(p, 0755)
|
||||
if errDir != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if src.Mode().IsRegular() {
|
||||
return errors.New("file already exists at path")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) open() {
|
||||
log.Info().Msg("opening database")
|
||||
dirname := filepath.Dir(viper.GetString("DbStorageLocation"))
|
||||
err := mkdirp(dirname)
|
||||
if err != nil {
|
||||
log.Panic().
|
||||
Err(err).
|
||||
Msg("db path erro")
|
||||
}
|
||||
|
||||
db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation"))
|
||||
if err != nil {
|
||||
log.Panic().
|
||||
Err(err).
|
||||
Msg("failed to open database")
|
||||
}
|
||||
m.db = db
|
||||
m.doMigrations()
|
||||
}
|
||||
49
database/model.go
Normal file
49
database/model.go
Normal file
@ -0,0 +1,49 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
)
|
||||
|
||||
type StoredToot struct {
|
||||
gorm.Model
|
||||
UUID uuid.UUID `gorm:"type:uuid;primary_key;"`
|
||||
//Original string `sql:"type:text"`
|
||||
Original []byte
|
||||
Hash string `gorm:"unique_index"`
|
||||
ServerCreated time.Time
|
||||
Acct string
|
||||
Content []byte
|
||||
URL string
|
||||
Hostname string
|
||||
}
|
||||
|
||||
type APInstance struct {
|
||||
gorm.Model
|
||||
UUID uuid.UUID `gorm:"type:uuid;primary_key;"`
|
||||
ErrorCount uint
|
||||
SuccessCount uint
|
||||
HighestID uint
|
||||
Hostname string `gorm:"type:varchar(100);unique_index"`
|
||||
Identified bool
|
||||
Fetching bool
|
||||
Disabled bool
|
||||
Implementation string
|
||||
NextFetch time.Time
|
||||
NodeInfoURL string
|
||||
ServerVersionString string
|
||||
ServerImplementationString string
|
||||
FSMState string
|
||||
}
|
||||
|
||||
// NB that when you add a model below you must add it to this list!
|
||||
func (m *Manager) doMigrations() {
|
||||
log.Info().Msg("doing database migrations if required")
|
||||
m.db.AutoMigrate(&APInstance{})
|
||||
m.db.AutoMigrate(&StoredToot{})
|
||||
}
|
||||
47
database/storageconnector.go
Normal file
47
database/storageconnector.go
Normal file
@ -0,0 +1,47 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
|
||||
"github.com/google/uuid"
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
)
|
||||
|
||||
func (m *Manager) TootExists(t *toot.Toot) bool {
|
||||
var try StoredToot
|
||||
if m.db.Where("Hash = ?", t.GetHash()).First(&try).RecordNotFound() {
|
||||
return false
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) StoreToot(t *toot.Toot) error {
|
||||
|
||||
nt := new(StoredToot)
|
||||
nt.UUID = uuid.New()
|
||||
nt.ServerCreated = t.Parsed.CreatedAt
|
||||
nt.Original = t.Original
|
||||
// FIXME normalize this, check for @ and append hostname if none
|
||||
nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost))
|
||||
nt.URL = t.Parsed.URL
|
||||
nt.Content = t.Parsed.Content
|
||||
nt.Hostname = strings.ToLower(t.FromHost)
|
||||
nt.Hash = t.GetHash()
|
||||
r := m.db.Create(&nt)
|
||||
//panic(fmt.Sprintf("%+v", t))
|
||||
return r.Error
|
||||
}
|
||||
|
||||
func (m *Manager) StoreToots(tc []*toot.Toot) error {
|
||||
for _, item := range tc {
|
||||
err := m.StoreToot(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
package db
|
||||
|
||||
import "git.eeqj.de/sneak/feta/process"
|
||||
import "github.com/jinzhu/gorm"
|
||||
import _ "github.com/jinzhu/gorm/dialects/sqlite" // required for orm
|
||||
|
||||
type savedInstance struct {
|
||||
gorm.Model
|
||||
hostname string
|
||||
software string
|
||||
}
|
||||
|
||||
func (f *process.Feta) databaseMigrations() {
|
||||
f.db.AutoMigrate(&savedInstance{})
|
||||
}
|
||||
18
go.mod
Normal file
18
go.mod
Normal file
@ -0,0 +1,18 @@
|
||||
module git.eeqj.de/sneak/feta
|
||||
|
||||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
|
||||
github.com/gin-gonic/gin v1.6.2
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/jinzhu/gorm v1.9.12
|
||||
github.com/k0kubun/pp v3.0.1+incompatible
|
||||
github.com/looplab/fsm v0.1.0
|
||||
github.com/mattn/go-colorable v0.1.6 // indirect
|
||||
github.com/mattn/go-isatty v0.0.12
|
||||
github.com/multiformats/go-multihash v0.0.13
|
||||
github.com/rs/zerolog v1.18.0
|
||||
github.com/spf13/viper v1.6.2
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
|
||||
)
|
||||
233
go.sum
Normal file
233
go.sum
Normal file
@ -0,0 +1,233 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1 h1:qwfOp+dwJnhdRFWsXkRMb+EZz0BgMQ8VD77OgBjuRUQ=
|
||||
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1/go.mod h1:AAlcXL9Ejp3TUsJRWJtjbIpK3p1L9z987raCTYL17j4=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM=
|
||||
github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
|
||||
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
|
||||
github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
|
||||
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q=
|
||||
github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/k0kubun/pp v1.3.0 h1:r9td75hcmetrcVbmsZRjnxcIbI9mhm+/N6iWyG4TWe0=
|
||||
github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
|
||||
github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20=
|
||||
github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
|
||||
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
|
||||
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
|
||||
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
|
||||
github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
|
||||
github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
@ -1,9 +1,12 @@
|
||||
package ingester
|
||||
|
||||
import "time"
|
||||
import "github.com/rs/zerolog/log"
|
||||
import "git.eeqj.de/sneak/feta/toot"
|
||||
import "git.eeqj.de/sneak/feta/storage"
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/feta/storage"
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// TootIngester is the data structure for the ingester process that is
|
||||
// responsible for storing the discovered toots
|
||||
@ -15,7 +18,7 @@ type TootIngester struct {
|
||||
|
||||
type seenTootMemo struct {
|
||||
lastSeen time.Time
|
||||
tootHash toot.Hash
|
||||
tootHash string
|
||||
}
|
||||
|
||||
// NewTootIngester returns a fresh TootIngester for your use
|
||||
@ -55,5 +58,5 @@ func (ti *TootIngester) storeToot(t *toot.Toot) {
|
||||
if ti.storageBackend == nil {
|
||||
panic("no storage backend")
|
||||
}
|
||||
ti.storageBackend.StoreToot(*t)
|
||||
ti.storageBackend.StoreToot(t)
|
||||
}
|
||||
|
||||
@ -1,20 +1,23 @@
|
||||
package instance
|
||||
|
||||
import "encoding/json"
|
||||
import "fmt"
|
||||
import "io/ioutil"
|
||||
import "net/http"
|
||||
import "strings"
|
||||
import "sync"
|
||||
import "time"
|
||||
import "errors"
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/feta/jsonapis"
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
"github.com/google/uuid"
|
||||
"github.com/looplab/fsm"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
//import "github.com/gin-gonic/gin"
|
||||
import "github.com/looplab/fsm"
|
||||
import "github.com/rs/zerolog/log"
|
||||
import "git.eeqj.de/sneak/feta/storage"
|
||||
import "git.eeqj.de/sneak/feta/toot"
|
||||
import "git.eeqj.de/sneak/feta/jsonapis"
|
||||
|
||||
const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
|
||||
const instanceNodeinfoTimeout = time.Second * 50
|
||||
@ -22,47 +25,43 @@ const instanceHTTPTimeout = time.Second * 120
|
||||
const instanceSpiderInterval = time.Second * 120
|
||||
const instanceErrorInterval = time.Second * 60 * 30
|
||||
|
||||
type instanceImplementation int
|
||||
|
||||
// Hostname is a special type for holding the hostname of an
|
||||
// instance (string)
|
||||
type Hostname string
|
||||
|
||||
const (
|
||||
implUnknown instanceImplementation = iota
|
||||
implMastodon
|
||||
implPleroma
|
||||
)
|
||||
|
||||
// Instance stores all the information we know about an instance
|
||||
type Instance struct {
|
||||
structLock sync.Mutex
|
||||
tootDestination chan *toot.Toot
|
||||
Disabled bool
|
||||
ErrorCount uint
|
||||
SuccessCount uint
|
||||
highestID int
|
||||
FSM *fsm.FSM
|
||||
Fetching bool
|
||||
HighestID uint
|
||||
Hostname string
|
||||
Identified bool
|
||||
fetching bool
|
||||
implementation instanceImplementation
|
||||
storageBackend *storage.TootStorageBackend
|
||||
Implementation string
|
||||
InitialFSMState string
|
||||
NextFetch time.Time
|
||||
nodeInfoURL string
|
||||
ServerVersionString string
|
||||
NodeInfoURL string
|
||||
ServerImplementationString string
|
||||
ServerVersionString string
|
||||
SuccessCount uint
|
||||
UUID uuid.UUID
|
||||
fetchingLock sync.Mutex
|
||||
fsm *fsm.FSM
|
||||
fsmLock sync.Mutex
|
||||
structLock sync.Mutex
|
||||
tootDestination chan *toot.Toot
|
||||
}
|
||||
|
||||
// New returns a new instance, argument is a function that operates on the
|
||||
// new instance
|
||||
func New(options ...func(i *Instance)) *Instance {
|
||||
i := new(Instance)
|
||||
i.UUID = uuid.New()
|
||||
i.setNextFetchAfter(1 * time.Second)
|
||||
i.InitialFSMState = "STATUS_UNKNOWN"
|
||||
|
||||
i.fsm = fsm.NewFSM(
|
||||
"STATUS_UNKNOWN",
|
||||
for _, opt := range options {
|
||||
opt(i)
|
||||
}
|
||||
|
||||
i.FSM = fsm.NewFSM(
|
||||
i.InitialFSMState,
|
||||
fsm.Events{
|
||||
{Name: "BEGIN_NODEINFO_URL_FETCH", Src: []string{"STATUS_UNKNOWN"}, Dst: "FETCHING_NODEINFO_URL"},
|
||||
{Name: "GOT_NODEINFO_URL", Src: []string{"FETCHING_NODEINFO_URL"}, Dst: "PRE_NODEINFO_FETCH"},
|
||||
@ -79,10 +78,6 @@ func New(options ...func(i *Instance)) *Instance {
|
||||
"enter_state": func(e *fsm.Event) { i.fsmEnterState(e) },
|
||||
},
|
||||
)
|
||||
|
||||
for _, opt := range options {
|
||||
opt(i)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
@ -90,7 +85,7 @@ func New(options ...func(i *Instance)) *Instance {
|
||||
func (i *Instance) Status() string {
|
||||
i.fsmLock.Lock()
|
||||
defer i.fsmLock.Unlock()
|
||||
return i.fsm.Current()
|
||||
return i.FSM.Current()
|
||||
}
|
||||
|
||||
// SetTootDestination takes a channel from the manager that all toots
|
||||
@ -105,7 +100,7 @@ func (i *Instance) SetTootDestination(d chan *toot.Toot) {
|
||||
func (i *Instance) Event(eventname string) {
|
||||
i.fsmLock.Lock()
|
||||
defer i.fsmLock.Unlock()
|
||||
i.fsm.Event(eventname)
|
||||
i.FSM.Event(eventname)
|
||||
}
|
||||
|
||||
func (i *Instance) fsmEnterState(e *fsm.Event) {
|
||||
@ -192,7 +187,7 @@ func (i *Instance) Tick() {
|
||||
func (i *Instance) nodeIdentified() bool {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
if i.implementation > implUnknown {
|
||||
if i.Implementation != "" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@ -282,7 +277,7 @@ func (i *Instance) fetchNodeInfoURL() error {
|
||||
Msg("success fetching url for nodeinfo")
|
||||
|
||||
i.Lock()
|
||||
i.nodeInfoURL = item.Href
|
||||
i.NodeInfoURL = item.Href
|
||||
i.Unlock()
|
||||
i.registerSuccess()
|
||||
i.Event("GOT_NODEINFO_URL")
|
||||
@ -317,7 +312,7 @@ func (i *Instance) fetchNodeInfo() error {
|
||||
//FIXME make sure the nodeinfourl is on the same domain as the instance
|
||||
//hostname
|
||||
i.Lock()
|
||||
url := i.nodeInfoURL
|
||||
url := i.NodeInfoURL
|
||||
i.Unlock()
|
||||
|
||||
i.Event("BEGIN_NODEINFO_FETCH")
|
||||
@ -362,7 +357,7 @@ func (i *Instance) fetchNodeInfo() error {
|
||||
Str("serverVersion", ni.Software.Version).
|
||||
Str("software", ni.Software.Name).
|
||||
Str("hostname", i.Hostname).
|
||||
Str("nodeInfoURL", i.nodeInfoURL).
|
||||
Str("nodeInfoURL", i.NodeInfoURL).
|
||||
Msg("received nodeinfo from instance")
|
||||
|
||||
i.Lock()
|
||||
@ -376,7 +371,7 @@ func (i *Instance) fetchNodeInfo() error {
|
||||
Str("software", ni.Software.Name).
|
||||
Msg("detected server software")
|
||||
i.Identified = true
|
||||
i.implementation = implPleroma
|
||||
i.Implementation = "pleroma"
|
||||
i.Unlock()
|
||||
i.registerSuccess()
|
||||
i.Event("GOT_NODEINFO")
|
||||
@ -387,7 +382,7 @@ func (i *Instance) fetchNodeInfo() error {
|
||||
Str("software", ni.Software.Name).
|
||||
Msg("detected server software")
|
||||
i.Identified = true
|
||||
i.implementation = implMastodon
|
||||
i.Implementation = "mastodon"
|
||||
i.Unlock()
|
||||
i.registerSuccess()
|
||||
i.Event("GOT_NODEINFO")
|
||||
|
||||
@ -1,16 +1,17 @@
|
||||
package locator
|
||||
|
||||
import "encoding/json"
|
||||
import "io/ioutil"
|
||||
import "net/http"
|
||||
import "time"
|
||||
import "sync"
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
import "github.com/rs/zerolog/log"
|
||||
import "golang.org/x/sync/semaphore"
|
||||
import "git.eeqj.de/sneak/feta/jsonapis"
|
||||
import "git.eeqj.de/sneak/feta/instance"
|
||||
import "git.eeqj.de/sneak/feta"
|
||||
"git.eeqj.de/sneak/feta/jsonapis"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
// IndexAPITimeout is the timeout for fetching json instance lists
|
||||
// from the listing servers
|
||||
@ -35,7 +36,7 @@ const pleromaIndexURL = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api
|
||||
type InstanceLocator struct {
|
||||
pleromaIndexNextRefresh *time.Time
|
||||
mastodonIndexNextRefresh *time.Time
|
||||
reportInstanceVia chan instance.Hostname
|
||||
reportInstanceVia chan string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
@ -58,13 +59,13 @@ func (il *InstanceLocator) unlock() {
|
||||
|
||||
// SetInstanceNotificationChannel is the way the instanceLocator returns
|
||||
// newly discovered instances back to the manager for query/addition
|
||||
func (il *InstanceLocator) SetInstanceNotificationChannel(via chan instance.Hostname) {
|
||||
func (il *InstanceLocator) SetInstanceNotificationChannel(via chan string) {
|
||||
il.lock()
|
||||
defer il.unlock()
|
||||
il.reportInstanceVia = via
|
||||
}
|
||||
|
||||
func (il *InstanceLocator) addInstance(hostname instance.Hostname) {
|
||||
func (il *InstanceLocator) addInstance(hostname string) {
|
||||
// receiver (InstanceManager) is responsible for de-duping against its
|
||||
// map, we just locate and spray, it manages
|
||||
il.reportInstanceVia <- hostname
|
||||
@ -120,8 +121,7 @@ func (il *InstanceLocator) Locate() {
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
c := feta.GetConfig()
|
||||
if time.Now().After(x.Add(c.LogReportInterval)) {
|
||||
if time.Now().After(x.Add(viper.GetDuration("LogReportInterval"))) {
|
||||
x = time.Now()
|
||||
log.Debug().
|
||||
Str("nextMastodonIndexRefresh", il.durationUntilNextMastodonIndexRefresh().String()).
|
||||
@ -198,7 +198,7 @@ func (il *InstanceLocator) locateMastodon() {
|
||||
Msg("received hosts from mastodon index")
|
||||
|
||||
for k := range hosts {
|
||||
il.addInstance(instance.Hostname(k))
|
||||
il.addInstance(k)
|
||||
}
|
||||
|
||||
}
|
||||
@ -266,7 +266,7 @@ func (il *InstanceLocator) locatePleroma() {
|
||||
Msg("received hosts from pleroma index")
|
||||
|
||||
for k := range hosts {
|
||||
il.addInstance(instance.Hostname(k))
|
||||
il.addInstance(k)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,38 +1,75 @@
|
||||
package manager
|
||||
|
||||
import "sync"
|
||||
import "time"
|
||||
import "runtime"
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
//import "github.com/gin-gonic/gin"
|
||||
import "github.com/rs/zerolog/log"
|
||||
import "git.eeqj.de/sneak/feta/toot"
|
||||
import "git.eeqj.de/sneak/feta/seeds"
|
||||
import "git.eeqj.de/sneak/feta/instance"
|
||||
"git.eeqj.de/sneak/feta/instance"
|
||||
"git.eeqj.de/sneak/feta/seeds"
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
const hostDiscoveryParallelism = 5
|
||||
|
||||
// LogReportInterval defines how long between logging internal
|
||||
// stats/reporting for user supervision
|
||||
var LogReportInterval = time.Second * 10
|
||||
// conform for storing toots
|
||||
type DatabaseStorage interface {
|
||||
ListInstances() ([]*instance.Instance, error)
|
||||
//StoreInstances([]*instance.Instance) error
|
||||
SaveInstance(*instance.Instance) error
|
||||
}
|
||||
|
||||
// InstanceManager is the main data structure for the goroutine that manages
|
||||
// the list of all known instances, fed by the locator
|
||||
type InstanceManager struct {
|
||||
mu sync.Mutex
|
||||
instances map[instance.Hostname]*instance.Instance
|
||||
newInstanceNotifications chan instance.Hostname
|
||||
db DatabaseStorage
|
||||
instances map[string]*instance.Instance
|
||||
newInstanceNotifications chan string
|
||||
tootDestination chan *toot.Toot
|
||||
startup time.Time
|
||||
hostAdderSemaphore chan bool
|
||||
nextDBSave time.Time
|
||||
}
|
||||
|
||||
// New returns a new InstanceManager for use by the Process
|
||||
func New() *InstanceManager {
|
||||
i := new(InstanceManager)
|
||||
i.hostAdderSemaphore = make(chan bool, hostDiscoveryParallelism)
|
||||
i.instances = make(map[instance.Hostname]*instance.Instance)
|
||||
return i
|
||||
func New(db DatabaseStorage) *InstanceManager {
|
||||
im := new(InstanceManager)
|
||||
im.db = db
|
||||
im.hostAdderSemaphore = make(chan bool, viper.GetInt("HostDiscoveryParallelism"))
|
||||
im.instances = make(map[string]*instance.Instance)
|
||||
im.RestoreFromDB()
|
||||
return im
|
||||
}
|
||||
|
||||
func (im *InstanceManager) RestoreFromDB() {
|
||||
newil, err := im.db.ListInstances()
|
||||
if err != nil {
|
||||
log.Panic().
|
||||
Err(err).
|
||||
Msg("cannot get instance list from db")
|
||||
}
|
||||
im.lock()
|
||||
defer im.unlock()
|
||||
count := 0
|
||||
for _, x := range newil {
|
||||
x.SetTootDestination(im.tootDestination)
|
||||
im.instances[x.Hostname] = x
|
||||
count = count + 1
|
||||
}
|
||||
log.Info().
|
||||
Int("count", count).
|
||||
Msg("restored instances from database")
|
||||
}
|
||||
|
||||
func (im *InstanceManager) SaveToDB() {
|
||||
for _, x := range im.ListInstances() {
|
||||
err := im.db.SaveInstance(x)
|
||||
if err != nil {
|
||||
log.Panic().
|
||||
Err(err).
|
||||
Msg("cannot write to db")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetTootDestination provides the instancemanager with a channel to the
|
||||
@ -41,30 +78,6 @@ func (im *InstanceManager) SetTootDestination(td chan *toot.Toot) {
|
||||
im.tootDestination = td
|
||||
}
|
||||
|
||||
func (im *InstanceManager) logCaller(msg string) {
|
||||
fpcs := make([]uintptr, 1)
|
||||
// Skip 2 levels to get the caller
|
||||
n := runtime.Callers(3, fpcs)
|
||||
if n == 0 {
|
||||
log.Debug().Msg("MSG: NO CALLER")
|
||||
}
|
||||
|
||||
caller := runtime.FuncForPC(fpcs[0] - 1)
|
||||
if caller == nil {
|
||||
log.Debug().Msg("MSG CALLER WAS NIL")
|
||||
}
|
||||
|
||||
// Print the file name and line number
|
||||
filename, line := caller.FileLine(fpcs[0] - 1)
|
||||
function := caller.Name()
|
||||
|
||||
log.Debug().
|
||||
Str("filename", filename).
|
||||
Int("linenum", line).
|
||||
Str("function", function).
|
||||
Msg(msg)
|
||||
}
|
||||
|
||||
func (im *InstanceManager) lock() {
|
||||
im.mu.Lock()
|
||||
}
|
||||
@ -77,7 +90,7 @@ func (im *InstanceManager) unlock() {
|
||||
// InstanceManager about the channel from the InstanceLocator so that the
|
||||
// InstanceLocator can provide it/us (the InstanceManager) with new
|
||||
// instance.Hostnames. We (the manager) deduplicate the list ourselves.
|
||||
func (im *InstanceManager) SetInstanceNotificationChannel(via chan instance.Hostname) {
|
||||
func (im *InstanceManager) SetInstanceNotificationChannel(via chan string) {
|
||||
im.lock()
|
||||
defer im.unlock()
|
||||
im.newInstanceNotifications = via
|
||||
@ -85,9 +98,9 @@ func (im *InstanceManager) SetInstanceNotificationChannel(via chan instance.Host
|
||||
|
||||
func (im *InstanceManager) receiveSeedInstanceHostnames() {
|
||||
for _, x := range seeds.SeedInstances {
|
||||
go func(tmp instance.Hostname) {
|
||||
go func(tmp string) {
|
||||
im.addInstanceByHostname(tmp)
|
||||
}(instance.Hostname(x))
|
||||
}(x)
|
||||
}
|
||||
}
|
||||
|
||||
@ -110,10 +123,15 @@ func (im *InstanceManager) Manage() {
|
||||
log.Info().Msg("InstanceManager tick")
|
||||
im.managerLoop()
|
||||
time.Sleep(1 * time.Second)
|
||||
if time.Now().After(x.Add(LogReportInterval)) {
|
||||
if time.Now().After(x.Add(viper.GetDuration("LogReportInterval"))) {
|
||||
x = time.Now()
|
||||
im.logInstanceReport()
|
||||
}
|
||||
|
||||
if im.nextDBSave.Before(time.Now()) {
|
||||
im.nextDBSave = time.Now().Add(time.Second * 60)
|
||||
im.SaveToDB()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -133,7 +151,7 @@ func (im *InstanceManager) managerLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
func (im *InstanceManager) hostnameExists(newhn instance.Hostname) bool {
|
||||
func (im *InstanceManager) hostnameExists(newhn string) bool {
|
||||
im.lock()
|
||||
defer im.unlock()
|
||||
for k := range im.instances {
|
||||
@ -144,7 +162,7 @@ func (im *InstanceManager) hostnameExists(newhn instance.Hostname) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (im *InstanceManager) addInstanceByHostname(newhn instance.Hostname) {
|
||||
func (im *InstanceManager) addInstanceByHostname(newhn string) {
|
||||
if im.hostnameExists(newhn) {
|
||||
// ignore adding new if we already know about it
|
||||
return
|
||||
@ -172,7 +190,7 @@ func (im *InstanceManager) addInstanceByHostname(newhn instance.Hostname) {
|
||||
}
|
||||
|
||||
func (im *InstanceManager) receiveNewInstanceHostnames() {
|
||||
var newhn instance.Hostname
|
||||
var newhn string
|
||||
for {
|
||||
newhn = <-im.newInstanceNotifications
|
||||
// receive them fast out of the channel, let the adding function lock to add
|
||||
|
||||
100
process/feta.go
100
process/feta.go
@ -1,27 +1,32 @@
|
||||
package process
|
||||
|
||||
import "os"
|
||||
import "time"
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
import "github.com/jinzhu/gorm"
|
||||
import _ "github.com/jinzhu/gorm/dialects/sqlite" // required for orm
|
||||
"git.eeqj.de/sneak/feta/database"
|
||||
"git.eeqj.de/sneak/feta/ingester"
|
||||
"git.eeqj.de/sneak/feta/locator"
|
||||
"git.eeqj.de/sneak/feta/manager"
|
||||
"git.eeqj.de/sneak/feta/storage"
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
"github.com/k0kubun/pp"
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
import "github.com/rs/zerolog"
|
||||
import "github.com/rs/zerolog/log"
|
||||
import "github.com/mattn/go-isatty"
|
||||
|
||||
import "git.eeqj.de/sneak/feta/ingester"
|
||||
import "git.eeqj.de/sneak/feta/storage"
|
||||
import "git.eeqj.de/sneak/feta/locator"
|
||||
import "git.eeqj.de/sneak/feta/manager"
|
||||
import "git.eeqj.de/sneak/feta/instance"
|
||||
// required for orm
|
||||
|
||||
// CLIEntry is the main entrypoint for the feta process from the cli
|
||||
func CLIEntry(version string, buildarch string) int {
|
||||
f := new(Feta)
|
||||
f.version = version
|
||||
f.buildarch = buildarch
|
||||
f.configure()
|
||||
f.setupLogging()
|
||||
f.setupDatabase()
|
||||
return f.runForever()
|
||||
}
|
||||
|
||||
@ -33,10 +38,43 @@ type Feta struct {
|
||||
manager *manager.InstanceManager
|
||||
ingester *ingester.TootIngester
|
||||
api *Server
|
||||
db *gorm.DB
|
||||
dbm *database.Manager
|
||||
startup time.Time
|
||||
}
|
||||
|
||||
func (f *Feta) configure() {
|
||||
viper.SetConfigName("feta")
|
||||
viper.SetConfigType("yaml")
|
||||
viper.AddConfigPath("/etc/feta") // path to look for the config file in
|
||||
viper.AddConfigPath("$HOME/.config/feta") // call multiple times to add many search paths
|
||||
|
||||
viper.SetEnvPrefix("FETA")
|
||||
viper.AutomaticEnv()
|
||||
|
||||
viper.SetDefault("Debug", false)
|
||||
viper.SetDefault("TootsToDisk", false)
|
||||
viper.SetDefault("TootsToDB", true)
|
||||
viper.SetDefault("HostDiscoveryParallelism", 5)
|
||||
viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d"))
|
||||
viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))
|
||||
viper.SetDefault("LogReportInterval", time.Second*10)
|
||||
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
|
||||
// Config file not found; ignore error if desired
|
||||
} else {
|
||||
// Config file was found but another error was produced
|
||||
log.Panic().
|
||||
Err(err).
|
||||
Msg("cannot read config file")
|
||||
}
|
||||
}
|
||||
|
||||
if viper.GetBool("debug") {
|
||||
pp.Print(viper.AllSettings())
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Feta) identify() {
|
||||
log.Info().
|
||||
Str("version", f.version).
|
||||
@ -44,6 +82,9 @@ func (f *Feta) identify() {
|
||||
Msg("starting")
|
||||
}
|
||||
|
||||
func (f *Feta) setupDatabase() {
|
||||
}
|
||||
|
||||
func (f *Feta) setupLogging() {
|
||||
|
||||
log.Logger = log.With().Caller().Logger()
|
||||
@ -66,7 +107,7 @@ func (f *Feta) setupLogging() {
|
||||
}
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
if os.Getenv("DEBUG") != "" {
|
||||
if viper.GetBool("debug") {
|
||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||
}
|
||||
|
||||
@ -77,19 +118,6 @@ func (f *Feta) uptime() time.Duration {
|
||||
return time.Since(f.startup)
|
||||
}
|
||||
|
||||
/*
|
||||
func (f *Feta) setupDatabase() {
|
||||
var err error
|
||||
f.db, err = gorm.Open("sqlite3", "feta.sqlite")
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
//f.databaseMigrations()
|
||||
}
|
||||
*/
|
||||
|
||||
func (f *Feta) runForever() int {
|
||||
f.startup = time.Now()
|
||||
|
||||
@ -97,10 +125,14 @@ func (f *Feta) runForever() int {
|
||||
|
||||
// FIXME move this channel creation into the manager's constructor
|
||||
// and add getters/setters on the manager/locator
|
||||
newInstanceHostnameNotifications := make(chan instance.Hostname)
|
||||
newInstanceHostnameNotifications := make(chan string)
|
||||
|
||||
f.dbm = database.New()
|
||||
|
||||
f.locator = locator.New()
|
||||
f.manager = manager.New()
|
||||
|
||||
f.manager = manager.New(f.dbm)
|
||||
|
||||
f.ingester = ingester.NewTootIngester()
|
||||
|
||||
home := os.Getenv("HOME")
|
||||
@ -108,8 +140,14 @@ func (f *Feta) runForever() int {
|
||||
panic("can't find home directory")
|
||||
}
|
||||
|
||||
diskBackend := storage.NewTootFSStorage(home + "/.local/feta")
|
||||
if viper.GetBool("TootsToDB") {
|
||||
f.ingester.SetStorageBackend(f.dbm)
|
||||
} else if viper.GetBool("TootsToDisk") {
|
||||
diskBackend := storage.NewTootFSStorage(viper.GetString("FSStorageLocation"))
|
||||
f.ingester.SetStorageBackend(diskBackend)
|
||||
} else {
|
||||
log.Info().Msg("toots will not be saved to disk")
|
||||
}
|
||||
|
||||
f.api = new(Server)
|
||||
f.api.SetFeta(f) // api needs to get to us to access data
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
package process
|
||||
|
||||
import "time"
|
||||
import "net/http"
|
||||
import "encoding/json"
|
||||
import "runtime"
|
||||
import "fmt"
|
||||
import "strings"
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
import "github.com/gin-gonic/gin"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type hash map[string]interface{}
|
||||
|
||||
|
||||
140
seeds/seeds.go
140
seeds/seeds.go
@ -19,9 +19,7 @@ var SeedInstances = [...]string{
|
||||
"kiwifarms.cc",
|
||||
"kiwifarms.is",
|
||||
"kiwifarms.net",
|
||||
"kneegrows.top",
|
||||
"knzk.me",
|
||||
"kowai.youkai.town",
|
||||
"koyu.space",
|
||||
"krauser.org",
|
||||
"kuko.hamburg",
|
||||
@ -34,12 +32,10 @@ var SeedInstances = [...]string{
|
||||
"layer8.space",
|
||||
"leftlibertarian.club",
|
||||
"lesbian.energy",
|
||||
"lets.saynoto.lgbt",
|
||||
"letsalllovela.in",
|
||||
"lgbtq.cool",
|
||||
"lgbtqia.is",
|
||||
"liberdon.com",
|
||||
"libertarianism.club",
|
||||
"librem.one",
|
||||
"librenet.co.za",
|
||||
"ligma.pro",
|
||||
@ -51,11 +47,9 @@ var SeedInstances = [...]string{
|
||||
"loci.onl",
|
||||
"logjam.city",
|
||||
"lol5.tun.a4.io",
|
||||
"loli.estate",
|
||||
"lolis.world",
|
||||
"lost-angles.im",
|
||||
"luvdon.cc",
|
||||
"luvdon.ddns.net",
|
||||
"m.1994.io",
|
||||
"m.apertron.com",
|
||||
"m.bnolet.me",
|
||||
@ -69,7 +63,6 @@ var SeedInstances = [...]string{
|
||||
"majak.de",
|
||||
"makito.me",
|
||||
"maly.io",
|
||||
"manx.social",
|
||||
"marchgenso.me",
|
||||
"mares.cafe",
|
||||
"mas.korrigan.tech",
|
||||
@ -92,8 +85,6 @@ var SeedInstances = [...]string{
|
||||
"mastodon.aventer.biz",
|
||||
"mastodon.blue",
|
||||
"mastodon.cipherbliss.com",
|
||||
"mastodon.circlelinego.com",
|
||||
"mastodon.codeplumbers.eu",
|
||||
"mastodon.coder.town",
|
||||
"mastodon.com.pl",
|
||||
"mastodon.corecoding.dev",
|
||||
@ -102,7 +93,6 @@ var SeedInstances = [...]string{
|
||||
"mastodon.echoz.io",
|
||||
"mastodon.eric.ovh",
|
||||
"mastodon.ericbeckers.nl",
|
||||
"mastodon.fail",
|
||||
"mastodon.freifunk-minden.de",
|
||||
"mastodon.fricloud.dk",
|
||||
"mastodon.funigtor.fr",
|
||||
@ -118,15 +108,12 @@ var SeedInstances = [...]string{
|
||||
"mastodon.ie",
|
||||
"mastodon.immae.eu",
|
||||
"mastodon.inferiorlattice.com",
|
||||
"mastodon.inhji.de",
|
||||
"mastodon.jectrum.de",
|
||||
"mastodon.jeder.pl",
|
||||
"mastodon.kerenon.com",
|
||||
"mastodon.kliu.io",
|
||||
"mastodon.kosebamse.com",
|
||||
"mastodon.leptonics.com",
|
||||
"mastodon.local.lubar.me",
|
||||
"mastodon.loliandstuff.moe",
|
||||
"mastodon.lubar.me",
|
||||
"mastodon.lunorian.is",
|
||||
"mastodon.macsnet.cz",
|
||||
@ -134,14 +121,11 @@ var SeedInstances = [...]string{
|
||||
"mastodon.me.uk",
|
||||
"mastodon.mynameisivan.ru",
|
||||
"mastodon.naoy.fr",
|
||||
"mastodon.nobodysstuff.de",
|
||||
"mastodon.ocf.berkeley.edu",
|
||||
"mastodon.openpsychology.net",
|
||||
"mastodon.org.ua",
|
||||
"mastodon.org.uk",
|
||||
"mastodon.otherreality.net",
|
||||
"mastodon.owls.io",
|
||||
"mastodon.redflag.social",
|
||||
"mastodon.roocita.com",
|
||||
"mastodon.rylees.net",
|
||||
"mastodon.scarletsisters.xyz",
|
||||
@ -154,18 +138,13 @@ var SeedInstances = [...]string{
|
||||
"mastodon.soses.ca",
|
||||
"mastodon.spiderden.net",
|
||||
"mastodon.starrevolution.org",
|
||||
"mastodon.syntik.fr",
|
||||
"mastodon.technology",
|
||||
"mastodon.technology",
|
||||
"mastodon.toni.im",
|
||||
"mastodon.toniozz75.fr",
|
||||
"mastodon.truf-kin.com",
|
||||
"mastodon.xhrpb.com",
|
||||
"mastodon.yolovision-inc.com",
|
||||
"mastodon.zapashcanon.fr",
|
||||
"mastodon.zwei.net",
|
||||
"mastofant.de",
|
||||
"masttest.zwei.net",
|
||||
"mcphail.uk",
|
||||
"me.frankmeeuwsen.xyz",
|
||||
"megadon.net",
|
||||
@ -181,24 +160,20 @@ var SeedInstances = [...]string{
|
||||
"mobile.co",
|
||||
"monsterpit.net",
|
||||
"moytura.org",
|
||||
"mst.mpdevel.com",
|
||||
"mst.thewebzone.net",
|
||||
"mst.vsta.org",
|
||||
"mstdn.alternanet.fr",
|
||||
"mstdn.ikebuku.ro",
|
||||
"mstdn.io",
|
||||
"mstdn.jp",
|
||||
"mstdn.maud.io",
|
||||
"mstdn.mx",
|
||||
"mstdn.novium.pw",
|
||||
"mstdn.openalgeria.org",
|
||||
"mstdn.social",
|
||||
"mstdn.tsukiyono.0am.jp",
|
||||
"mstdn.waifu.space",
|
||||
"mstdn.xxil.cc",
|
||||
"mu.zaitcev.nu",
|
||||
"mudl.us",
|
||||
"multicast.social",
|
||||
"music.pawoo.net",
|
||||
"myflog.net",
|
||||
"mypolis.zapto.org",
|
||||
@ -222,16 +197,13 @@ var SeedInstances = [...]string{
|
||||
"norden.social",
|
||||
"nordenmedia.com",
|
||||
"not-develop.gab.com",
|
||||
"not.phrack.fyi",
|
||||
"npf.mlpol.net",
|
||||
"nsfw.social",
|
||||
"nudie.social",
|
||||
"nyaa.social",
|
||||
"octodon.social",
|
||||
"odin.run",
|
||||
"ohhi.icu",
|
||||
"oneway.masto.host",
|
||||
"opensim.fun",
|
||||
"order.life",
|
||||
"oslo.town",
|
||||
"our.wtf",
|
||||
@ -254,7 +226,6 @@ var SeedInstances = [...]string{
|
||||
"pl.knotteye.cc",
|
||||
"pl.kotobank.ch",
|
||||
"pl.koyu.space",
|
||||
"pl.kys.moe",
|
||||
"pl.ohno.host",
|
||||
"pl.smuglo.li",
|
||||
"pl.wowana.me",
|
||||
@ -273,27 +244,20 @@ var SeedInstances = [...]string{
|
||||
"pleroma.quaylessed.icu",
|
||||
"pleroma.rareome.ga",
|
||||
"pleroma.soykaf.com",
|
||||
"pleroma.teromene.fr",
|
||||
"pleroma.travnewmatic.com",
|
||||
"pleroma.tuxcrafting.cf",
|
||||
"pleroma.yorha.club",
|
||||
"pltest.feminism.lgbt",
|
||||
"plural.cafe",
|
||||
"pokemon.men",
|
||||
"polycule.club",
|
||||
"pornfed.social",
|
||||
"porntoot.com",
|
||||
"post.mashek.net",
|
||||
"pouet.jablon.fr",
|
||||
"ppl.town",
|
||||
"preteengirls.biz",
|
||||
"pridelands.io",
|
||||
"princess.cat",
|
||||
"privacytools.io",
|
||||
"producers.masto.host",
|
||||
"programmer.technology",
|
||||
"programmingsocks.com",
|
||||
"project.social",
|
||||
"protohype.net",
|
||||
"prsm.space",
|
||||
"psyopshop.com",
|
||||
@ -305,16 +269,12 @@ var SeedInstances = [...]string{
|
||||
"queersin.space",
|
||||
"quey.org",
|
||||
"quitter.pw",
|
||||
"r3bl.social",
|
||||
"rainbowdash.net",
|
||||
"raki.social",
|
||||
"rapefeminists.network",
|
||||
"rebels.rest",
|
||||
"redliberal.com",
|
||||
"redroo.ml",
|
||||
"redterrorcollective.net",
|
||||
"relay-mypolis.zapto.org",
|
||||
"relay.selfhosting.rocks",
|
||||
"remotenode.host",
|
||||
"rhubarb.land",
|
||||
"rigcz.club",
|
||||
@ -327,7 +287,6 @@ var SeedInstances = [...]string{
|
||||
"rrfarmbot.appspot.com",
|
||||
"rubber.social",
|
||||
"rva.party",
|
||||
"s.b252.gq",
|
||||
"s.huggingservers.uk",
|
||||
"sackheads.social",
|
||||
"sadposting.space",
|
||||
@ -344,7 +303,6 @@ var SeedInstances = [...]string{
|
||||
"shitasstits.life",
|
||||
"shitposter.club",
|
||||
"shpposter.club",
|
||||
"simstodon.com",
|
||||
"simulacron.de",
|
||||
"sinblr.com",
|
||||
"skippers-bin.com",
|
||||
@ -354,28 +312,21 @@ var SeedInstances = [...]string{
|
||||
"sn.angry.im",
|
||||
"snabelen.no",
|
||||
"snaggletooth.life",
|
||||
"snel.social",
|
||||
"snuskete.net",
|
||||
"soc.psychedelic.cat",
|
||||
"social.1in9.net",
|
||||
"social.adlerweb.info",
|
||||
"social.allthefallen.ninja",
|
||||
"social.au2pb.net",
|
||||
"social.avareborn.de",
|
||||
"social.azkware.net",
|
||||
"social.b252.gq",
|
||||
"social.backbord.net",
|
||||
"social.bam.yt",
|
||||
"social.bau-ha.us",
|
||||
"social.beepboop.ga",
|
||||
"social.cereza.de",
|
||||
"social.cloudsumu.com",
|
||||
"social.culturewar.us",
|
||||
"social.cutienaut.club",
|
||||
"social.digimortal.org",
|
||||
"social.elqhost.net",
|
||||
"social.end-the-stigma.com",
|
||||
"social.enyutech.io",
|
||||
"social.fab-l3.org",
|
||||
"social.fedi.farm",
|
||||
"social.fff-du.de",
|
||||
@ -389,7 +340,6 @@ var SeedInstances = [...]string{
|
||||
"social.heldscal.la",
|
||||
"social.heroicwisdom.com",
|
||||
"social.hidamari.blue",
|
||||
"social.hodakov.me",
|
||||
"social.homunyan.com",
|
||||
"social.i2p.rocks",
|
||||
"social.imirhil.fr",
|
||||
@ -398,12 +348,10 @@ var SeedInstances = [...]string{
|
||||
"social.lansky.name",
|
||||
"social.librem.one",
|
||||
"social.longden.me",
|
||||
"social.louisoft01.moe",
|
||||
"social.lucci.xyz",
|
||||
"social.luschmar.ch",
|
||||
"social.lyte.dev",
|
||||
"social.mark.atwood.name",
|
||||
"social.mhtube.de",
|
||||
"social.minkenstein.de",
|
||||
"social.mjb.im",
|
||||
"social.mochi.academy",
|
||||
@ -412,43 +360,34 @@ var SeedInstances = [...]string{
|
||||
"social.net.ua",
|
||||
"social.netdc.ca",
|
||||
"social.niicow974.fr",
|
||||
"social.nobodyhasthe.biz",
|
||||
"social.nofftopia.com",
|
||||
"social.noscraft.cf",
|
||||
"social.offline.network",
|
||||
"social.omniatv.com",
|
||||
"social.panthermodern.net",
|
||||
"social.privacytools.io",
|
||||
"social.proyectolanuevatierra.com",
|
||||
"social.puri.sm",
|
||||
"social.putz.space",
|
||||
"social.quodverum.com",
|
||||
"social.radio.af",
|
||||
"social.raptorengineering.io",
|
||||
"social.rosnovsky.us",
|
||||
"social.ryankes.eu",
|
||||
"social.seattle.wa.us",
|
||||
"social.secline.de",
|
||||
"social.skankhunt42.pw",
|
||||
"social.sunshinegardens.org",
|
||||
"social.super-niche.club",
|
||||
"social.taker.fr",
|
||||
"social.targaryen.house",
|
||||
"social.tchncs.de",
|
||||
"social.thisisjoes.site",
|
||||
"social.tomica.me",
|
||||
"social.troll.academy",
|
||||
"social.wiuwiu.de",
|
||||
"social.zwei.net",
|
||||
"sociala.me",
|
||||
"socialnetwork.ninja",
|
||||
"socl.win",
|
||||
"socnet.supes.com",
|
||||
"soderstrom.social",
|
||||
"soteria.mastodon.host",
|
||||
"souk.getloci.com",
|
||||
"southflorida.social",
|
||||
"spacetime.social",
|
||||
"speakfree.world",
|
||||
"spinster.dev",
|
||||
"spinster.xyz",
|
||||
@ -461,7 +400,6 @@ var SeedInstances = [...]string{
|
||||
"sweet.sugarcube.pw",
|
||||
"swingset.social",
|
||||
"switter.at",
|
||||
"switter.co",
|
||||
"syrup.zone",
|
||||
"take.iteasy.club",
|
||||
"takeoverthe.world",
|
||||
@ -473,7 +411,6 @@ var SeedInstances = [...]string{
|
||||
"techflake.ch",
|
||||
"the.hedgehoghunter.club",
|
||||
"the.scream.zone",
|
||||
"thechad.zone",
|
||||
"thefreestate.xyz",
|
||||
"thelballwiki.gq",
|
||||
"thetower.xyz",
|
||||
@ -485,7 +422,6 @@ var SeedInstances = [...]string{
|
||||
"toot.devfs.xyz",
|
||||
"toot.flairy.de",
|
||||
"toot.forumanalogue.fr",
|
||||
"toot.kiez.xyz",
|
||||
"toot.love",
|
||||
"toot.my",
|
||||
"toot.nx-pod.de",
|
||||
@ -503,7 +439,6 @@ var SeedInstances = [...]string{
|
||||
"travel-friends.chat",
|
||||
"tri.cash",
|
||||
"triangletoot.party",
|
||||
"triggerhub.ru",
|
||||
"tron.buzz",
|
||||
"twimblr.xyz",
|
||||
"twitter.1d4.us",
|
||||
@ -512,15 +447,12 @@ var SeedInstances = [...]string{
|
||||
"unsafe.space",
|
||||
"unsocial.pztrn.name",
|
||||
"va11hal.la",
|
||||
"vampire.estate",
|
||||
"veenus.art",
|
||||
"veenus.art",
|
||||
"voice.masto.host",
|
||||
"voluntaryism.club",
|
||||
"vulpine.club",
|
||||
"wagesofsinisdeath.com",
|
||||
"waifu.social",
|
||||
"waifuappreciation.club",
|
||||
"warc.space",
|
||||
"weeaboo.space",
|
||||
"weedis.life",
|
||||
@ -551,3 +483,75 @@ var SeedInstances = [...]string{
|
||||
"zion-techs.com",
|
||||
"zomglol.wtf",
|
||||
}
|
||||
|
||||
// removed on 2020-03-27 for nxdomain:
|
||||
var DeadSeedInstances = [...]string{
|
||||
"kneegrows.top",
|
||||
"kowai.youkai.town",
|
||||
"lets.saynoto.lgbt",
|
||||
"libertarianism.club",
|
||||
"loli.estate",
|
||||
"luvdon.ddns.net",
|
||||
"manx.social",
|
||||
"mastodon.circlelinego.com",
|
||||
"mastodon.codeplumbers.eu",
|
||||
"mastodon.fail",
|
||||
"mastodon.inhji.de",
|
||||
"mastodon.jeder.pl",
|
||||
"mastodon.loliandstuff.moe",
|
||||
"mastodon.nobodysstuff.de",
|
||||
"mastodon.openpsychology.net",
|
||||
"mastodon.redflag.social",
|
||||
"mastodon.syntik.fr",
|
||||
"mastodon.toniozz75.fr",
|
||||
"mastodon.yolovision-inc.com",
|
||||
"mastodon.zwei.net",
|
||||
"masttest.zwei.net",
|
||||
"mst.mpdevel.com",
|
||||
"mstdn.alternanet.fr",
|
||||
"mstdn.openalgeria.org",
|
||||
"multicast.social",
|
||||
"not.phrack.fyi",
|
||||
"nyaa.social",
|
||||
"opensim.fun",
|
||||
"pl.kys.moe",
|
||||
"pleroma.teromene.fr",
|
||||
"pleroma.yorha.club",
|
||||
"pltest.feminism.lgbt",
|
||||
"porntoot.com",
|
||||
"post.mashek.net",
|
||||
"preteengirls.biz",
|
||||
"project.social",
|
||||
"r3bl.social",
|
||||
"raki.social",
|
||||
"rapefeminists.network",
|
||||
"relay.selfhosting.rocks",
|
||||
"s.b252.gq",
|
||||
"simstodon.com",
|
||||
"snel.social",
|
||||
"soc.psychedelic.cat",
|
||||
"social.allthefallen.ninja",
|
||||
"social.bam.yt",
|
||||
"social.culturewar.us",
|
||||
"social.digimortal.org",
|
||||
"social.enyutech.io",
|
||||
"social.hodakov.me",
|
||||
"social.louisoft01.moe",
|
||||
"social.mhtube.de",
|
||||
"social.nobodyhasthe.biz",
|
||||
"social.noscraft.cf",
|
||||
"social.puri.sm",
|
||||
"social.rosnovsky.us",
|
||||
"social.super-niche.club",
|
||||
"social.tomica.me",
|
||||
"social.zwei.net",
|
||||
"socialnetwork.ninja",
|
||||
"spacetime.social",
|
||||
"switter.co",
|
||||
"thechad.zone",
|
||||
"toot.kiez.xyz",
|
||||
"triggerhub.ru",
|
||||
"vampire.estate",
|
||||
"wagesofsinisdeath.com",
|
||||
"waifuappreciation.club",
|
||||
}
|
||||
|
||||
@ -1,21 +1,15 @@
|
||||
package storage
|
||||
|
||||
import "errors"
|
||||
import "io/ioutil"
|
||||
import "path/filepath"
|
||||
import "os"
|
||||
import "strings"
|
||||
import "sync"
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
import "git.eeqj.de/sneak/feta/toot"
|
||||
|
||||
// TootStorageBackend is the interface to which storage backends must
|
||||
// conform for storing toots
|
||||
type TootStorageBackend interface {
|
||||
TootExists(t toot.Toot) bool
|
||||
StoreToot(t toot.Toot) error
|
||||
StoreToots(tc []*toot.Toot) error
|
||||
}
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
)
|
||||
|
||||
// TootFSStorage is a TootStorageBackend that writes to the local
|
||||
// filesystem.
|
||||
@ -35,7 +29,7 @@ func NewTootFSStorage(root string) *TootFSStorage {
|
||||
func (ts *TootFSStorage) StoreToots(tc []*toot.Toot) error {
|
||||
var returnErrors []string
|
||||
for _, item := range tc {
|
||||
err := ts.StoreToot(*item)
|
||||
err := ts.StoreToot(item)
|
||||
if err != nil {
|
||||
returnErrors = append(returnErrors, err.Error())
|
||||
continue
|
||||
@ -50,7 +44,7 @@ func (ts *TootFSStorage) StoreToots(tc []*toot.Toot) error {
|
||||
// TootExists checks to see if we have already written a toot to disk or
|
||||
// not. Note that the ingester de-dupes with a table in memory so that this
|
||||
// will only really get used on app restarts
|
||||
func (ts *TootFSStorage) TootExists(t toot.Toot) bool {
|
||||
func (ts *TootFSStorage) TootExists(t *toot.Toot) bool {
|
||||
path := t.DiskStoragePath()
|
||||
full := ts.root + "/" + path
|
||||
_, err := os.Stat(full)
|
||||
@ -61,7 +55,7 @@ func (ts *TootFSStorage) TootExists(t toot.Toot) bool {
|
||||
}
|
||||
|
||||
// StoreToot writes a single toot to disk
|
||||
func (ts *TootFSStorage) StoreToot(t toot.Toot) error {
|
||||
func (ts *TootFSStorage) StoreToot(t *toot.Toot) error {
|
||||
path := t.DiskStoragePath()
|
||||
full := ts.root + "/" + path
|
||||
dir := filepath.Dir(full)
|
||||
@ -76,7 +70,7 @@ func (ts *TootFSStorage) StoreToot(t toot.Toot) error {
|
||||
// toots in ram forever until the computer fills up and catches fire and explodes
|
||||
type TootMemoryStorage struct {
|
||||
sync.Mutex
|
||||
toots map[toot.Hash]toot.Toot
|
||||
toots map[string]*toot.Toot
|
||||
//maxSize uint // FIXME support eviction
|
||||
}
|
||||
|
||||
@ -84,12 +78,12 @@ type TootMemoryStorage struct {
|
||||
// ram forever
|
||||
func NewTootMemoryStorage() *TootMemoryStorage {
|
||||
ts := new(TootMemoryStorage)
|
||||
ts.toots = make(map[toot.Hash]toot.Toot)
|
||||
ts.toots = make(map[string]*toot.Toot)
|
||||
return ts
|
||||
}
|
||||
|
||||
// StoreToot saves a single toot into an in-memory hashtable
|
||||
func (ts *TootMemoryStorage) StoreToot(t toot.Toot) {
|
||||
func (ts *TootMemoryStorage) StoreToot(t *toot.Toot) {
|
||||
if ts.TootExists(t) {
|
||||
return
|
||||
}
|
||||
@ -100,7 +94,7 @@ func (ts *TootMemoryStorage) StoreToot(t toot.Toot) {
|
||||
}
|
||||
|
||||
// TootExists checks to see if we have a toot in memory already
|
||||
func (ts *TootMemoryStorage) TootExists(t toot.Toot) bool {
|
||||
func (ts *TootMemoryStorage) TootExists(t *toot.Toot) bool {
|
||||
ts.Lock()
|
||||
defer ts.Unlock()
|
||||
if _, ok := ts.toots[t.Hash]; ok { //this syntax is so gross
|
||||
13
storage/interface.go
Normal file
13
storage/interface.go
Normal file
@ -0,0 +1,13 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"git.eeqj.de/sneak/feta/toot"
|
||||
)
|
||||
|
||||
// TootStorageBackend is the interface to which storage backends must
|
||||
// conform for storing toots
|
||||
type TootStorageBackend interface {
|
||||
TootExists(t *toot.Toot) bool
|
||||
StoreToot(t *toot.Toot) error
|
||||
StoreToots(tc []*toot.Toot) error
|
||||
}
|
||||
32
toot/toot.go
32
toot/toot.go
@ -1,27 +1,30 @@
|
||||
package toot
|
||||
|
||||
import "fmt"
|
||||
import "encoding/json"
|
||||
import "errors"
|
||||
import "strings"
|
||||
import "git.eeqj.de/sneak/feta/jsonapis"
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/feta/jsonapis"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
//import "github.com/davecgh/go-spew/spew"
|
||||
import "github.com/rs/zerolog/log"
|
||||
|
||||
//import "encoding/hex"
|
||||
import mh "github.com/multiformats/go-multihash"
|
||||
import mhopts "github.com/multiformats/go-multihash/opts"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
|
||||
mhopts "github.com/multiformats/go-multihash/opts"
|
||||
)
|
||||
|
||||
// Hash is a type for storing a string-based base58 multihash of a
|
||||
// toot's identity
|
||||
type Hash string
|
||||
|
||||
// Toot is an object we use internally for storing a discovered toot
|
||||
type Toot struct {
|
||||
Original []byte
|
||||
Parsed *jsonapis.APISerializedToot
|
||||
Hash Hash
|
||||
Hash string
|
||||
FromHost string
|
||||
}
|
||||
|
||||
@ -111,7 +114,14 @@ func (t *Toot) identityHashInput() string {
|
||||
)
|
||||
}
|
||||
|
||||
func (t *Toot) GetHash() string {
|
||||
if t.Hash == "" {
|
||||
t.calcHash()
|
||||
}
|
||||
return t.Hash
|
||||
}
|
||||
|
||||
func (t *Toot) calcHash() {
|
||||
hi := t.identityHashInput()
|
||||
t.Hash = Hash(t.multiHash([]byte(hi)))
|
||||
t.Hash = string(t.multiHash([]byte(hi)))
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user