From 23c1b08798005b6f5683a2caf2c7f526c8312d98 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 27 Mar 2020 16:02:36 -0700 Subject: [PATCH] working toward storing state in db --- Makefile | 6 +- config.go | 19 ---- database/dbmodel.go | 37 +++++++ database/manager.go | 62 ++++++++++++ db/dbmodel.go | 15 --- go.mod | 18 ++++ go.sum | 233 +++++++++++++++++++++++++++++++++++++++++++ instance/instance.go | 32 +++--- locator/locator.go | 27 ++--- manager/manager.go | 49 +++------ process/feta.go | 72 ++++++++++--- process/handlers.go | 16 +-- seeds/seeds.go | 140 +++++++++++++------------- storage/tootstore.go | 20 ++-- 14 files changed, 553 insertions(+), 193 deletions(-) delete mode 100644 config.go create mode 100644 database/dbmodel.go create mode 100644 database/manager.go delete mode 100644 db/dbmodel.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/Makefile b/Makefile index d778d9b..0e89794 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,10 @@ ifneq ($(UNAME_S),Darwin) GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)" endif -default: run +default: build -rundebug: build - GOTRACEBACK=all DEBUG=1 ./$(FN) +debug: build + GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) run: build ./$(FN) diff --git a/config.go b/config.go deleted file mode 100644 index 746b5a7..0000000 --- a/config.go +++ /dev/null @@ -1,19 +0,0 @@ -package feta - -import "time" - -// FIXME this should use viper or something - -// Config stores the configuration for the feta process -type Config struct { - LogReportInterval time.Duration - FSStorageLocation string -} - -// GetConfig returns the config -func GetConfig() *Config { - c := new(Config) - c.LogReportInterval = time.Second * 10 - c.FSStorageLocation = "/home/sneak/Library/ApplicationSupport/feta/tootarchive" - return c -} diff --git a/database/dbmodel.go b/database/dbmodel.go new file mode 100644 index 0000000..cc818b4 --- /dev/null +++ b/database/dbmodel.go @@ -0,0 +1,37 @@ +package database + +import ( + "time" + + "github.com/google/uuid" + "github.com/jinzhu/gorm" + + _ "github.com/jinzhu/gorm/dialects/sqlite" +) + +// NB that when you add a model below you must add it to this list! + +func (m *Manager) doMigrations() { + m.db.AutoMigrate(&instance{}) +} + +type instance struct { + gorm.Model + Identifier uuid.UUID + ErrorCount uint + SuccessCount uint + HighestID int + Hostname string + Identified bool + Fetching bool + Disabled bool + Implementation string + NextFetch time.Time + NodeInfoURL string + ServerVersionString string + ServerImplementationString string +} + +func (m *Manager) ListInstances() { + // FIXME have this produce a list of Instance +} diff --git a/database/manager.go b/database/manager.go new file mode 100644 index 0000000..b3149f3 --- /dev/null +++ b/database/manager.go @@ -0,0 +1,62 @@ +package database + +import ( + "errors" + "os" + "path/filepath" + + "github.com/jinzhu/gorm" + + _ "github.com/jinzhu/gorm/dialects/sqlite" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) + +type Manager struct { + db *gorm.DB +} + +func New() *Manager { + m := new(Manager) + m.init() + return m +} + +func (m *Manager) init() { + m.open() +} + +func mkdirp(p string) error { + src, err := os.Stat(p) + if os.IsNotExist(err) { + errDir := os.MkdirAll(p, 0755) + if errDir != nil { + return err + } + return nil + } + if src.Mode().IsRegular() { + return errors.New("file already exists at path") + } + return nil +} + +func (m *Manager) open() { + log.Info().Msg("opening database") + dirname := filepath.Dir(viper.GetString("DbStorageLocation")) + err := mkdirp(dirname) + if err != nil { + log.Panic(). + Err(err). + Msg("db path erro") + } + + db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation")) + if err != nil { + log.Panic(). + Err(err). + Msg("failed to open database") + } + m.db = db + m.doMigrations() +} diff --git a/db/dbmodel.go b/db/dbmodel.go deleted file mode 100644 index 65e1a63..0000000 --- a/db/dbmodel.go +++ /dev/null @@ -1,15 +0,0 @@ -package db - -import "git.eeqj.de/sneak/feta/process" -import "github.com/jinzhu/gorm" -import _ "github.com/jinzhu/gorm/dialects/sqlite" // required for orm - -type savedInstance struct { - gorm.Model - hostname string - software string -} - -func (f *process.Feta) databaseMigrations() { - f.db.AutoMigrate(&savedInstance{}) -} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..641faa0 --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module git.eeqj.de/sneak/feta + +go 1.14 + +require ( + github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1 + github.com/gin-gonic/gin v1.6.2 + github.com/google/uuid v1.1.1 + github.com/jinzhu/gorm v1.9.12 + github.com/k0kubun/pp v3.0.1+incompatible + github.com/looplab/fsm v0.1.0 + github.com/mattn/go-colorable v0.1.6 // indirect + github.com/mattn/go-isatty v0.0.12 + github.com/multiformats/go-multihash v0.0.13 + github.com/rs/zerolog v1.18.0 + github.com/spf13/viper v1.6.2 + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b6d4abf --- /dev/null +++ b/go.sum @@ -0,0 +1,233 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1 h1:qwfOp+dwJnhdRFWsXkRMb+EZz0BgMQ8VD77OgBjuRUQ= +github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1/go.mod h1:AAlcXL9Ejp3TUsJRWJtjbIpK3p1L9z987raCTYL17j4= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM= +github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q= +github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/pp v1.3.0 h1:r9td75hcmetrcVbmsZRjnxcIbI9mhm+/N6iWyG4TWe0= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20= +github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= +github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/instance/instance.go b/instance/instance.go index a910c2f..aee444c 100644 --- a/instance/instance.go +++ b/instance/instance.go @@ -1,20 +1,24 @@ package instance -import "encoding/json" -import "fmt" -import "io/ioutil" -import "net/http" -import "strings" -import "sync" -import "time" -import "errors" +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + "sync" + "time" + + "git.eeqj.de/sneak/feta/jsonapis" + "git.eeqj.de/sneak/feta/storage" + "git.eeqj.de/sneak/feta/toot" + "github.com/google/uuid" + "github.com/looplab/fsm" + "github.com/rs/zerolog/log" +) //import "github.com/gin-gonic/gin" -import "github.com/looplab/fsm" -import "github.com/rs/zerolog/log" -import "git.eeqj.de/sneak/feta/storage" -import "git.eeqj.de/sneak/feta/toot" -import "git.eeqj.de/sneak/feta/jsonapis" const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" const instanceNodeinfoTimeout = time.Second * 50 @@ -36,6 +40,7 @@ const ( // Instance stores all the information we know about an instance type Instance struct { + Identifier uuid.UUID structLock sync.Mutex tootDestination chan *toot.Toot ErrorCount uint @@ -44,6 +49,7 @@ type Instance struct { Hostname string Identified bool fetching bool + disabled bool implementation instanceImplementation storageBackend *storage.TootStorageBackend NextFetch time.Time diff --git a/locator/locator.go b/locator/locator.go index 25a232e..c00bfa2 100644 --- a/locator/locator.go +++ b/locator/locator.go @@ -1,16 +1,20 @@ package locator -import "encoding/json" -import "io/ioutil" -import "net/http" -import "time" -import "sync" +import ( + "encoding/json" + "io/ioutil" + "net/http" + "sync" + "time" -import "github.com/rs/zerolog/log" -import "golang.org/x/sync/semaphore" -import "git.eeqj.de/sneak/feta/jsonapis" -import "git.eeqj.de/sneak/feta/instance" -import "git.eeqj.de/sneak/feta" + "git.eeqj.de/sneak/feta/instance" + "git.eeqj.de/sneak/feta/jsonapis" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" + "golang.org/x/sync/semaphore" +) + +//import "git.eeqj.de/sneak/feta" // IndexAPITimeout is the timeout for fetching json instance lists // from the listing servers @@ -120,8 +124,7 @@ func (il *InstanceLocator) Locate() { time.Sleep(1 * time.Second) - c := feta.GetConfig() - if time.Now().After(x.Add(c.LogReportInterval)) { + if time.Now().After(x.Add(viper.GetDuration("LogReportInterval"))) { x = time.Now() log.Debug(). Str("nextMastodonIndexRefresh", il.durationUntilNextMastodonIndexRefresh().String()). diff --git a/manager/manager.go b/manager/manager.go index b489bc4..c379111 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -1,16 +1,17 @@ package manager -import "sync" -import "time" -import "runtime" +import ( + "sync" + "time" + + "git.eeqj.de/sneak/feta/instance" + "git.eeqj.de/sneak/feta/seeds" + "git.eeqj.de/sneak/feta/toot" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) //import "github.com/gin-gonic/gin" -import "github.com/rs/zerolog/log" -import "git.eeqj.de/sneak/feta/toot" -import "git.eeqj.de/sneak/feta/seeds" -import "git.eeqj.de/sneak/feta/instance" - -const hostDiscoveryParallelism = 5 // LogReportInterval defines how long between logging internal // stats/reporting for user supervision @@ -24,13 +25,15 @@ type InstanceManager struct { newInstanceNotifications chan instance.Hostname tootDestination chan *toot.Toot startup time.Time + hostDiscoveryParallelism int hostAdderSemaphore chan bool } // New returns a new InstanceManager for use by the Process func New() *InstanceManager { i := new(InstanceManager) - i.hostAdderSemaphore = make(chan bool, hostDiscoveryParallelism) + i.hostDiscoveryParallelism = viper.GetInt("HostDiscoveryParallelism") + i.hostAdderSemaphore = make(chan bool, i.hostDiscoveryParallelism) i.instances = make(map[instance.Hostname]*instance.Instance) return i } @@ -41,30 +44,6 @@ func (im *InstanceManager) SetTootDestination(td chan *toot.Toot) { im.tootDestination = td } -func (im *InstanceManager) logCaller(msg string) { - fpcs := make([]uintptr, 1) - // Skip 2 levels to get the caller - n := runtime.Callers(3, fpcs) - if n == 0 { - log.Debug().Msg("MSG: NO CALLER") - } - - caller := runtime.FuncForPC(fpcs[0] - 1) - if caller == nil { - log.Debug().Msg("MSG CALLER WAS NIL") - } - - // Print the file name and line number - filename, line := caller.FileLine(fpcs[0] - 1) - function := caller.Name() - - log.Debug(). - Str("filename", filename). - Int("linenum", line). - Str("function", function). - Msg(msg) -} - func (im *InstanceManager) lock() { im.mu.Lock() } @@ -198,6 +177,7 @@ func (im *InstanceManager) logInstanceReport() { // ListInstances dumps a slice of all Instances the InstanceManager knows // about func (im *InstanceManager) ListInstances() []*instance.Instance { + // FIXME make this pull from db var out []*instance.Instance im.lock() defer im.unlock() @@ -208,6 +188,7 @@ func (im *InstanceManager) ListInstances() []*instance.Instance { } func (im *InstanceManager) instanceSummaryReport() map[string]uint { + // FIXME make this pull from db r := make(map[string]uint) for _, v := range im.ListInstances() { v.Lock() diff --git a/process/feta.go b/process/feta.go index 1e78aab..9cfc607 100644 --- a/process/feta.go +++ b/process/feta.go @@ -1,27 +1,33 @@ package process -import "os" -import "time" +import ( + "os" + "time" -import "github.com/jinzhu/gorm" -import _ "github.com/jinzhu/gorm/dialects/sqlite" // required for orm + "git.eeqj.de/sneak/feta/database" + "git.eeqj.de/sneak/feta/ingester" + "git.eeqj.de/sneak/feta/instance" + "git.eeqj.de/sneak/feta/locator" + "git.eeqj.de/sneak/feta/manager" + "git.eeqj.de/sneak/feta/storage" + _ "github.com/jinzhu/gorm/dialects/sqlite" + "github.com/k0kubun/pp" + "github.com/mattn/go-isatty" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) -import "github.com/rs/zerolog" -import "github.com/rs/zerolog/log" -import "github.com/mattn/go-isatty" - -import "git.eeqj.de/sneak/feta/ingester" -import "git.eeqj.de/sneak/feta/storage" -import "git.eeqj.de/sneak/feta/locator" -import "git.eeqj.de/sneak/feta/manager" -import "git.eeqj.de/sneak/feta/instance" +// required for orm // CLIEntry is the main entrypoint for the feta process from the cli func CLIEntry(version string, buildarch string) int { f := new(Feta) f.version = version f.buildarch = buildarch + f.configure() f.setupLogging() + f.setupDatabase() return f.runForever() } @@ -33,10 +39,41 @@ type Feta struct { manager *manager.InstanceManager ingester *ingester.TootIngester api *Server - db *gorm.DB + dbm *database.Manager startup time.Time } +func (f *Feta) configure() { + viper.SetConfigName("feta") + viper.SetConfigType("yaml") + viper.AddConfigPath("/etc/feta") // path to look for the config file in + viper.AddConfigPath("$HOME/.config/feta") // call multiple times to add many search paths + + viper.SetEnvPrefix("FETA") + viper.AutomaticEnv() + + viper.SetDefault("Debug", false) + viper.SetDefault("HostDiscoveryParallelism", 5) + viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d")) + viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db")) + viper.SetDefault("LogReportInterval", time.Second*10) + + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + // Config file not found; ignore error if desired + } else { + // Config file was found but another error was produced + log.Panic(). + Err(err). + Msg("cannot read config file") + } + } + + if viper.GetBool("debug") { + pp.Print(viper.AllSettings()) + } +} + func (f *Feta) identify() { log.Info(). Str("version", f.version). @@ -44,6 +81,10 @@ func (f *Feta) identify() { Msg("starting") } +func (f *Feta) setupDatabase() { + f.dbm = database.New() +} + func (f *Feta) setupLogging() { log.Logger = log.With().Caller().Logger() @@ -66,7 +107,7 @@ func (f *Feta) setupLogging() { } zerolog.SetGlobalLevel(zerolog.InfoLevel) - if os.Getenv("DEBUG") != "" { + if viper.GetBool("debug") { zerolog.SetGlobalLevel(zerolog.DebugLevel) } @@ -82,6 +123,7 @@ func (f *Feta) setupDatabase() { var err error f.db, err = gorm.Open("sqlite3", "feta.sqlite") + if err != nil { panic(err) } diff --git a/process/handlers.go b/process/handlers.go index a1ac43d..fb616e9 100644 --- a/process/handlers.go +++ b/process/handlers.go @@ -1,13 +1,15 @@ package process -import "time" -import "net/http" -import "encoding/json" -import "runtime" -import "fmt" -import "strings" +import ( + "encoding/json" + "fmt" + "net/http" + "runtime" + "strings" + "time" -import "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin" +) type hash map[string]interface{} diff --git a/seeds/seeds.go b/seeds/seeds.go index dbee079..8c8159f 100644 --- a/seeds/seeds.go +++ b/seeds/seeds.go @@ -19,9 +19,7 @@ var SeedInstances = [...]string{ "kiwifarms.cc", "kiwifarms.is", "kiwifarms.net", - "kneegrows.top", "knzk.me", - "kowai.youkai.town", "koyu.space", "krauser.org", "kuko.hamburg", @@ -34,12 +32,10 @@ var SeedInstances = [...]string{ "layer8.space", "leftlibertarian.club", "lesbian.energy", - "lets.saynoto.lgbt", "letsalllovela.in", "lgbtq.cool", "lgbtqia.is", "liberdon.com", - "libertarianism.club", "librem.one", "librenet.co.za", "ligma.pro", @@ -51,11 +47,9 @@ var SeedInstances = [...]string{ "loci.onl", "logjam.city", "lol5.tun.a4.io", - "loli.estate", "lolis.world", "lost-angles.im", "luvdon.cc", - "luvdon.ddns.net", "m.1994.io", "m.apertron.com", "m.bnolet.me", @@ -69,7 +63,6 @@ var SeedInstances = [...]string{ "majak.de", "makito.me", "maly.io", - "manx.social", "marchgenso.me", "mares.cafe", "mas.korrigan.tech", @@ -92,8 +85,6 @@ var SeedInstances = [...]string{ "mastodon.aventer.biz", "mastodon.blue", "mastodon.cipherbliss.com", - "mastodon.circlelinego.com", - "mastodon.codeplumbers.eu", "mastodon.coder.town", "mastodon.com.pl", "mastodon.corecoding.dev", @@ -102,7 +93,6 @@ var SeedInstances = [...]string{ "mastodon.echoz.io", "mastodon.eric.ovh", "mastodon.ericbeckers.nl", - "mastodon.fail", "mastodon.freifunk-minden.de", "mastodon.fricloud.dk", "mastodon.funigtor.fr", @@ -118,15 +108,12 @@ var SeedInstances = [...]string{ "mastodon.ie", "mastodon.immae.eu", "mastodon.inferiorlattice.com", - "mastodon.inhji.de", "mastodon.jectrum.de", - "mastodon.jeder.pl", "mastodon.kerenon.com", "mastodon.kliu.io", "mastodon.kosebamse.com", "mastodon.leptonics.com", "mastodon.local.lubar.me", - "mastodon.loliandstuff.moe", "mastodon.lubar.me", "mastodon.lunorian.is", "mastodon.macsnet.cz", @@ -134,14 +121,11 @@ var SeedInstances = [...]string{ "mastodon.me.uk", "mastodon.mynameisivan.ru", "mastodon.naoy.fr", - "mastodon.nobodysstuff.de", "mastodon.ocf.berkeley.edu", - "mastodon.openpsychology.net", "mastodon.org.ua", "mastodon.org.uk", "mastodon.otherreality.net", "mastodon.owls.io", - "mastodon.redflag.social", "mastodon.roocita.com", "mastodon.rylees.net", "mastodon.scarletsisters.xyz", @@ -154,18 +138,13 @@ var SeedInstances = [...]string{ "mastodon.soses.ca", "mastodon.spiderden.net", "mastodon.starrevolution.org", - "mastodon.syntik.fr", "mastodon.technology", "mastodon.technology", "mastodon.toni.im", - "mastodon.toniozz75.fr", "mastodon.truf-kin.com", "mastodon.xhrpb.com", - "mastodon.yolovision-inc.com", "mastodon.zapashcanon.fr", - "mastodon.zwei.net", "mastofant.de", - "masttest.zwei.net", "mcphail.uk", "me.frankmeeuwsen.xyz", "megadon.net", @@ -181,24 +160,20 @@ var SeedInstances = [...]string{ "mobile.co", "monsterpit.net", "moytura.org", - "mst.mpdevel.com", "mst.thewebzone.net", "mst.vsta.org", - "mstdn.alternanet.fr", "mstdn.ikebuku.ro", "mstdn.io", "mstdn.jp", "mstdn.maud.io", "mstdn.mx", "mstdn.novium.pw", - "mstdn.openalgeria.org", "mstdn.social", "mstdn.tsukiyono.0am.jp", "mstdn.waifu.space", "mstdn.xxil.cc", "mu.zaitcev.nu", "mudl.us", - "multicast.social", "music.pawoo.net", "myflog.net", "mypolis.zapto.org", @@ -222,16 +197,13 @@ var SeedInstances = [...]string{ "norden.social", "nordenmedia.com", "not-develop.gab.com", - "not.phrack.fyi", "npf.mlpol.net", "nsfw.social", "nudie.social", - "nyaa.social", "octodon.social", "odin.run", "ohhi.icu", "oneway.masto.host", - "opensim.fun", "order.life", "oslo.town", "our.wtf", @@ -254,7 +226,6 @@ var SeedInstances = [...]string{ "pl.knotteye.cc", "pl.kotobank.ch", "pl.koyu.space", - "pl.kys.moe", "pl.ohno.host", "pl.smuglo.li", "pl.wowana.me", @@ -273,27 +244,20 @@ var SeedInstances = [...]string{ "pleroma.quaylessed.icu", "pleroma.rareome.ga", "pleroma.soykaf.com", - "pleroma.teromene.fr", "pleroma.travnewmatic.com", "pleroma.tuxcrafting.cf", - "pleroma.yorha.club", - "pltest.feminism.lgbt", "plural.cafe", "pokemon.men", "polycule.club", "pornfed.social", - "porntoot.com", - "post.mashek.net", "pouet.jablon.fr", "ppl.town", - "preteengirls.biz", "pridelands.io", "princess.cat", "privacytools.io", "producers.masto.host", "programmer.technology", "programmingsocks.com", - "project.social", "protohype.net", "prsm.space", "psyopshop.com", @@ -305,16 +269,12 @@ var SeedInstances = [...]string{ "queersin.space", "quey.org", "quitter.pw", - "r3bl.social", "rainbowdash.net", - "raki.social", - "rapefeminists.network", "rebels.rest", "redliberal.com", "redroo.ml", "redterrorcollective.net", "relay-mypolis.zapto.org", - "relay.selfhosting.rocks", "remotenode.host", "rhubarb.land", "rigcz.club", @@ -327,7 +287,6 @@ var SeedInstances = [...]string{ "rrfarmbot.appspot.com", "rubber.social", "rva.party", - "s.b252.gq", "s.huggingservers.uk", "sackheads.social", "sadposting.space", @@ -344,7 +303,6 @@ var SeedInstances = [...]string{ "shitasstits.life", "shitposter.club", "shpposter.club", - "simstodon.com", "simulacron.de", "sinblr.com", "skippers-bin.com", @@ -354,28 +312,21 @@ var SeedInstances = [...]string{ "sn.angry.im", "snabelen.no", "snaggletooth.life", - "snel.social", "snuskete.net", - "soc.psychedelic.cat", "social.1in9.net", "social.adlerweb.info", - "social.allthefallen.ninja", "social.au2pb.net", "social.avareborn.de", "social.azkware.net", "social.b252.gq", "social.backbord.net", - "social.bam.yt", "social.bau-ha.us", "social.beepboop.ga", "social.cereza.de", "social.cloudsumu.com", - "social.culturewar.us", "social.cutienaut.club", - "social.digimortal.org", "social.elqhost.net", "social.end-the-stigma.com", - "social.enyutech.io", "social.fab-l3.org", "social.fedi.farm", "social.fff-du.de", @@ -389,7 +340,6 @@ var SeedInstances = [...]string{ "social.heldscal.la", "social.heroicwisdom.com", "social.hidamari.blue", - "social.hodakov.me", "social.homunyan.com", "social.i2p.rocks", "social.imirhil.fr", @@ -398,12 +348,10 @@ var SeedInstances = [...]string{ "social.lansky.name", "social.librem.one", "social.longden.me", - "social.louisoft01.moe", "social.lucci.xyz", "social.luschmar.ch", "social.lyte.dev", "social.mark.atwood.name", - "social.mhtube.de", "social.minkenstein.de", "social.mjb.im", "social.mochi.academy", @@ -412,43 +360,34 @@ var SeedInstances = [...]string{ "social.net.ua", "social.netdc.ca", "social.niicow974.fr", - "social.nobodyhasthe.biz", "social.nofftopia.com", - "social.noscraft.cf", "social.offline.network", "social.omniatv.com", "social.panthermodern.net", "social.privacytools.io", "social.proyectolanuevatierra.com", - "social.puri.sm", "social.putz.space", "social.quodverum.com", "social.radio.af", "social.raptorengineering.io", - "social.rosnovsky.us", "social.ryankes.eu", "social.seattle.wa.us", "social.secline.de", "social.skankhunt42.pw", "social.sunshinegardens.org", - "social.super-niche.club", "social.taker.fr", "social.targaryen.house", "social.tchncs.de", "social.thisisjoes.site", - "social.tomica.me", "social.troll.academy", "social.wiuwiu.de", - "social.zwei.net", "sociala.me", - "socialnetwork.ninja", "socl.win", "socnet.supes.com", "soderstrom.social", "soteria.mastodon.host", "souk.getloci.com", "southflorida.social", - "spacetime.social", "speakfree.world", "spinster.dev", "spinster.xyz", @@ -461,7 +400,6 @@ var SeedInstances = [...]string{ "sweet.sugarcube.pw", "swingset.social", "switter.at", - "switter.co", "syrup.zone", "take.iteasy.club", "takeoverthe.world", @@ -473,7 +411,6 @@ var SeedInstances = [...]string{ "techflake.ch", "the.hedgehoghunter.club", "the.scream.zone", - "thechad.zone", "thefreestate.xyz", "thelballwiki.gq", "thetower.xyz", @@ -485,7 +422,6 @@ var SeedInstances = [...]string{ "toot.devfs.xyz", "toot.flairy.de", "toot.forumanalogue.fr", - "toot.kiez.xyz", "toot.love", "toot.my", "toot.nx-pod.de", @@ -503,7 +439,6 @@ var SeedInstances = [...]string{ "travel-friends.chat", "tri.cash", "triangletoot.party", - "triggerhub.ru", "tron.buzz", "twimblr.xyz", "twitter.1d4.us", @@ -512,15 +447,12 @@ var SeedInstances = [...]string{ "unsafe.space", "unsocial.pztrn.name", "va11hal.la", - "vampire.estate", "veenus.art", "veenus.art", "voice.masto.host", "voluntaryism.club", "vulpine.club", - "wagesofsinisdeath.com", "waifu.social", - "waifuappreciation.club", "warc.space", "weeaboo.space", "weedis.life", @@ -551,3 +483,75 @@ var SeedInstances = [...]string{ "zion-techs.com", "zomglol.wtf", } + +// removed on 2020-03-27 for nxdomain: +var DeadSeedInstances = [...]string{ + "kneegrows.top", + "kowai.youkai.town", + "lets.saynoto.lgbt", + "libertarianism.club", + "loli.estate", + "luvdon.ddns.net", + "manx.social", + "mastodon.circlelinego.com", + "mastodon.codeplumbers.eu", + "mastodon.fail", + "mastodon.inhji.de", + "mastodon.jeder.pl", + "mastodon.loliandstuff.moe", + "mastodon.nobodysstuff.de", + "mastodon.openpsychology.net", + "mastodon.redflag.social", + "mastodon.syntik.fr", + "mastodon.toniozz75.fr", + "mastodon.yolovision-inc.com", + "mastodon.zwei.net", + "masttest.zwei.net", + "mst.mpdevel.com", + "mstdn.alternanet.fr", + "mstdn.openalgeria.org", + "multicast.social", + "not.phrack.fyi", + "nyaa.social", + "opensim.fun", + "pl.kys.moe", + "pleroma.teromene.fr", + "pleroma.yorha.club", + "pltest.feminism.lgbt", + "porntoot.com", + "post.mashek.net", + "preteengirls.biz", + "project.social", + "r3bl.social", + "raki.social", + "rapefeminists.network", + "relay.selfhosting.rocks", + "s.b252.gq", + "simstodon.com", + "snel.social", + "soc.psychedelic.cat", + "social.allthefallen.ninja", + "social.bam.yt", + "social.culturewar.us", + "social.digimortal.org", + "social.enyutech.io", + "social.hodakov.me", + "social.louisoft01.moe", + "social.mhtube.de", + "social.nobodyhasthe.biz", + "social.noscraft.cf", + "social.puri.sm", + "social.rosnovsky.us", + "social.super-niche.club", + "social.tomica.me", + "social.zwei.net", + "socialnetwork.ninja", + "spacetime.social", + "switter.co", + "thechad.zone", + "toot.kiez.xyz", + "triggerhub.ru", + "vampire.estate", + "wagesofsinisdeath.com", + "waifuappreciation.club", +} diff --git a/storage/tootstore.go b/storage/tootstore.go index 92964f4..b488555 100644 --- a/storage/tootstore.go +++ b/storage/tootstore.go @@ -1,13 +1,15 @@ package storage -import "errors" -import "io/ioutil" -import "path/filepath" -import "os" -import "strings" -import "sync" +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" -import "git.eeqj.de/sneak/feta/toot" + "git.eeqj.de/sneak/feta/toot" +) // TootStorageBackend is the interface to which storage backends must // conform for storing toots @@ -17,6 +19,10 @@ type TootStorageBackend interface { StoreToots(tc []*toot.Toot) error } +type TootDBStorage struct { + db string +} + // TootFSStorage is a TootStorageBackend that writes to the local // filesystem. type TootFSStorage struct {