diff --git a/.gitignore b/.gitignore index 3b5cd0c..0cee278 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -fediverse-archive +feta output/ diff --git a/Dockerfile b/Dockerfile index 043e773..e677868 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.13 as builder -WORKDIR /go/src/github.com/sneak/fediverse-archive +WORKDIR /go/src/github.com/sneak/feta COPY . . RUN go get -v && make build @@ -13,11 +13,11 @@ RUN tar cvfz go-src.tgz src && du -sh * FROM alpine -COPY --from=builder /go/src/github.com/sneak/fediverse-archive/fediverse-archive /bin/fediverse-archive +COPY --from=builder /go/src/github.com/sneak/feta/feta /bin/feta # put the source in there too for safekeeping COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz -CMD /bin/fediverse-archive +CMD /bin/feta # FIXME add testing diff --git a/Makefile b/Makefile index cc70b68..20a3ba3 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ BUILDUSER := $(shell whoami) BUILDHOST := $(shell hostname -s) BUILDARCH := $(shell uname -m) -FN := fediverse-archive +FN := feta IMAGENAME := sneak/$(FN) UNAME_S := $(shell uname -s) diff --git a/instance.go b/instance.go index 3daa7a3..5a445fe 100644 --- a/instance.go +++ b/instance.go @@ -4,6 +4,7 @@ import "encoding/json" import "fmt" import "net/http" import "strings" +import "sync" import "time" import "github.com/rs/zerolog/log" @@ -11,6 +12,8 @@ import "github.com/rs/zerolog/log" const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" const NODE_TIMEOUT = time.Second * 10 +const ONE_HOUR = time.Second * 60 * 60 +const ONE_DAY = time.Second * 60 * 60 * 24 type ServerImplementation int @@ -21,26 +24,23 @@ const ( ) type Instance struct { + sync.Mutex errorCount uint highestId int hostName string impl ServerImplementation - lastFailure *time.Time + lastError *time.Time lastSuccess *time.Time + nextCheck *time.Time nodeInfoUrl string serverVersion string - identified bool - up bool - shouldSkip bool } func NewInstance(hostname string) *Instance { + foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second) i := new(Instance) i.hostName = hostname - foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second) - i.lastSuccess = &foreverago - i.lastFailure = &foreverago - i.identified = false + i.nextCheck = &foreverago i.up = false go func() { i.detectNodeType() @@ -48,10 +48,28 @@ func NewInstance(hostname string) *Instance { return i } +func (i *Instance) setNextCheck(d *time.Duration) { + i.Lock() + defer i.Unlock() + i.nextCheck = d +} + +func (i *Instance) dueForCheck() bool { + i.Lock() + defer i.Unlock() + if i.nextCheck <= time.Now() { + return true + } + return false +} + func (i *Instance) detectNodeType() { + i.Lock() if i.impl > ServerUnknown { + i.Unlock() return } + i.Unlock() i.fetchNodeInfo() } @@ -81,12 +99,16 @@ type NodeInfoVersionTwoSchema struct { } func (i *Instance) registerError() { + i.Lock() + defer i.Unlock() i.errorCount = i.errorCount + 1 t := time.Now() - i.lastFailure = &t + i.lastError = &t } func (i *Instance) registerSuccess() { + i.Lock() + defer i.Unlock() t := time.Now() i.lastSuccess = &t } @@ -125,7 +147,10 @@ func (i *Instance) fetchNodeInfoURL() { Str("hostname", i.hostName). Str("nodeinfourl", item.Href). Msg("success fetching url for nodeinfo") + + i.Lock() i.nodeInfoUrl = item.Href + i.Unlock() i.registerSuccess() return } @@ -140,10 +165,18 @@ func (i *Instance) fetchNodeInfoURL() { func (i *Instance) fetchNodeInfo() { i.fetchNodeInfoURL() + + i.Lock() + failure := false if i.nodeInfoUrl == "" { log.Error(). Str("hostname", i.hostName). Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined") + failure = true + } + i.Unlock() + + if failure == true { return } @@ -153,7 +186,11 @@ func (i *Instance) fetchNodeInfo() { //FIXME make sure the nodeinfourl is on the same domain as the instance //hostname - resp, err := c.Get(i.nodeInfoUrl) + i.Lock() + url := i.nodeInfoUrl + i.Unlock() + + resp, err := c.Get(url) if err != nil { log.Error(). @@ -178,6 +215,8 @@ func (i *Instance) fetchNodeInfo() { Str("nodeInfoUrl", i.nodeInfoUrl). Msg("received nodeinfo from instance") + i.Lock() + defer i.Unlock() i.serverVersion = ni.Software.Version ni.Software.Name = strings.ToLower(ni.Software.Name) @@ -210,12 +249,16 @@ func (i *Instance) fetchNodeInfo() { } func (i *Instance) fetchRecentToots() ([]byte, error) { - if i.impl == ServerMastodon { + i.Lock() + impl := i.impl + i.Unlock() + + if impl == ServerMastodon { return i.fetchRecentTootsJsonFromMastodon() - } else if i.impl == ServerPleroma { + } else if impl == ServerPleroma { return i.fetchRecentTootsJsonFromPleroma() } else { - panic("nope") + panic("unimplemented") } } @@ -228,8 +271,3 @@ func (i *Instance) fetchRecentTootsJsonFromMastodon() ([]byte, error) { //url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName) return nil, nil } - -func fetchLatestToots(lastId int) { - log.Debug().Msg("This message appears only when log level set to Debug") - log.Info().Msg("This message appears when log level set to Debug or Info") -} diff --git a/locator.go b/locator.go index e5f1b3c..5668eab 100644 --- a/locator.go +++ b/locator.go @@ -93,12 +93,12 @@ func NewInstanceLocator() *InstanceLocator { } func (i *InstanceLocator) addInstance(hostname string) { + i.Lock() + defer i.Unlock() // only add it if we haven't seen the hostname before if i.instances[hostname] == nil { log.Debug().Str("hostname", hostname).Msgf("adding discovered instance") - i.Lock() i.instances[hostname] = NewInstance(hostname) - i.Unlock() } } @@ -118,7 +118,14 @@ func (i *InstanceLocator) Locate() { time.Sleep(120 * time.Second) - i.instanceReport() + r := i.instanceReport() + + log.Debug(). + Uint("up", r.up). + Uint("total", r.total). + Uint("identified", r.identified). + Msg("instance report") + } type InstanceLocatorReport struct { @@ -136,6 +143,8 @@ func (i *InstanceLocator) NumInstances() uint { } func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { + i.Lock() + defer i.Unlock() r := new(InstanceLocatorReport) r.total = uint(len(i.instances)) @@ -150,12 +159,6 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { } } - log.Debug(). - Uint("up", r.up). - Uint("total", r.total). - Uint("identified", r.identified). - Msg("instance report") - return r } @@ -179,8 +182,8 @@ func (i *InstanceLocator) locateMastodon() { i.addInstance(instance.Name) } - i.Lock() t := time.Now() + i.Lock() i.mastodonIndexLastRefresh = &t i.Unlock() } @@ -204,8 +207,8 @@ func (i *InstanceLocator) locatePleroma() { for _, instance := range *pi { i.addInstance(instance.Domain) } - i.Lock() t := time.Now() + i.Lock() i.pleromaIndexLastRefresh = &t i.Unlock() } diff --git a/webserver.go b/webserver.go index 453c631..f8e2e44 100644 --- a/webserver.go +++ b/webserver.go @@ -49,14 +49,18 @@ func (a *TootArchiverAPIServer) getRouter() *gin.Engine { r.GET("/", func(c *gin.Context) { c.JSON(200, gin.H{ - // FIXME add more stuff here - "status": "ok", - "now": time.Now().UTC().Format(time.RFC3339), - "uptime": a.archiver.Uptime().String(), - "instances": a.archiver.locator.NumInstances(), + // FIXME(sneak) add more stuff here + "status": "ok", + "now": time.Now().UTC().Format(time.RFC3339), + "uptime": a.archiver.Uptime().String(), + "instances": gin.H{ + "total": a.archiver.locator.instanceReport().total, + "up": a.archiver.locator.instanceReport().up, + "identified": a.archiver.locator.instanceReport().identified, + }, }) }) - // FIXME add more status routes here + // FIXME(sneak) add more status routes here return r }