This commit is contained in:
Jeffrey Paul 2019-11-03 02:56:50 -08:00
parent 3aec504aa1
commit 052600663c
6 changed files with 85 additions and 40 deletions

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
fediverse-archive feta
output/ output/

View File

@ -1,6 +1,6 @@
FROM golang:1.13 as builder FROM golang:1.13 as builder
WORKDIR /go/src/github.com/sneak/fediverse-archive WORKDIR /go/src/github.com/sneak/feta
COPY . . COPY . .
RUN go get -v && make build RUN go get -v && make build
@ -13,11 +13,11 @@ RUN tar cvfz go-src.tgz src && du -sh *
FROM alpine FROM alpine
COPY --from=builder /go/src/github.com/sneak/fediverse-archive/fediverse-archive /bin/fediverse-archive COPY --from=builder /go/src/github.com/sneak/feta/feta /bin/feta
# put the source in there too for safekeeping # put the source in there too for safekeeping
COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
CMD /bin/fediverse-archive CMD /bin/feta
# FIXME add testing # FIXME add testing

View File

@ -6,7 +6,7 @@ BUILDUSER := $(shell whoami)
BUILDHOST := $(shell hostname -s) BUILDHOST := $(shell hostname -s)
BUILDARCH := $(shell uname -m) BUILDARCH := $(shell uname -m)
FN := fediverse-archive FN := feta
IMAGENAME := sneak/$(FN) IMAGENAME := sneak/$(FN)
UNAME_S := $(shell uname -s) UNAME_S := $(shell uname -s)

View File

@ -4,6 +4,7 @@ import "encoding/json"
import "fmt" import "fmt"
import "net/http" import "net/http"
import "strings" import "strings"
import "sync"
import "time" import "time"
import "github.com/rs/zerolog/log" import "github.com/rs/zerolog/log"
@ -11,6 +12,8 @@ import "github.com/rs/zerolog/log"
const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
const NODE_TIMEOUT = time.Second * 10 const NODE_TIMEOUT = time.Second * 10
const ONE_HOUR = time.Second * 60 * 60
const ONE_DAY = time.Second * 60 * 60 * 24
type ServerImplementation int type ServerImplementation int
@ -21,26 +24,23 @@ const (
) )
type Instance struct { type Instance struct {
sync.Mutex
errorCount uint errorCount uint
highestId int highestId int
hostName string hostName string
impl ServerImplementation impl ServerImplementation
lastFailure *time.Time lastError *time.Time
lastSuccess *time.Time lastSuccess *time.Time
nextCheck *time.Time
nodeInfoUrl string nodeInfoUrl string
serverVersion string serverVersion string
identified bool
up bool
shouldSkip bool
} }
func NewInstance(hostname string) *Instance { func NewInstance(hostname string) *Instance {
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
i := new(Instance) i := new(Instance)
i.hostName = hostname i.hostName = hostname
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second) i.nextCheck = &foreverago
i.lastSuccess = &foreverago
i.lastFailure = &foreverago
i.identified = false
i.up = false i.up = false
go func() { go func() {
i.detectNodeType() i.detectNodeType()
@ -48,10 +48,28 @@ func NewInstance(hostname string) *Instance {
return i return i
} }
func (i *Instance) setNextCheck(d *time.Duration) {
i.Lock()
defer i.Unlock()
i.nextCheck = d
}
func (i *Instance) dueForCheck() bool {
i.Lock()
defer i.Unlock()
if i.nextCheck <= time.Now() {
return true
}
return false
}
func (i *Instance) detectNodeType() { func (i *Instance) detectNodeType() {
i.Lock()
if i.impl > ServerUnknown { if i.impl > ServerUnknown {
i.Unlock()
return return
} }
i.Unlock()
i.fetchNodeInfo() i.fetchNodeInfo()
} }
@ -81,12 +99,16 @@ type NodeInfoVersionTwoSchema struct {
} }
func (i *Instance) registerError() { func (i *Instance) registerError() {
i.Lock()
defer i.Unlock()
i.errorCount = i.errorCount + 1 i.errorCount = i.errorCount + 1
t := time.Now() t := time.Now()
i.lastFailure = &t i.lastError = &t
} }
func (i *Instance) registerSuccess() { func (i *Instance) registerSuccess() {
i.Lock()
defer i.Unlock()
t := time.Now() t := time.Now()
i.lastSuccess = &t i.lastSuccess = &t
} }
@ -125,7 +147,10 @@ func (i *Instance) fetchNodeInfoURL() {
Str("hostname", i.hostName). Str("hostname", i.hostName).
Str("nodeinfourl", item.Href). Str("nodeinfourl", item.Href).
Msg("success fetching url for nodeinfo") Msg("success fetching url for nodeinfo")
i.Lock()
i.nodeInfoUrl = item.Href i.nodeInfoUrl = item.Href
i.Unlock()
i.registerSuccess() i.registerSuccess()
return return
} }
@ -140,10 +165,18 @@ func (i *Instance) fetchNodeInfoURL() {
func (i *Instance) fetchNodeInfo() { func (i *Instance) fetchNodeInfo() {
i.fetchNodeInfoURL() i.fetchNodeInfoURL()
i.Lock()
failure := false
if i.nodeInfoUrl == "" { if i.nodeInfoUrl == "" {
log.Error(). log.Error().
Str("hostname", i.hostName). Str("hostname", i.hostName).
Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined") Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined")
failure = true
}
i.Unlock()
if failure == true {
return return
} }
@ -153,7 +186,11 @@ func (i *Instance) fetchNodeInfo() {
//FIXME make sure the nodeinfourl is on the same domain as the instance //FIXME make sure the nodeinfourl is on the same domain as the instance
//hostname //hostname
resp, err := c.Get(i.nodeInfoUrl) i.Lock()
url := i.nodeInfoUrl
i.Unlock()
resp, err := c.Get(url)
if err != nil { if err != nil {
log.Error(). log.Error().
@ -178,6 +215,8 @@ func (i *Instance) fetchNodeInfo() {
Str("nodeInfoUrl", i.nodeInfoUrl). Str("nodeInfoUrl", i.nodeInfoUrl).
Msg("received nodeinfo from instance") Msg("received nodeinfo from instance")
i.Lock()
defer i.Unlock()
i.serverVersion = ni.Software.Version i.serverVersion = ni.Software.Version
ni.Software.Name = strings.ToLower(ni.Software.Name) ni.Software.Name = strings.ToLower(ni.Software.Name)
@ -210,12 +249,16 @@ func (i *Instance) fetchNodeInfo() {
} }
func (i *Instance) fetchRecentToots() ([]byte, error) { func (i *Instance) fetchRecentToots() ([]byte, error) {
if i.impl == ServerMastodon { i.Lock()
impl := i.impl
i.Unlock()
if impl == ServerMastodon {
return i.fetchRecentTootsJsonFromMastodon() return i.fetchRecentTootsJsonFromMastodon()
} else if i.impl == ServerPleroma { } else if impl == ServerPleroma {
return i.fetchRecentTootsJsonFromPleroma() return i.fetchRecentTootsJsonFromPleroma()
} else { } else {
panic("nope") panic("unimplemented")
} }
} }
@ -228,8 +271,3 @@ func (i *Instance) fetchRecentTootsJsonFromMastodon() ([]byte, error) {
//url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName) //url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName)
return nil, nil return nil, nil
} }
func fetchLatestToots(lastId int) {
log.Debug().Msg("This message appears only when log level set to Debug")
log.Info().Msg("This message appears when log level set to Debug or Info")
}

View File

@ -93,12 +93,12 @@ func NewInstanceLocator() *InstanceLocator {
} }
func (i *InstanceLocator) addInstance(hostname string) { func (i *InstanceLocator) addInstance(hostname string) {
i.Lock()
defer i.Unlock()
// only add it if we haven't seen the hostname before // only add it if we haven't seen the hostname before
if i.instances[hostname] == nil { if i.instances[hostname] == nil {
log.Debug().Str("hostname", hostname).Msgf("adding discovered instance") log.Debug().Str("hostname", hostname).Msgf("adding discovered instance")
i.Lock()
i.instances[hostname] = NewInstance(hostname) i.instances[hostname] = NewInstance(hostname)
i.Unlock()
} }
} }
@ -118,7 +118,14 @@ func (i *InstanceLocator) Locate() {
time.Sleep(120 * time.Second) time.Sleep(120 * time.Second)
i.instanceReport() r := i.instanceReport()
log.Debug().
Uint("up", r.up).
Uint("total", r.total).
Uint("identified", r.identified).
Msg("instance report")
} }
type InstanceLocatorReport struct { type InstanceLocatorReport struct {
@ -136,6 +143,8 @@ func (i *InstanceLocator) NumInstances() uint {
} }
func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
i.Lock()
defer i.Unlock()
r := new(InstanceLocatorReport) r := new(InstanceLocatorReport)
r.total = uint(len(i.instances)) r.total = uint(len(i.instances))
@ -150,12 +159,6 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
} }
} }
log.Debug().
Uint("up", r.up).
Uint("total", r.total).
Uint("identified", r.identified).
Msg("instance report")
return r return r
} }
@ -179,8 +182,8 @@ func (i *InstanceLocator) locateMastodon() {
i.addInstance(instance.Name) i.addInstance(instance.Name)
} }
i.Lock()
t := time.Now() t := time.Now()
i.Lock()
i.mastodonIndexLastRefresh = &t i.mastodonIndexLastRefresh = &t
i.Unlock() i.Unlock()
} }
@ -204,8 +207,8 @@ func (i *InstanceLocator) locatePleroma() {
for _, instance := range *pi { for _, instance := range *pi {
i.addInstance(instance.Domain) i.addInstance(instance.Domain)
} }
i.Lock()
t := time.Now() t := time.Now()
i.Lock()
i.pleromaIndexLastRefresh = &t i.pleromaIndexLastRefresh = &t
i.Unlock() i.Unlock()
} }

View File

@ -49,14 +49,18 @@ func (a *TootArchiverAPIServer) getRouter() *gin.Engine {
r.GET("/", func(c *gin.Context) { r.GET("/", func(c *gin.Context) {
c.JSON(200, gin.H{ c.JSON(200, gin.H{
// FIXME add more stuff here // FIXME(sneak) add more stuff here
"status": "ok", "status": "ok",
"now": time.Now().UTC().Format(time.RFC3339), "now": time.Now().UTC().Format(time.RFC3339),
"uptime": a.archiver.Uptime().String(), "uptime": a.archiver.Uptime().String(),
"instances": a.archiver.locator.NumInstances(), "instances": gin.H{
"total": a.archiver.locator.instanceReport().total,
"up": a.archiver.locator.instanceReport().up,
"identified": a.archiver.locator.instanceReport().identified,
},
}) })
}) })
// FIXME add more status routes here // FIXME(sneak) add more status routes here
return r return r
} }