renamed
This commit is contained in:
parent
3aec504aa1
commit
052600663c
|
@ -1,2 +1,2 @@
|
||||||
fediverse-archive
|
feta
|
||||||
output/
|
output/
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
FROM golang:1.13 as builder
|
FROM golang:1.13 as builder
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/sneak/fediverse-archive
|
WORKDIR /go/src/github.com/sneak/feta
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
RUN go get -v && make build
|
RUN go get -v && make build
|
||||||
|
@ -13,11 +13,11 @@ RUN tar cvfz go-src.tgz src && du -sh *
|
||||||
|
|
||||||
FROM alpine
|
FROM alpine
|
||||||
|
|
||||||
COPY --from=builder /go/src/github.com/sneak/fediverse-archive/fediverse-archive /bin/fediverse-archive
|
COPY --from=builder /go/src/github.com/sneak/feta/feta /bin/feta
|
||||||
|
|
||||||
# put the source in there too for safekeeping
|
# put the source in there too for safekeeping
|
||||||
COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
|
COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
|
||||||
|
|
||||||
CMD /bin/fediverse-archive
|
CMD /bin/feta
|
||||||
|
|
||||||
# FIXME add testing
|
# FIXME add testing
|
||||||
|
|
2
Makefile
2
Makefile
|
@ -6,7 +6,7 @@ BUILDUSER := $(shell whoami)
|
||||||
BUILDHOST := $(shell hostname -s)
|
BUILDHOST := $(shell hostname -s)
|
||||||
BUILDARCH := $(shell uname -m)
|
BUILDARCH := $(shell uname -m)
|
||||||
|
|
||||||
FN := fediverse-archive
|
FN := feta
|
||||||
IMAGENAME := sneak/$(FN)
|
IMAGENAME := sneak/$(FN)
|
||||||
|
|
||||||
UNAME_S := $(shell uname -s)
|
UNAME_S := $(shell uname -s)
|
||||||
|
|
74
instance.go
74
instance.go
|
@ -4,6 +4,7 @@ import "encoding/json"
|
||||||
import "fmt"
|
import "fmt"
|
||||||
import "net/http"
|
import "net/http"
|
||||||
import "strings"
|
import "strings"
|
||||||
|
import "sync"
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
import "github.com/rs/zerolog/log"
|
import "github.com/rs/zerolog/log"
|
||||||
|
@ -11,6 +12,8 @@ import "github.com/rs/zerolog/log"
|
||||||
const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
|
const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
|
||||||
|
|
||||||
const NODE_TIMEOUT = time.Second * 10
|
const NODE_TIMEOUT = time.Second * 10
|
||||||
|
const ONE_HOUR = time.Second * 60 * 60
|
||||||
|
const ONE_DAY = time.Second * 60 * 60 * 24
|
||||||
|
|
||||||
type ServerImplementation int
|
type ServerImplementation int
|
||||||
|
|
||||||
|
@ -21,26 +24,23 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
type Instance struct {
|
type Instance struct {
|
||||||
|
sync.Mutex
|
||||||
errorCount uint
|
errorCount uint
|
||||||
highestId int
|
highestId int
|
||||||
hostName string
|
hostName string
|
||||||
impl ServerImplementation
|
impl ServerImplementation
|
||||||
lastFailure *time.Time
|
lastError *time.Time
|
||||||
lastSuccess *time.Time
|
lastSuccess *time.Time
|
||||||
|
nextCheck *time.Time
|
||||||
nodeInfoUrl string
|
nodeInfoUrl string
|
||||||
serverVersion string
|
serverVersion string
|
||||||
identified bool
|
|
||||||
up bool
|
|
||||||
shouldSkip bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewInstance(hostname string) *Instance {
|
func NewInstance(hostname string) *Instance {
|
||||||
|
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
|
||||||
i := new(Instance)
|
i := new(Instance)
|
||||||
i.hostName = hostname
|
i.hostName = hostname
|
||||||
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
|
i.nextCheck = &foreverago
|
||||||
i.lastSuccess = &foreverago
|
|
||||||
i.lastFailure = &foreverago
|
|
||||||
i.identified = false
|
|
||||||
i.up = false
|
i.up = false
|
||||||
go func() {
|
go func() {
|
||||||
i.detectNodeType()
|
i.detectNodeType()
|
||||||
|
@ -48,10 +48,28 @@ func NewInstance(hostname string) *Instance {
|
||||||
return i
|
return i
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (i *Instance) setNextCheck(d *time.Duration) {
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
|
i.nextCheck = d
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Instance) dueForCheck() bool {
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
|
if i.nextCheck <= time.Now() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func (i *Instance) detectNodeType() {
|
func (i *Instance) detectNodeType() {
|
||||||
|
i.Lock()
|
||||||
if i.impl > ServerUnknown {
|
if i.impl > ServerUnknown {
|
||||||
|
i.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
i.Unlock()
|
||||||
i.fetchNodeInfo()
|
i.fetchNodeInfo()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,12 +99,16 @@ type NodeInfoVersionTwoSchema struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *Instance) registerError() {
|
func (i *Instance) registerError() {
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
i.errorCount = i.errorCount + 1
|
i.errorCount = i.errorCount + 1
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
i.lastFailure = &t
|
i.lastError = &t
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *Instance) registerSuccess() {
|
func (i *Instance) registerSuccess() {
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
i.lastSuccess = &t
|
i.lastSuccess = &t
|
||||||
}
|
}
|
||||||
|
@ -125,7 +147,10 @@ func (i *Instance) fetchNodeInfoURL() {
|
||||||
Str("hostname", i.hostName).
|
Str("hostname", i.hostName).
|
||||||
Str("nodeinfourl", item.Href).
|
Str("nodeinfourl", item.Href).
|
||||||
Msg("success fetching url for nodeinfo")
|
Msg("success fetching url for nodeinfo")
|
||||||
|
|
||||||
|
i.Lock()
|
||||||
i.nodeInfoUrl = item.Href
|
i.nodeInfoUrl = item.Href
|
||||||
|
i.Unlock()
|
||||||
i.registerSuccess()
|
i.registerSuccess()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -140,10 +165,18 @@ func (i *Instance) fetchNodeInfoURL() {
|
||||||
|
|
||||||
func (i *Instance) fetchNodeInfo() {
|
func (i *Instance) fetchNodeInfo() {
|
||||||
i.fetchNodeInfoURL()
|
i.fetchNodeInfoURL()
|
||||||
|
|
||||||
|
i.Lock()
|
||||||
|
failure := false
|
||||||
if i.nodeInfoUrl == "" {
|
if i.nodeInfoUrl == "" {
|
||||||
log.Error().
|
log.Error().
|
||||||
Str("hostname", i.hostName).
|
Str("hostname", i.hostName).
|
||||||
Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined")
|
Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined")
|
||||||
|
failure = true
|
||||||
|
}
|
||||||
|
i.Unlock()
|
||||||
|
|
||||||
|
if failure == true {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -153,7 +186,11 @@ func (i *Instance) fetchNodeInfo() {
|
||||||
|
|
||||||
//FIXME make sure the nodeinfourl is on the same domain as the instance
|
//FIXME make sure the nodeinfourl is on the same domain as the instance
|
||||||
//hostname
|
//hostname
|
||||||
resp, err := c.Get(i.nodeInfoUrl)
|
i.Lock()
|
||||||
|
url := i.nodeInfoUrl
|
||||||
|
i.Unlock()
|
||||||
|
|
||||||
|
resp, err := c.Get(url)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().
|
log.Error().
|
||||||
|
@ -178,6 +215,8 @@ func (i *Instance) fetchNodeInfo() {
|
||||||
Str("nodeInfoUrl", i.nodeInfoUrl).
|
Str("nodeInfoUrl", i.nodeInfoUrl).
|
||||||
Msg("received nodeinfo from instance")
|
Msg("received nodeinfo from instance")
|
||||||
|
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
i.serverVersion = ni.Software.Version
|
i.serverVersion = ni.Software.Version
|
||||||
|
|
||||||
ni.Software.Name = strings.ToLower(ni.Software.Name)
|
ni.Software.Name = strings.ToLower(ni.Software.Name)
|
||||||
|
@ -210,12 +249,16 @@ func (i *Instance) fetchNodeInfo() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *Instance) fetchRecentToots() ([]byte, error) {
|
func (i *Instance) fetchRecentToots() ([]byte, error) {
|
||||||
if i.impl == ServerMastodon {
|
i.Lock()
|
||||||
|
impl := i.impl
|
||||||
|
i.Unlock()
|
||||||
|
|
||||||
|
if impl == ServerMastodon {
|
||||||
return i.fetchRecentTootsJsonFromMastodon()
|
return i.fetchRecentTootsJsonFromMastodon()
|
||||||
} else if i.impl == ServerPleroma {
|
} else if impl == ServerPleroma {
|
||||||
return i.fetchRecentTootsJsonFromPleroma()
|
return i.fetchRecentTootsJsonFromPleroma()
|
||||||
} else {
|
} else {
|
||||||
panic("nope")
|
panic("unimplemented")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -228,8 +271,3 @@ func (i *Instance) fetchRecentTootsJsonFromMastodon() ([]byte, error) {
|
||||||
//url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName)
|
//url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName)
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchLatestToots(lastId int) {
|
|
||||||
log.Debug().Msg("This message appears only when log level set to Debug")
|
|
||||||
log.Info().Msg("This message appears when log level set to Debug or Info")
|
|
||||||
}
|
|
||||||
|
|
25
locator.go
25
locator.go
|
@ -93,12 +93,12 @@ func NewInstanceLocator() *InstanceLocator {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *InstanceLocator) addInstance(hostname string) {
|
func (i *InstanceLocator) addInstance(hostname string) {
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
// only add it if we haven't seen the hostname before
|
// only add it if we haven't seen the hostname before
|
||||||
if i.instances[hostname] == nil {
|
if i.instances[hostname] == nil {
|
||||||
log.Debug().Str("hostname", hostname).Msgf("adding discovered instance")
|
log.Debug().Str("hostname", hostname).Msgf("adding discovered instance")
|
||||||
i.Lock()
|
|
||||||
i.instances[hostname] = NewInstance(hostname)
|
i.instances[hostname] = NewInstance(hostname)
|
||||||
i.Unlock()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -118,7 +118,14 @@ func (i *InstanceLocator) Locate() {
|
||||||
|
|
||||||
time.Sleep(120 * time.Second)
|
time.Sleep(120 * time.Second)
|
||||||
|
|
||||||
i.instanceReport()
|
r := i.instanceReport()
|
||||||
|
|
||||||
|
log.Debug().
|
||||||
|
Uint("up", r.up).
|
||||||
|
Uint("total", r.total).
|
||||||
|
Uint("identified", r.identified).
|
||||||
|
Msg("instance report")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type InstanceLocatorReport struct {
|
type InstanceLocatorReport struct {
|
||||||
|
@ -136,6 +143,8 @@ func (i *InstanceLocator) NumInstances() uint {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
|
func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
|
||||||
|
i.Lock()
|
||||||
|
defer i.Unlock()
|
||||||
r := new(InstanceLocatorReport)
|
r := new(InstanceLocatorReport)
|
||||||
|
|
||||||
r.total = uint(len(i.instances))
|
r.total = uint(len(i.instances))
|
||||||
|
@ -150,12 +159,6 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().
|
|
||||||
Uint("up", r.up).
|
|
||||||
Uint("total", r.total).
|
|
||||||
Uint("identified", r.identified).
|
|
||||||
Msg("instance report")
|
|
||||||
|
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -179,8 +182,8 @@ func (i *InstanceLocator) locateMastodon() {
|
||||||
i.addInstance(instance.Name)
|
i.addInstance(instance.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
i.Lock()
|
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
|
i.Lock()
|
||||||
i.mastodonIndexLastRefresh = &t
|
i.mastodonIndexLastRefresh = &t
|
||||||
i.Unlock()
|
i.Unlock()
|
||||||
}
|
}
|
||||||
|
@ -204,8 +207,8 @@ func (i *InstanceLocator) locatePleroma() {
|
||||||
for _, instance := range *pi {
|
for _, instance := range *pi {
|
||||||
i.addInstance(instance.Domain)
|
i.addInstance(instance.Domain)
|
||||||
}
|
}
|
||||||
i.Lock()
|
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
|
i.Lock()
|
||||||
i.pleromaIndexLastRefresh = &t
|
i.pleromaIndexLastRefresh = &t
|
||||||
i.Unlock()
|
i.Unlock()
|
||||||
}
|
}
|
||||||
|
|
10
webserver.go
10
webserver.go
|
@ -49,14 +49,18 @@ func (a *TootArchiverAPIServer) getRouter() *gin.Engine {
|
||||||
|
|
||||||
r.GET("/", func(c *gin.Context) {
|
r.GET("/", func(c *gin.Context) {
|
||||||
c.JSON(200, gin.H{
|
c.JSON(200, gin.H{
|
||||||
// FIXME add more stuff here
|
// FIXME(sneak) add more stuff here
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"now": time.Now().UTC().Format(time.RFC3339),
|
"now": time.Now().UTC().Format(time.RFC3339),
|
||||||
"uptime": a.archiver.Uptime().String(),
|
"uptime": a.archiver.Uptime().String(),
|
||||||
"instances": a.archiver.locator.NumInstances(),
|
"instances": gin.H{
|
||||||
|
"total": a.archiver.locator.instanceReport().total,
|
||||||
|
"up": a.archiver.locator.instanceReport().up,
|
||||||
|
"identified": a.archiver.locator.instanceReport().identified,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
// FIXME add more status routes here
|
// FIXME(sneak) add more status routes here
|
||||||
|
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue