sneak/feta
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

renamed

Browse Source
This commit is contained in:
Jeffrey Paul 2019-11-03 02:56:50 -08:00
parent 3aec504aa1
commit 052600663c
6 changed files with 85 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
fediverse-archive
feta
output/

6
Dockerfile
View File

@ -1,6 +1,6 @@
FROM golang:1.13 as builder
WORKDIR /go/src/github.com/sneak/fediverse-archive
WORKDIR /go/src/github.com/sneak/feta
COPY . .
RUN go get -v && make build
@ -13,11 +13,11 @@ RUN tar cvfz go-src.tgz src && du -sh *
FROM alpine
COPY --from=builder /go/src/github.com/sneak/fediverse-archive/fediverse-archive /bin/fediverse-archive
COPY --from=builder /go/src/github.com/sneak/feta/feta /bin/feta
# put the source in there too for safekeeping
COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
CMD /bin/fediverse-archive
CMD /bin/feta
# FIXME add testing

2
Makefile
View File

@ -6,7 +6,7 @@ BUILDUSER := $(shell whoami)
BUILDHOST := $(shell hostname -s)
BUILDARCH := $(shell uname -m)
FN := fediverse-archive
FN := feta
IMAGENAME := sneak/$(FN)
UNAME_S := $(shell uname -s)

74
instance.go
View File

@ -4,6 +4,7 @@ import "encoding/json"
import "fmt"
import "net/http"
import "strings"
import "sync"
import "time"
import "github.com/rs/zerolog/log"
@ -11,6 +12,8 @@ import "github.com/rs/zerolog/log"
const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
const NODE_TIMEOUT = time.Second * 10
const ONE_HOUR = time.Second * 60 * 60
const ONE_DAY = time.Second * 60 * 60 * 24
type ServerImplementation int
@ -21,26 +24,23 @@ const (
)
type Instance struct {
sync.Mutex
errorCount uint
highestId int
hostName string
impl ServerImplementation
lastFailure *time.Time
lastError *time.Time
lastSuccess *time.Time
nextCheck *time.Time
nodeInfoUrl string
serverVersion string
identified bool
up bool
shouldSkip bool
}
func NewInstance(hostname string) *Instance {
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
i := new(Instance)
i.hostName = hostname
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
i.lastSuccess = &foreverago
i.lastFailure = &foreverago
i.identified = false
i.nextCheck = &foreverago
i.up = false
go func() {
i.detectNodeType()
@ -48,10 +48,28 @@ func NewInstance(hostname string) *Instance {
return i
}
func (i *Instance) setNextCheck(d *time.Duration) {
i.Lock()
defer i.Unlock()
i.nextCheck = d
}
func (i *Instance) dueForCheck() bool {
i.Lock()
defer i.Unlock()
if i.nextCheck <= time.Now() {
return true
}
return false
}
func (i *Instance) detectNodeType() {
i.Lock()
if i.impl > ServerUnknown {
i.Unlock()
return
}
i.Unlock()
i.fetchNodeInfo()
}
@ -81,12 +99,16 @@ type NodeInfoVersionTwoSchema struct {
}
func (i *Instance) registerError() {
i.Lock()
defer i.Unlock()
i.errorCount = i.errorCount + 1
t := time.Now()
i.lastFailure = &t
i.lastError = &t
}
func (i *Instance) registerSuccess() {
i.Lock()
defer i.Unlock()
t := time.Now()
i.lastSuccess = &t
}
@ -125,7 +147,10 @@ func (i *Instance) fetchNodeInfoURL() {
Str("hostname", i.hostName).
Str("nodeinfourl", item.Href).
Msg("success fetching url for nodeinfo")
i.Lock()
i.nodeInfoUrl = item.Href
i.Unlock()
i.registerSuccess()
return
}
@ -140,10 +165,18 @@ func (i *Instance) fetchNodeInfoURL() {
func (i *Instance) fetchNodeInfo() {
i.fetchNodeInfoURL()
i.Lock()
failure := false
if i.nodeInfoUrl == "" {
log.Error().
Str("hostname", i.hostName).
Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined")
failure = true
}
i.Unlock()
if failure == true {
return
}
@ -153,7 +186,11 @@ func (i *Instance) fetchNodeInfo() {
//FIXME make sure the nodeinfourl is on the same domain as the instance
//hostname
resp, err := c.Get(i.nodeInfoUrl)
i.Lock()
url := i.nodeInfoUrl
i.Unlock()
resp, err := c.Get(url)
if err != nil {
log.Error().
@ -178,6 +215,8 @@ func (i *Instance) fetchNodeInfo() {
Str("nodeInfoUrl", i.nodeInfoUrl).
Msg("received nodeinfo from instance")
i.Lock()
defer i.Unlock()
i.serverVersion = ni.Software.Version
ni.Software.Name = strings.ToLower(ni.Software.Name)
@ -210,12 +249,16 @@ func (i *Instance) fetchNodeInfo() {
}
func (i *Instance) fetchRecentToots() ([]byte, error) {
if i.impl == ServerMastodon {
i.Lock()
impl := i.impl
i.Unlock()
if impl == ServerMastodon {
return i.fetchRecentTootsJsonFromMastodon()
} else if i.impl == ServerPleroma {
} else if impl == ServerPleroma {
return i.fetchRecentTootsJsonFromPleroma()
} else {
panic("nope")
panic("unimplemented")
}
}
@ -228,8 +271,3 @@ func (i *Instance) fetchRecentTootsJsonFromMastodon() ([]byte, error) {
//url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName)
return nil, nil
}
func fetchLatestToots(lastId int) {
log.Debug().Msg("This message appears only when log level set to Debug")
log.Info().Msg("This message appears when log level set to Debug or Info")
}

25
locator.go
View File

@ -93,12 +93,12 @@ func NewInstanceLocator() *InstanceLocator {
}
func (i *InstanceLocator) addInstance(hostname string) {
i.Lock()
defer i.Unlock()
// only add it if we haven't seen the hostname before
if i.instances[hostname] == nil {
log.Debug().Str("hostname", hostname).Msgf("adding discovered instance")
i.Lock()
i.instances[hostname] = NewInstance(hostname)
i.Unlock()
}
}
@ -118,7 +118,14 @@ func (i *InstanceLocator) Locate() {
time.Sleep(120 * time.Second)
i.instanceReport()
r := i.instanceReport()
log.Debug().
Uint("up", r.up).
Uint("total", r.total).
Uint("identified", r.identified).
Msg("instance report")
}
type InstanceLocatorReport struct {
@ -136,6 +143,8 @@ func (i *InstanceLocator) NumInstances() uint {
}
func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
i.Lock()
defer i.Unlock()
r := new(InstanceLocatorReport)
r.total = uint(len(i.instances))
@ -150,12 +159,6 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
}
}
log.Debug().
Uint("up", r.up).
Uint("total", r.total).
Uint("identified", r.identified).
Msg("instance report")
return r
}
@ -179,8 +182,8 @@ func (i *InstanceLocator) locateMastodon() {
i.addInstance(instance.Name)
}
i.Lock()
t := time.Now()
i.Lock()
i.mastodonIndexLastRefresh = &t
i.Unlock()
}
@ -204,8 +207,8 @@ func (i *InstanceLocator) locatePleroma() {
for _, instance := range *pi {
i.addInstance(instance.Domain)
}
i.Lock()
t := time.Now()
i.Lock()
i.pleromaIndexLastRefresh = &t
i.Unlock()
}

16
webserver.go
View File

@ -49,14 +49,18 @@ func (a *TootArchiverAPIServer) getRouter() *gin.Engine {
r.GET("/", func(c *gin.Context) {
c.JSON(200, gin.H{
// FIXME add more stuff here
"status": "ok",
"now": time.Now().UTC().Format(time.RFC3339),
"uptime": a.archiver.Uptime().String(),
"instances": a.archiver.locator.NumInstances(),
// FIXME(sneak) add more stuff here
"status": "ok",
"now": time.Now().UTC().Format(time.RFC3339),
"uptime": a.archiver.Uptime().String(),
"instances": gin.H{
"total": a.archiver.locator.instanceReport().total,
"up": a.archiver.locator.instanceReport().up,
"identified": a.archiver.locator.instanceReport().identified,
},
})
})
// FIXME add more status routes here
// FIXME(sneak) add more status routes here
return r
}