sneak/feta
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

moved some stuff around, renamed some things

Browse Source
This commit is contained in:
Jeffrey Paul 2019-11-03 04:37:33 -08:00
parent fc03989d08
commit 5f00c3441b
4 changed files with 358 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -46,7 +46,10 @@ fmt:
test: build-docker-image test: build-docker-image
build-docker-image: is_uncommitted:
git diff --exit-code >/dev/null 2>&1
build-docker-image: is_uncommitted
docker build -t $(IMAGENAME):$(VERSION) -t $(IMAGENAME):latest -t $(IMAGENAME):$(BUILDTIMETAG) . docker build -t $(IMAGENAME):$(VERSION) -t $(IMAGENAME):latest -t $(IMAGENAME):$(BUILDTIMETAG) .
dist: build-docker-image dist: build-docker-image

162
instance.go
View File

@ -2,6 +2,7 @@ package main
import "encoding/json" import "encoding/json"
import "fmt" import "fmt"
import "io/ioutil"
import "net/http" import "net/http"
import "strings" import "strings"
import "sync" import "sync"
@ -11,49 +12,57 @@ import "github.com/rs/zerolog/log"
const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
const NODE_TIMEOUT = time.Second * 10 const INSTANCE_HTTP_TIMEOUT = time.Second * 60
const ONE_HOUR = time.Second * 60 * 60
const ONE_DAY = time.Second * 60 * 60 * 24
type ServerImplementation int const INSTANCE_SPIDER_INTERVAL = time.Second * 60
const INSTANCE_ERROR_INTERVAL = time.Second * 60 * 30
type InstanceImplementation int
const ( const (
ServerUnknown ServerImplementation = iota Unknown InstanceImplementation = iota
ServerMastodon Mastodon
ServerPleroma Pleroma
)
type InstanceStatus int
const (
InstanceStatusNone InstanceStatus = iota
InstanceStatusUnknown
InstanceStatusAlive
InstanceStatusFailure
) )
type Instance struct { type Instance struct {
sync.Mutex sync.Mutex
errorCount uint errorCount uint
successCount uint
highestId int highestId int
hostName string hostName string
up bool
identified bool identified bool
impl ServerImplementation impl InstanceImplementation
lastError *time.Time status InstanceStatus
lastSuccess *time.Time
nextCheck *time.Time nextCheck *time.Time
nodeInfoUrl string nodeInfoUrl string
serverVersion string serverVersion string
} }
func NewInstance(hostname string) *Instance { func NewInstance(hostname string) *Instance {
foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
i := new(Instance) i := new(Instance)
i.hostName = hostname i.hostName = hostname
i.nextCheck = &foreverago i.status = InstanceStatusUnknown
i.up = false t := time.Now().Add(-1 * time.Second)
go func() { i.nextCheck = &t
i.detectNodeType() // FIXME make checks detect the node type instead of in the constructor
}()
return i return i
} }
func (i *Instance) setNextCheck(d *time.Duration) { func (i *Instance) setNextCheck(d time.Duration) {
i.Lock() i.Lock()
defer i.Unlock() defer i.Unlock()
then := time.Now().Add(*d) then := time.Now().Add(d)
i.nextCheck = &then i.nextCheck = &then
} }
@ -65,7 +74,7 @@ func (i *Instance) dueForCheck() bool {
func (i *Instance) detectNodeType() { func (i *Instance) detectNodeType() {
i.Lock() i.Lock()
if i.impl > ServerUnknown { if i.impl > Unknown {
i.Unlock() i.Unlock()
return return
} }
@ -73,50 +82,24 @@ func (i *Instance) detectNodeType() {
i.fetchNodeInfo() i.fetchNodeInfo()
} }
type NodeInfoWellKnownResponse struct {
Links []struct {
Rel string `json:"rel"`
Href string `json:"href"`
} `json:"links"`
}
type NodeInfoVersionTwoSchema struct {
Version string `json:"version"`
Software struct {
Name string `json:"name"`
Version string `json:"version"`
} `json:"software"`
Protocols []string `json:"protocols"`
Usage struct {
Users struct {
Total int `json:"total"`
ActiveMonth int `json:"activeMonth"`
ActiveHalfyear int `json:"activeHalfyear"`
} `json:"users"`
LocalPosts int `json:"localPosts"`
} `json:"usage"`
OpenRegistrations bool `json:"openRegistrations"`
}
func (i *Instance) registerError() { func (i *Instance) registerError() {
i.setNextCheck(INSTANCE_ERROR_INTERVAL)
i.Lock() i.Lock()
defer i.Unlock() defer i.Unlock()
i.errorCount = i.errorCount + 1 i.errorCount++
t := time.Now()
i.lastError = &t
} }
func (i *Instance) registerSuccess() { func (i *Instance) registerSuccess() {
i.setNextCheck(INSTANCE_SPIDER_INTERVAL)
i.Lock() i.Lock()
defer i.Unlock() defer i.Unlock()
t := time.Now() i.successCount++
i.lastSuccess = &t
} }
func (i *Instance) fetchNodeInfoURL() { func (i *Instance) fetchNodeInfoURL() {
url := fmt.Sprintf("https://%s/.well-known/nodeinfo", i.hostName) url := fmt.Sprintf("https://%s/.well-known/nodeinfo", i.hostName)
var c = &http.Client{ var c = &http.Client{
Timeout: NODE_TIMEOUT, Timeout: INSTANCE_HTTP_TIMEOUT,
} }
log.Debug(). log.Debug().
@ -126,21 +109,37 @@ func (i *Instance) fetchNodeInfoURL() {
resp, err := c.Get(url) resp, err := c.Get(url)
if err != nil { if err != nil {
log.Error(). log.Debug().
Str("hostname", i.hostName). Str("hostname", i.hostName).
Err(err).
Msg("unable to fetch nodeinfo, node is down?") Msg("unable to fetch nodeinfo, node is down?")
i.registerError() i.registerError()
} else {
i.up = true // node is alive and responding to us
nir := new(NodeInfoWellKnownResponse)
err = json.NewDecoder(resp.Body).Decode(&nir)
if err != nil {
log.Error().
Str("hostname", i.hostName).
Msg("unable to parse nodeinfo")
i.registerError()
return return
} }
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Debug().
Str("hostname", i.hostName).
Err(err).
Msg("unable to read nodeinfo")
i.registerError()
return
}
nir := new(NodeInfoWellKnownResponse)
err = json.Unmarshal(body, &nir)
if err != nil {
log.Error().
Str("hostname", i.hostName).
Err(err).
Msg("unable to parse nodeinfo, node is weird")
i.registerError()
return
}
for _, item := range nir.Links { for _, item := range nir.Links {
if item.Rel == NodeInfoSchemaVersionTwoName { if item.Rel == NodeInfoSchemaVersionTwoName {
log.Info(). log.Info().
@ -155,13 +154,13 @@ func (i *Instance) fetchNodeInfoURL() {
return return
} }
} }
log.Error(). log.Error().
Str("hostname", i.hostName). Str("hostname", i.hostName).
Msg("incomplete nodeinfo") Msg("incomplete nodeinfo")
i.registerError() i.registerError()
return return
} }
}
func (i *Instance) fetchNodeInfo() { func (i *Instance) fetchNodeInfo() {
i.fetchNodeInfoURL() i.fetchNodeInfoURL()
@ -181,7 +180,7 @@ func (i *Instance) fetchNodeInfo() {
} }
var c = &http.Client{ var c = &http.Client{
Timeout: NODE_TIMEOUT, Timeout: INSTANCE_HTTP_TIMEOUT,
} }
//FIXME make sure the nodeinfourl is on the same domain as the instance //FIXME make sure the nodeinfourl is on the same domain as the instance
@ -195,15 +194,31 @@ func (i *Instance) fetchNodeInfo() {
if err != nil { if err != nil {
log.Error(). log.Error().
Str("hostname", i.hostName). Str("hostname", i.hostName).
Msgf("unable to fetch nodeinfo data: %s", err) Err(err).
Msgf("unable to fetch nodeinfo data")
i.registerError() i.registerError()
} else { return
ni := new(NodeInfoVersionTwoSchema) }
err = json.NewDecoder(resp.Body).Decode(&ni)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Error(). log.Error().
Str("hostname", i.hostName). Str("hostname", i.hostName).
Msgf("unable to parse nodeinfo: %s", err) Err(err).
Msgf("unable to read nodeinfo data")
i.registerError()
return
}
ni := new(NodeInfoVersionTwoSchema)
err = json.Unmarshal(body, &ni)
if err != nil {
log.Error().
Str("hostname", i.hostName).
Err(err).
Msgf("unable to parse nodeinfo")
i.registerError() i.registerError()
return return
} }
@ -228,7 +243,8 @@ func (i *Instance) fetchNodeInfo() {
Msg("detected server software") Msg("detected server software")
i.registerSuccess() i.registerSuccess()
i.identified = true i.identified = true
i.impl = ServerPleroma i.impl = Pleroma
i.status = InstanceStatusAlive
} else if ni.Software.Name == "mastodon" { } else if ni.Software.Name == "mastodon" {
log.Info(). log.Info().
Str("hostname", i.hostName). Str("hostname", i.hostName).
@ -236,7 +252,8 @@ func (i *Instance) fetchNodeInfo() {
Msg("detected server software") Msg("detected server software")
i.registerSuccess() i.registerSuccess()
i.identified = true i.identified = true
i.impl = ServerMastodon i.impl = Mastodon
i.status = InstanceStatusAlive
} else { } else {
log.Error(). log.Error().
Str("hostname", i.hostName). Str("hostname", i.hostName).
@ -246,16 +263,15 @@ func (i *Instance) fetchNodeInfo() {
} }
return return
} }
}
func (i *Instance) fetchRecentToots() ([]byte, error) { func (i *Instance) fetchRecentToots() ([]byte, error) {
i.Lock() i.Lock()
impl := i.impl impl := i.impl
i.Unlock() i.Unlock()
if impl == ServerMastodon { if impl == Mastodon {
return i.fetchRecentTootsJsonFromMastodon() return i.fetchRecentTootsJsonFromMastodon()
} else if impl == ServerPleroma { } else if impl == Pleroma {
return i.fetchRecentTootsJsonFromPleroma() return i.fetchRecentTootsJsonFromPleroma()
} else { } else {
panic("unimplemented") panic("unimplemented")

88
jsonapis.go Normal file
View File

@ -0,0 +1,88 @@
package main
import "time"
// thank fuck for https://mholt.github.io/json-to-go/ otherwise
// this would have been a giant pain in the dick
type MastodonIndexResponse struct {
Instances []struct {
ID string `json:"_id"`
AddedAt time.Time `json:"addedAt"`
Name string `json:"name"`
Downchecks int `json:"downchecks"`
Upchecks int `json:"upchecks"`
HTTPSRank interface{} `json:"https_rank"`
HTTPSScore int `json:"https_score"`
ObsRank string `json:"obs_rank"`
ObsScore int `json:"obs_score"`
Ipv6 bool `json:"ipv6"`
Up bool `json:"up"`
Users int `json:"users"`
Statuses string `json:"statuses"`
Connections int `json:"connections"`
OpenRegistrations bool `json:"openRegistrations"`
Uptime float64 `json:"uptime"`
Version string `json:"version"`
VersionScore int `json:"version_score"`
UpdatedAt time.Time `json:"updatedAt"`
CheckedAt time.Time `json:"checkedAt"`
Dead bool `json:"dead"`
ObsDate time.Time `json:"obs_date"`
Second60 int `json:"second60"`
Second int `json:"second"`
ActiveUserCount interface{} `json:"active_user_count,omitempty"`
FirstUserCreatedAt interface{} `json:"first_user_created_at,omitempty"`
Thumbnail string `json:"thumbnail"`
ApUpdatedAt time.Time `json:"apUpdatedAt"`
Second5 int `json:"second5"`
RawVersion string `json:"raw_version"`
ActivityPrevw struct {
Statuses int `json:"statuses"`
Logins int `json:"logins"`
Registrations int `json:"registrations"`
} `json:"activity_prevw,omitempty"`
Mastodon bool `json:"mastodon"`
UptimeStr string `json:"uptime_str"`
Score int `json:"score"`
ScoreStr string `json:"score_str"`
} `json:"instances"`
}
type PleromaIndexResponse []struct {
Domain string `json:"domain"`
Title string `json:"title"`
Thumbnail string `json:"thumbnail"`
Registration bool `json:"registration"`
Chat bool `json:"chat"`
Gopher bool `json:"gopher"`
WhoToFollow bool `json:"who_to_follow"`
MediaProxy bool `json:"media_proxy"`
ScopeOptions bool `json:"scope_options"`
AccountActivationRequired bool `json:"account_activation_required"`
TextLimit int `json:"text_limit"`
}
type NodeInfoVersionTwoSchema struct {
Version string `json:"version"`
Software struct {
Name string `json:"name"`
Version string `json:"version"`
} `json:"software"`
Protocols []string `json:"protocols"`
Usage struct {
Users struct {
Total int `json:"total"`
ActiveMonth int `json:"activeMonth"`
ActiveHalfyear int `json:"activeHalfyear"`
} `json:"users"`
LocalPosts int `json:"localPosts"`
} `json:"usage"`
OpenRegistrations bool `json:"openRegistrations"`
}
type NodeInfoWellKnownResponse struct {
Links []struct {
Rel string `json:"rel"`
Href string `json:"href"`
} `json:"links"`
}

212
locator.go
View File

@ -2,84 +2,27 @@ package main
import "encoding/json" import "encoding/json"
import "fmt" import "fmt"
import "io/ioutil"
import "net/http" import "net/http"
import "sync" import "sync"
import "time" import "time"
import "github.com/rs/zerolog/log" import "github.com/rs/zerolog/log"
const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const INDEX_API_TIMEOUT = time.Second * 60
var foreverago = time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
// check with indices only hourly // check with indices only hourly
var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 var INDEX_CHECK_INTERVAL = time.Second * 60 * 60
// thank fuck for https://mholt.github.io/json-to-go/ otherwise // check with indices after 10 mins if they failed
// this would have been a giant pain in the dick var INDEX_ERROR_INTERVAL = time.Second * 60 * 10
type MastodonIndexResponse struct {
Instances []struct {
ID string `json:"_id"`
AddedAt time.Time `json:"addedAt"`
Name string `json:"name"`
Downchecks int `json:"downchecks"`
Upchecks int `json:"upchecks"`
HTTPSRank interface{} `json:"https_rank"`
HTTPSScore int `json:"https_score"`
ObsRank string `json:"obs_rank"`
ObsScore int `json:"obs_score"`
Ipv6 bool `json:"ipv6"`
Up bool `json:"up"`
Users int `json:"users"`
Statuses string `json:"statuses"`
Connections int `json:"connections"`
OpenRegistrations bool `json:"openRegistrations"`
Uptime float64 `json:"uptime"`
Version string `json:"version"`
VersionScore int `json:"version_score"`
UpdatedAt time.Time `json:"updatedAt"`
CheckedAt time.Time `json:"checkedAt"`
Dead bool `json:"dead"`
ObsDate time.Time `json:"obs_date"`
Second60 int `json:"second60"`
Second int `json:"second"`
ActiveUserCount interface{} `json:"active_user_count,omitempty"`
FirstUserCreatedAt interface{} `json:"first_user_created_at,omitempty"`
Thumbnail string `json:"thumbnail"`
ApUpdatedAt time.Time `json:"apUpdatedAt"`
Second5 int `json:"second5"`
RawVersion string `json:"raw_version"`
ActivityPrevw struct {
Statuses int `json:"statuses"`
Logins int `json:"logins"`
Registrations int `json:"registrations"`
} `json:"activity_prevw,omitempty"`
Mastodon bool `json:"mastodon"`
UptimeStr string `json:"uptime_str"`
Score int `json:"score"`
ScoreStr string `json:"score_str"`
} `json:"instances"`
}
const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"
const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi"
type PleromaIndexResponse []struct {
Domain string `json:"domain"`
Title string `json:"title"`
Thumbnail string `json:"thumbnail"`
Registration bool `json:"registration"`
Chat bool `json:"chat"`
Gopher bool `json:"gopher"`
WhoToFollow bool `json:"who_to_follow"`
MediaProxy bool `json:"media_proxy"`
ScopeOptions bool `json:"scope_options"`
AccountActivationRequired bool `json:"account_activation_required"`
TextLimit int `json:"text_limit"`
}
type InstanceLocator struct { type InstanceLocator struct {
pleromaIndexLastRefresh *time.Time pleromaIndexNextRefresh *time.Time
mastodonIndexLastRefresh *time.Time mastodonIndexNextRefresh *time.Time
instances map[string]*Instance instances map[string]*Instance
sync.Mutex sync.Mutex
} }
@ -87,8 +30,9 @@ type InstanceLocator struct {
func NewInstanceLocator() *InstanceLocator { func NewInstanceLocator() *InstanceLocator {
i := new(InstanceLocator) i := new(InstanceLocator)
i.instances = make(map[string]*Instance) i.instances = make(map[string]*Instance)
i.pleromaIndexLastRefresh = &foreverago n := time.Now()
i.mastodonIndexLastRefresh = &foreverago i.pleromaIndexNextRefresh = &n
i.mastodonIndexNextRefresh = &n
return i return i
} }
@ -97,35 +41,44 @@ func (i *InstanceLocator) addInstance(hostname string) {
defer i.Unlock() defer i.Unlock()
// only add it if we haven't seen the hostname before // only add it if we haven't seen the hostname before
if i.instances[hostname] == nil { if i.instances[hostname] == nil {
log.Debug().Str("hostname", hostname).Msgf("adding discovered instance") log.Info().Str("hostname", hostname).Msgf("adding discovered instance")
i.instances[hostname] = NewInstance(hostname) i.instances[hostname] = NewInstance(hostname)
} }
} }
func (i *InstanceLocator) Locate() { func (i *InstanceLocator) Locate() {
log.Debug(). x := 0
Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)). for {
Send() if i.pleromaIndexNextRefresh.Before(time.Now()) {
log.Debug().
Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)).
Send()
i.locateMastodon()
i.locatePleroma() i.locatePleroma()
}
time.Sleep(120 * time.Second) if i.mastodonIndexNextRefresh.Before(time.Now()) {
i.locateMastodon()
r := i.instanceReport() }
time.Sleep(1 * time.Second)
x++
if x == 60 {
x = 0
log.Debug(). log.Debug().
Str("nextmastodonupdate", i.mastodonIndexNextRefresh.Format(time.RFC3339)).
Send()
log.Debug().
Str("nextpleromaupdate", i.pleromaIndexNextRefresh.Format(time.RFC3339)).
Send()
i.logInstanceReport()
}
}
}
func (i *InstanceLocator) logInstanceReport() {
r := i.instanceReport()
log.Info().
Uint("up", r.up). Uint("up", r.up).
Uint("total", r.total). Uint("total", r.total).
Uint("identified", r.identified). Uint("identified", r.identified).
Msg("instance report") Msg("instance report")
} }
type InstanceLocatorReport struct { type InstanceLocatorReport struct {
@ -154,7 +107,7 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
r.identified = r.identified + 1 r.identified = r.identified + 1
} }
if elem.up == true { if elem.status == InstanceStatusAlive {
r.up = r.up + 1 r.up = r.up + 1
} }
} }
@ -163,54 +116,97 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
} }
func (i *InstanceLocator) locateMastodon() { func (i *InstanceLocator) locateMastodon() {
var netClient = &http.Client{ var c = &http.Client{
Timeout: NODE_TIMEOUT, Timeout: INDEX_API_TIMEOUT,
} }
resp, err := netClient.Get(mastodonIndexUrl)
resp, err := c.Get(mastodonIndexUrl)
if err != nil {
log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
i.Lock()
i.mastodonIndexNextRefresh = &t
i.Unlock()
return
}
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Warn().Msgf("unable to fetch mastodon instance list: %s", err) log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
} else { t := time.Now().Add(INDEX_ERROR_INTERVAL)
// it worked i.Lock()
i.mastodonIndexNextRefresh = &t
i.Unlock()
return
}
mi := new(MastodonIndexResponse) mi := new(MastodonIndexResponse)
err = json.NewDecoder(resp.Body).Decode(&mi) err = json.Unmarshal(body, &mi)
if err != nil { if err != nil {
log.Warn().Msgf("unable to parse mastodon instance list: %s", err) log.Error().Msgf("unable to parse mastodon instance list: %s", err)
} else { t := time.Now().Add(INDEX_ERROR_INTERVAL)
i.Lock()
i.mastodonIndexNextRefresh = &t
i.Unlock()
return
}
for _, instance := range mi.Instances { for _, instance := range mi.Instances {
i.addInstance(instance.Name) i.addInstance(instance.Name)
} }
t := time.Now() t := time.Now().Add(INDEX_CHECK_INTERVAL)
i.Lock() i.Lock()
i.mastodonIndexLastRefresh = &t i.mastodonIndexNextRefresh = &t
i.Unlock() i.Unlock()
} }
}
}
func (i *InstanceLocator) locatePleroma() { func (i *InstanceLocator) locatePleroma() {
var netClient = &http.Client{ var c = &http.Client{
Timeout: NODE_TIMEOUT, Timeout: INDEX_API_TIMEOUT,
} }
resp, err := netClient.Get(pleromaIndexUrl) resp, err := c.Get(pleromaIndexUrl)
if err != nil { if err != nil {
log.Warn().Msgf("unable to fetch pleroma instance list: %s", err) log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
} else { t := time.Now().Add(INDEX_ERROR_INTERVAL)
i.Lock()
i.pleromaIndexNextRefresh = &t
i.Unlock()
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
i.Lock()
i.pleromaIndexNextRefresh = &t
i.Unlock()
return
}
// fetch worked // fetch worked
pi := new(PleromaIndexResponse) pi := new(PleromaIndexResponse)
err = json.NewDecoder(resp.Body).Decode(&pi) err = json.Unmarshal(body, &pi)
if err != nil { if err != nil {
log.Warn().Msgf("unable to parse pleroma instance list: %s", err) log.Warn().Msgf("unable to parse pleroma instance list: %s", err)
} else { t := time.Now().Add(INDEX_ERROR_INTERVAL)
i.Lock()
i.pleromaIndexNextRefresh = &t
i.Unlock()
return
}
for _, instance := range *pi { for _, instance := range *pi {
i.addInstance(instance.Domain) i.addInstance(instance.Domain)
} }
t := time.Now() t := time.Now().Add(INDEX_CHECK_INTERVAL)
i.Lock() i.Lock()
i.pleromaIndexLastRefresh = &t i.pleromaIndexNextRefresh = &t
i.Unlock() i.Unlock()
} }
}
}