diff --git a/Makefile b/Makefile index 62131b6..47bb03a 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,10 @@ fmt: test: build-docker-image -build-docker-image: +is_uncommitted: + git diff --exit-code >/dev/null 2>&1 + +build-docker-image: is_uncommitted docker build -t $(IMAGENAME):$(VERSION) -t $(IMAGENAME):latest -t $(IMAGENAME):$(BUILDTIMETAG) . dist: build-docker-image diff --git a/instance.go b/instance.go index 8056d52..96d188a 100644 --- a/instance.go +++ b/instance.go @@ -2,6 +2,7 @@ package main import "encoding/json" import "fmt" +import "io/ioutil" import "net/http" import "strings" import "sync" @@ -11,49 +12,57 @@ import "github.com/rs/zerolog/log" const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" -const NODE_TIMEOUT = time.Second * 10 -const ONE_HOUR = time.Second * 60 * 60 -const ONE_DAY = time.Second * 60 * 60 * 24 +const INSTANCE_HTTP_TIMEOUT = time.Second * 60 -type ServerImplementation int +const INSTANCE_SPIDER_INTERVAL = time.Second * 60 + +const INSTANCE_ERROR_INTERVAL = time.Second * 60 * 30 + +type InstanceImplementation int const ( - ServerUnknown ServerImplementation = iota - ServerMastodon - ServerPleroma + Unknown InstanceImplementation = iota + Mastodon + Pleroma +) + +type InstanceStatus int + +const ( + InstanceStatusNone InstanceStatus = iota + InstanceStatusUnknown + InstanceStatusAlive + InstanceStatusFailure ) type Instance struct { sync.Mutex errorCount uint + successCount uint highestId int hostName string - up bool identified bool - impl ServerImplementation - lastError *time.Time - lastSuccess *time.Time + impl InstanceImplementation + status InstanceStatus nextCheck *time.Time nodeInfoUrl string serverVersion string } func NewInstance(hostname string) *Instance { - foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second) i := new(Instance) i.hostName = hostname - i.nextCheck = &foreverago - i.up = false - go func() { - i.detectNodeType() - }() + i.status = InstanceStatusUnknown + t := time.Now().Add(-1 * time.Second) + i.nextCheck = &t + // FIXME make checks detect the node type instead of in the constructor return i } -func (i *Instance) setNextCheck(d *time.Duration) { +func (i *Instance) setNextCheck(d time.Duration) { i.Lock() defer i.Unlock() - then := time.Now().Add(*d) + then := time.Now().Add(d) i.nextCheck = &then } @@ -65,7 +74,7 @@ func (i *Instance) dueForCheck() bool { func (i *Instance) detectNodeType() { i.Lock() - if i.impl > ServerUnknown { + if i.impl > Unknown { i.Unlock() return } @@ -73,50 +82,24 @@ func (i *Instance) detectNodeType() { i.fetchNodeInfo() } -type NodeInfoWellKnownResponse struct { - Links []struct { - Rel string `json:"rel"` - Href string `json:"href"` - } `json:"links"` -} - -type NodeInfoVersionTwoSchema struct { - Version string `json:"version"` - Software struct { - Name string `json:"name"` - Version string `json:"version"` - } `json:"software"` - Protocols []string `json:"protocols"` - Usage struct { - Users struct { - Total int `json:"total"` - ActiveMonth int `json:"activeMonth"` - ActiveHalfyear int `json:"activeHalfyear"` - } `json:"users"` - LocalPosts int `json:"localPosts"` - } `json:"usage"` - OpenRegistrations bool `json:"openRegistrations"` -} - func (i *Instance) registerError() { + i.setNextCheck(INSTANCE_ERROR_INTERVAL) i.Lock() defer i.Unlock() - i.errorCount = i.errorCount + 1 - t := time.Now() - i.lastError = &t + i.errorCount++ } func (i *Instance) registerSuccess() { + i.setNextCheck(INSTANCE_SPIDER_INTERVAL) i.Lock() defer i.Unlock() - t := time.Now() - i.lastSuccess = &t + i.successCount++ } func (i *Instance) fetchNodeInfoURL() { url := fmt.Sprintf("https://%s/.well-known/nodeinfo", i.hostName) var c = &http.Client{ - Timeout: NODE_TIMEOUT, + Timeout: INSTANCE_HTTP_TIMEOUT, } log.Debug(). @@ -126,41 +109,57 @@ func (i *Instance) fetchNodeInfoURL() { resp, err := c.Get(url) if err != nil { - log.Error(). + log.Debug(). Str("hostname", i.hostName). + Err(err). Msg("unable to fetch nodeinfo, node is down?") i.registerError() - } else { - i.up = true // node is alive and responding to us - nir := new(NodeInfoWellKnownResponse) - err = json.NewDecoder(resp.Body).Decode(&nir) - if err != nil { - log.Error(). - Str("hostname", i.hostName). - Msg("unable to parse nodeinfo") - i.registerError() - return - } - for _, item := range nir.Links { - if item.Rel == NodeInfoSchemaVersionTwoName { - log.Info(). - Str("hostname", i.hostName). - Str("nodeinfourl", item.Href). - Msg("success fetching url for nodeinfo") - - i.Lock() - i.nodeInfoUrl = item.Href - i.Unlock() - i.registerSuccess() - return - } - } - log.Error(). - Str("hostname", i.hostName). - Msg("incomplete nodeinfo") - i.registerError() return } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + log.Debug(). + Str("hostname", i.hostName). + Err(err). + Msg("unable to read nodeinfo") + i.registerError() + return + } + + nir := new(NodeInfoWellKnownResponse) + err = json.Unmarshal(body, &nir) + if err != nil { + log.Error(). + Str("hostname", i.hostName). + Err(err). + Msg("unable to parse nodeinfo, node is weird") + i.registerError() + return + } + + for _, item := range nir.Links { + if item.Rel == NodeInfoSchemaVersionTwoName { + log.Info(). + Str("hostname", i.hostName). + Str("nodeinfourl", item.Href). + Msg("success fetching url for nodeinfo") + + i.Lock() + i.nodeInfoUrl = item.Href + i.Unlock() + i.registerSuccess() + return + } + } + + log.Error(). + Str("hostname", i.hostName). + Msg("incomplete nodeinfo") + i.registerError() + return } func (i *Instance) fetchNodeInfo() { @@ -181,7 +180,7 @@ func (i *Instance) fetchNodeInfo() { } var c = &http.Client{ - Timeout: NODE_TIMEOUT, + Timeout: INSTANCE_HTTP_TIMEOUT, } //FIXME make sure the nodeinfourl is on the same domain as the instance @@ -195,57 +194,74 @@ func (i *Instance) fetchNodeInfo() { if err != nil { log.Error(). Str("hostname", i.hostName). - Msgf("unable to fetch nodeinfo data: %s", err) + Err(err). + Msgf("unable to fetch nodeinfo data") i.registerError() - } else { - ni := new(NodeInfoVersionTwoSchema) - err = json.NewDecoder(resp.Body).Decode(&ni) - if err != nil { - log.Error(). - Str("hostname", i.hostName). - Msgf("unable to parse nodeinfo: %s", err) - i.registerError() - return - } - - log.Info(). - Str("serverVersion", ni.Software.Version). - Str("software", ni.Software.Name). - Str("hostName", i.hostName). - Str("nodeInfoUrl", i.nodeInfoUrl). - Msg("received nodeinfo from instance") - - i.Lock() - defer i.Unlock() - i.serverVersion = ni.Software.Version - - ni.Software.Name = strings.ToLower(ni.Software.Name) - - if ni.Software.Name == "pleroma" { - log.Info(). - Str("hostname", i.hostName). - Str("software", ni.Software.Name). - Msg("detected server software") - i.registerSuccess() - i.identified = true - i.impl = ServerPleroma - } else if ni.Software.Name == "mastodon" { - log.Info(). - Str("hostname", i.hostName). - Str("software", ni.Software.Name). - Msg("detected server software") - i.registerSuccess() - i.identified = true - i.impl = ServerMastodon - } else { - log.Error(). - Str("hostname", i.hostName). - Str("software", ni.Software.Name). - Msg("unknown implementation on server") - i.registerError() - } return } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + log.Error(). + Str("hostname", i.hostName). + Err(err). + Msgf("unable to read nodeinfo data") + i.registerError() + return + } + + ni := new(NodeInfoVersionTwoSchema) + err = json.Unmarshal(body, &ni) + if err != nil { + log.Error(). + Str("hostname", i.hostName). + Err(err). + Msgf("unable to parse nodeinfo") + i.registerError() + return + } + + log.Info(). + Str("serverVersion", ni.Software.Version). + Str("software", ni.Software.Name). + Str("hostName", i.hostName). + Str("nodeInfoUrl", i.nodeInfoUrl). + Msg("received nodeinfo from instance") + + i.Lock() + defer i.Unlock() + i.serverVersion = ni.Software.Version + + ni.Software.Name = strings.ToLower(ni.Software.Name) + + if ni.Software.Name == "pleroma" { + log.Info(). + Str("hostname", i.hostName). + Str("software", ni.Software.Name). + Msg("detected server software") + i.registerSuccess() + i.identified = true + i.impl = Pleroma + i.status = InstanceStatusAlive + } else if ni.Software.Name == "mastodon" { + log.Info(). + Str("hostname", i.hostName). + Str("software", ni.Software.Name). + Msg("detected server software") + i.registerSuccess() + i.identified = true + i.impl = Mastodon + i.status = InstanceStatusAlive + } else { + log.Error(). + Str("hostname", i.hostName). + Str("software", ni.Software.Name). + Msg("unknown implementation on server") + i.registerError() + } + return } func (i *Instance) fetchRecentToots() ([]byte, error) { @@ -253,9 +269,9 @@ func (i *Instance) fetchRecentToots() ([]byte, error) { impl := i.impl i.Unlock() - if impl == ServerMastodon { + if impl == Mastodon { return i.fetchRecentTootsJsonFromMastodon() - } else if impl == ServerPleroma { + } else if impl == Pleroma { return i.fetchRecentTootsJsonFromPleroma() } else { panic("unimplemented") diff --git a/jsonapis.go b/jsonapis.go new file mode 100644 index 0000000..b671389 --- /dev/null +++ b/jsonapis.go @@ -0,0 +1,88 @@ +package main + +import "time" + +// thank fuck for https://mholt.github.io/json-to-go/ otherwise +// this would have been a giant pain in the dick +type MastodonIndexResponse struct { + Instances []struct { + ID string `json:"_id"` + AddedAt time.Time `json:"addedAt"` + Name string `json:"name"` + Downchecks int `json:"downchecks"` + Upchecks int `json:"upchecks"` + HTTPSRank interface{} `json:"https_rank"` + HTTPSScore int `json:"https_score"` + ObsRank string `json:"obs_rank"` + ObsScore int `json:"obs_score"` + Ipv6 bool `json:"ipv6"` + Up bool `json:"up"` + Users int `json:"users"` + Statuses string `json:"statuses"` + Connections int `json:"connections"` + OpenRegistrations bool `json:"openRegistrations"` + Uptime float64 `json:"uptime"` + Version string `json:"version"` + VersionScore int `json:"version_score"` + UpdatedAt time.Time `json:"updatedAt"` + CheckedAt time.Time `json:"checkedAt"` + Dead bool `json:"dead"` + ObsDate time.Time `json:"obs_date"` + Second60 int `json:"second60"` + Second int `json:"second"` + ActiveUserCount interface{} `json:"active_user_count,omitempty"` + FirstUserCreatedAt interface{} `json:"first_user_created_at,omitempty"` + Thumbnail string `json:"thumbnail"` + ApUpdatedAt time.Time `json:"apUpdatedAt"` + Second5 int `json:"second5"` + RawVersion string `json:"raw_version"` + ActivityPrevw struct { + Statuses int `json:"statuses"` + Logins int `json:"logins"` + Registrations int `json:"registrations"` + } `json:"activity_prevw,omitempty"` + Mastodon bool `json:"mastodon"` + UptimeStr string `json:"uptime_str"` + Score int `json:"score"` + ScoreStr string `json:"score_str"` + } `json:"instances"` +} + +type PleromaIndexResponse []struct { + Domain string `json:"domain"` + Title string `json:"title"` + Thumbnail string `json:"thumbnail"` + Registration bool `json:"registration"` + Chat bool `json:"chat"` + Gopher bool `json:"gopher"` + WhoToFollow bool `json:"who_to_follow"` + MediaProxy bool `json:"media_proxy"` + ScopeOptions bool `json:"scope_options"` + AccountActivationRequired bool `json:"account_activation_required"` + TextLimit int `json:"text_limit"` +} + +type NodeInfoVersionTwoSchema struct { + Version string `json:"version"` + Software struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"software"` + Protocols []string `json:"protocols"` + Usage struct { + Users struct { + Total int `json:"total"` + ActiveMonth int `json:"activeMonth"` + ActiveHalfyear int `json:"activeHalfyear"` + } `json:"users"` + LocalPosts int `json:"localPosts"` + } `json:"usage"` + OpenRegistrations bool `json:"openRegistrations"` +} + +type NodeInfoWellKnownResponse struct { + Links []struct { + Rel string `json:"rel"` + Href string `json:"href"` + } `json:"links"` +} diff --git a/locator.go b/locator.go index 5668eab..c53dffd 100644 --- a/locator.go +++ b/locator.go @@ -2,84 +2,27 @@ package main import "encoding/json" import "fmt" +import "io/ioutil" import "net/http" import "sync" import "time" import "github.com/rs/zerolog/log" -const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" - -var foreverago = time.Now().Add((-1 * 86400 * 365 * 100) * time.Second) +const INDEX_API_TIMEOUT = time.Second * 60 // check with indices only hourly var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 -// thank fuck for https://mholt.github.io/json-to-go/ otherwise -// this would have been a giant pain in the dick -type MastodonIndexResponse struct { - Instances []struct { - ID string `json:"_id"` - AddedAt time.Time `json:"addedAt"` - Name string `json:"name"` - Downchecks int `json:"downchecks"` - Upchecks int `json:"upchecks"` - HTTPSRank interface{} `json:"https_rank"` - HTTPSScore int `json:"https_score"` - ObsRank string `json:"obs_rank"` - ObsScore int `json:"obs_score"` - Ipv6 bool `json:"ipv6"` - Up bool `json:"up"` - Users int `json:"users"` - Statuses string `json:"statuses"` - Connections int `json:"connections"` - OpenRegistrations bool `json:"openRegistrations"` - Uptime float64 `json:"uptime"` - Version string `json:"version"` - VersionScore int `json:"version_score"` - UpdatedAt time.Time `json:"updatedAt"` - CheckedAt time.Time `json:"checkedAt"` - Dead bool `json:"dead"` - ObsDate time.Time `json:"obs_date"` - Second60 int `json:"second60"` - Second int `json:"second"` - ActiveUserCount interface{} `json:"active_user_count,omitempty"` - FirstUserCreatedAt interface{} `json:"first_user_created_at,omitempty"` - Thumbnail string `json:"thumbnail"` - ApUpdatedAt time.Time `json:"apUpdatedAt"` - Second5 int `json:"second5"` - RawVersion string `json:"raw_version"` - ActivityPrevw struct { - Statuses int `json:"statuses"` - Logins int `json:"logins"` - Registrations int `json:"registrations"` - } `json:"activity_prevw,omitempty"` - Mastodon bool `json:"mastodon"` - UptimeStr string `json:"uptime_str"` - Score int `json:"score"` - ScoreStr string `json:"score_str"` - } `json:"instances"` -} +// check with indices after 10 mins if they failed +var INDEX_ERROR_INTERVAL = time.Second * 60 * 10 +const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" -type PleromaIndexResponse []struct { - Domain string `json:"domain"` - Title string `json:"title"` - Thumbnail string `json:"thumbnail"` - Registration bool `json:"registration"` - Chat bool `json:"chat"` - Gopher bool `json:"gopher"` - WhoToFollow bool `json:"who_to_follow"` - MediaProxy bool `json:"media_proxy"` - ScopeOptions bool `json:"scope_options"` - AccountActivationRequired bool `json:"account_activation_required"` - TextLimit int `json:"text_limit"` -} - type InstanceLocator struct { - pleromaIndexLastRefresh *time.Time - mastodonIndexLastRefresh *time.Time + pleromaIndexNextRefresh *time.Time + mastodonIndexNextRefresh *time.Time instances map[string]*Instance sync.Mutex } @@ -87,8 +30,9 @@ type InstanceLocator struct { func NewInstanceLocator() *InstanceLocator { i := new(InstanceLocator) i.instances = make(map[string]*Instance) - i.pleromaIndexLastRefresh = &foreverago - i.mastodonIndexLastRefresh = &foreverago + n := time.Now() + i.pleromaIndexNextRefresh = &n + i.mastodonIndexNextRefresh = &n return i } @@ -97,35 +41,44 @@ func (i *InstanceLocator) addInstance(hostname string) { defer i.Unlock() // only add it if we haven't seen the hostname before if i.instances[hostname] == nil { - log.Debug().Str("hostname", hostname).Msgf("adding discovered instance") + log.Info().Str("hostname", hostname).Msgf("adding discovered instance") i.instances[hostname] = NewInstance(hostname) } } func (i *InstanceLocator) Locate() { - log.Debug(). - Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)). - Send() + x := 0 + for { + if i.pleromaIndexNextRefresh.Before(time.Now()) { + i.locatePleroma() + } + if i.mastodonIndexNextRefresh.Before(time.Now()) { + i.locateMastodon() + } + time.Sleep(1 * time.Second) + x++ + if x == 60 { + x = 0 + log.Debug(). + Str("nextmastodonupdate", i.mastodonIndexNextRefresh.Format(time.RFC3339)). + Send() + log.Debug(). + Str("nextpleromaupdate", i.pleromaIndexNextRefresh.Format(time.RFC3339)). + Send() + i.logInstanceReport() + } + } - log.Debug(). - Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)). - Send() - - i.locateMastodon() - - i.locatePleroma() - - time.Sleep(120 * time.Second) +} +func (i *InstanceLocator) logInstanceReport() { r := i.instanceReport() - - log.Debug(). + log.Info(). Uint("up", r.up). Uint("total", r.total). Uint("identified", r.identified). Msg("instance report") - } type InstanceLocatorReport struct { @@ -154,7 +107,7 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { r.identified = r.identified + 1 } - if elem.up == true { + if elem.status == InstanceStatusAlive { r.up = r.up + 1 } } @@ -163,54 +116,97 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { } func (i *InstanceLocator) locateMastodon() { - var netClient = &http.Client{ - Timeout: NODE_TIMEOUT, + var c = &http.Client{ + Timeout: INDEX_API_TIMEOUT, } - resp, err := netClient.Get(mastodonIndexUrl) + + resp, err := c.Get(mastodonIndexUrl) + if err != nil { + log.Error().Msgf("unable to fetch mastodon instance list: %s", err) + t := time.Now().Add(INDEX_ERROR_INTERVAL) + i.Lock() + i.mastodonIndexNextRefresh = &t + i.Unlock() + return + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Warn().Msgf("unable to fetch mastodon instance list: %s", err) - } else { - // it worked - mi := new(MastodonIndexResponse) - err = json.NewDecoder(resp.Body).Decode(&mi) - if err != nil { - log.Warn().Msgf("unable to parse mastodon instance list: %s", err) - } else { - for _, instance := range mi.Instances { - i.addInstance(instance.Name) - } - - t := time.Now() - i.Lock() - i.mastodonIndexLastRefresh = &t - i.Unlock() - } + log.Error().Msgf("unable to fetch mastodon instance list: %s", err) + t := time.Now().Add(INDEX_ERROR_INTERVAL) + i.Lock() + i.mastodonIndexNextRefresh = &t + i.Unlock() + return } + + mi := new(MastodonIndexResponse) + err = json.Unmarshal(body, &mi) + if err != nil { + log.Error().Msgf("unable to parse mastodon instance list: %s", err) + t := time.Now().Add(INDEX_ERROR_INTERVAL) + i.Lock() + i.mastodonIndexNextRefresh = &t + i.Unlock() + return + } + + for _, instance := range mi.Instances { + i.addInstance(instance.Name) + } + + t := time.Now().Add(INDEX_CHECK_INTERVAL) + i.Lock() + i.mastodonIndexNextRefresh = &t + i.Unlock() } func (i *InstanceLocator) locatePleroma() { - var netClient = &http.Client{ - Timeout: NODE_TIMEOUT, + var c = &http.Client{ + Timeout: INDEX_API_TIMEOUT, } - resp, err := netClient.Get(pleromaIndexUrl) + resp, err := c.Get(pleromaIndexUrl) + if err != nil { - log.Warn().Msgf("unable to fetch pleroma instance list: %s", err) - } else { - // fetch worked - pi := new(PleromaIndexResponse) - err = json.NewDecoder(resp.Body).Decode(&pi) - if err != nil { - log.Warn().Msgf("unable to parse pleroma instance list: %s", err) - } else { - for _, instance := range *pi { - i.addInstance(instance.Domain) - } - t := time.Now() - i.Lock() - i.pleromaIndexLastRefresh = &t - i.Unlock() - } + log.Error().Msgf("unable to fetch pleroma instance list: %s", err) + t := time.Now().Add(INDEX_ERROR_INTERVAL) + i.Lock() + i.pleromaIndexNextRefresh = &t + i.Unlock() + return } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + log.Error().Msgf("unable to fetch pleroma instance list: %s", err) + t := time.Now().Add(INDEX_ERROR_INTERVAL) + i.Lock() + i.pleromaIndexNextRefresh = &t + i.Unlock() + return + } + + // fetch worked + pi := new(PleromaIndexResponse) + err = json.Unmarshal(body, &pi) + if err != nil { + log.Warn().Msgf("unable to parse pleroma instance list: %s", err) + t := time.Now().Add(INDEX_ERROR_INTERVAL) + i.Lock() + i.pleromaIndexNextRefresh = &t + i.Unlock() + return + } + + for _, instance := range *pi { + i.addInstance(instance.Domain) + } + t := time.Now().Add(INDEX_CHECK_INTERVAL) + i.Lock() + i.pleromaIndexNextRefresh = &t + i.Unlock() }