package main import "encoding/json" import "fmt" import "io/ioutil" import "net/http" import "time" import "sync" import "github.com/rs/zerolog/log" const INDEX_API_TIMEOUT = time.Second * 60 // check with indices only hourly var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 // check with indices after 10 mins if they failed var INDEX_ERROR_INTERVAL = time.Second * 60 * 10 var LOG_REPORT_INTERVAL = time.Second * 60 const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" type InstanceLocator struct { pleromaIndexNextRefresh *time.Time mastodonIndexNextRefresh *time.Time reportInstanceVia chan InstanceHostname instances map[string]*Instance sync.Mutex } func NewInstanceLocator() *InstanceLocator { i := new(InstanceLocator) i.instances = make(map[string]*Instance) n := time.Now() i.pleromaIndexNextRefresh = &n i.mastodonIndexNextRefresh = &n return i } func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) { self.Lock() defer self.Unlock() self.reportInstanceVia = via } func (self *InstanceLocator) addInstance(hostname string) { self.Lock() defer self.Unlock() // only add it if we haven't seen the hostname before if self.instances[hostname] == nil { log.Info().Str("hostname", hostname).Msgf("adding discovered instance") self.instances[hostname] = NewInstance(hostname) } } func (self *InstanceLocator) Locate() { x := time.Now() for { if self.pleromaIndexNextRefresh.Before(time.Now()) { self.locatePleroma() } if self.mastodonIndexNextRefresh.Before(time.Now()) { self.locateMastodon() } time.Sleep(1 * time.Second) if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) { x = time.Now() log.Debug(). Str("nextMastodonIndexFetch", self.mastodonIndexNextRefresh.Format(time.RFC3339)). Send() log.Debug(). Str("nextPleromaIndexFetch", self.pleromaIndexNextRefresh.Format(time.RFC3339)). Send() self.logInstanceReport() } } } func (self *InstanceLocator) logInstanceReport() { r := self.instanceReport() log.Info(). Uint("up", r.up). Uint("total", r.total). Uint("identified", r.identified). Msg("instance report") } type InstanceLocatorReport struct { up uint identified uint total uint } func (r *InstanceLocatorReport) String() string { return fmt.Sprintf("up=%d identified=%d total=%d", r.up, r.identified, r.total) } func (self *InstanceLocator) NumInstances() uint { return self.instanceReport().total } func (self *InstanceLocator) instanceReport() *InstanceLocatorReport { self.Lock() defer self.Unlock() r := new(InstanceLocatorReport) r.total = uint(len(self.instances)) for _, elem := range self.instances { if elem.identified == true { r.identified = r.identified + 1 } if elem.status == InstanceStatusAlive { r.up = r.up + 1 } } return r } func (self *InstanceLocator) locateMastodon() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } resp, err := c.Get(mastodonIndexUrl) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } mi := new(MastodonIndexResponse) err = json.Unmarshal(body, &mi) if err != nil { log.Error().Msgf("unable to parse mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } for _, instance := range mi.Instances { self.addInstance(instance.Name) } t := time.Now().Add(INDEX_CHECK_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() } func (self *InstanceLocator) locatePleroma() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } resp, err := c.Get(pleromaIndexUrl) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } // fetch worked pi := new(PleromaIndexResponse) err = json.Unmarshal(body, &pi) if err != nil { log.Warn().Msgf("unable to parse pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } for _, instance := range *pi { self.addInstance(instance.Domain) } t := time.Now().Add(INDEX_CHECK_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() }