package main import "encoding/json" import "fmt" import "io/ioutil" import "net/http" import "sync" import "time" import "github.com/rs/zerolog/log" const INDEX_API_TIMEOUT = time.Second * 60 // check with indices only hourly var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 // check with indices after 10 mins if they failed var INDEX_ERROR_INTERVAL = time.Second * 60 * 10 const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" type InstanceLocator struct { pleromaIndexNextRefresh *time.Time mastodonIndexNextRefresh *time.Time instances map[string]*Instance sync.Mutex } func NewInstanceLocator() *InstanceLocator { i := new(InstanceLocator) i.instances = make(map[string]*Instance) n := time.Now() i.pleromaIndexNextRefresh = &n i.mastodonIndexNextRefresh = &n return i } func (i *InstanceLocator) addInstance(hostname string) { i.Lock() defer i.Unlock() // only add it if we haven't seen the hostname before if i.instances[hostname] == nil { log.Info().Str("hostname", hostname).Msgf("adding discovered instance") i.instances[hostname] = NewInstance(hostname) } } func (i *InstanceLocator) Locate() { x := 0 for { if i.pleromaIndexNextRefresh.Before(time.Now()) { i.locatePleroma() } if i.mastodonIndexNextRefresh.Before(time.Now()) { i.locateMastodon() } time.Sleep(1 * time.Second) x++ if x == 60 { x = 0 log.Debug(). Str("nextmastodonupdate", i.mastodonIndexNextRefresh.Format(time.RFC3339)). Send() log.Debug(). Str("nextpleromaupdate", i.pleromaIndexNextRefresh.Format(time.RFC3339)). Send() i.logInstanceReport() } } } func (i *InstanceLocator) logInstanceReport() { r := i.instanceReport() log.Info(). Uint("up", r.up). Uint("total", r.total). Uint("identified", r.identified). Msg("instance report") } type InstanceLocatorReport struct { up uint identified uint total uint } func (r *InstanceLocatorReport) String() string { return fmt.Sprintf("up=%d identified=%d total=%d", r.up, r.identified, r.total) } func (i *InstanceLocator) NumInstances() uint { return i.instanceReport().total } func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { i.Lock() defer i.Unlock() r := new(InstanceLocatorReport) r.total = uint(len(i.instances)) for _, elem := range i.instances { if elem.identified == true { r.identified = r.identified + 1 } if elem.status == InstanceStatusAlive { r.up = r.up + 1 } } return r } func (i *InstanceLocator) locateMastodon() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } resp, err := c.Get(mastodonIndexUrl) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) i.Lock() i.mastodonIndexNextRefresh = &t i.Unlock() return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) i.Lock() i.mastodonIndexNextRefresh = &t i.Unlock() return } mi := new(MastodonIndexResponse) err = json.Unmarshal(body, &mi) if err != nil { log.Error().Msgf("unable to parse mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) i.Lock() i.mastodonIndexNextRefresh = &t i.Unlock() return } for _, instance := range mi.Instances { i.addInstance(instance.Name) } t := time.Now().Add(INDEX_CHECK_INTERVAL) i.Lock() i.mastodonIndexNextRefresh = &t i.Unlock() } func (i *InstanceLocator) locatePleroma() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } resp, err := c.Get(pleromaIndexUrl) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) i.Lock() i.pleromaIndexNextRefresh = &t i.Unlock() return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) i.Lock() i.pleromaIndexNextRefresh = &t i.Unlock() return } // fetch worked pi := new(PleromaIndexResponse) err = json.Unmarshal(body, &pi) if err != nil { log.Warn().Msgf("unable to parse pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) i.Lock() i.pleromaIndexNextRefresh = &t i.Unlock() return } for _, instance := range *pi { i.addInstance(instance.Domain) } t := time.Now().Add(INDEX_CHECK_INTERVAL) i.Lock() i.pleromaIndexNextRefresh = &t i.Unlock() }