package feta import "encoding/json" import "io/ioutil" import "net/http" import "time" import "sync" import "github.com/rs/zerolog/log" const INDEX_API_TIMEOUT = time.Second * 60 var USER_AGENT = "https://github.com/sneak/feta indexer bot; sneak@sneak.berlin for feedback" // INDEX_CHECK_INTERVAL defines the interval for downloading new lists from // the index APIs run by mastodon/pleroma (default: 1h) var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 // INDEX_ERROR_INTERVAL is used for when the index fetch/parse fails // (default: 10m) var INDEX_ERROR_INTERVAL = time.Second * 60 * 10 // LOG_REPORT_INTERVAL defines how long between logging internal // stats/reporting for user supervision var LOG_REPORT_INTERVAL = time.Second * 60 const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" type InstanceLocator struct { pleromaIndexNextRefresh *time.Time mastodonIndexNextRefresh *time.Time mastodonIndexFetchLock sync.Mutex pleromaIndexFetchLock sync.Mutex reportInstanceVia chan InstanceHostname sync.Mutex } func NewInstanceLocator() *InstanceLocator { i := new(InstanceLocator) n := time.Now() i.pleromaIndexNextRefresh = &n i.mastodonIndexNextRefresh = &n return i } func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) { self.Lock() defer self.Unlock() self.reportInstanceVia = via } func (self *InstanceLocator) addInstance(hostname InstanceHostname) { // receiver (InstanceManager) is responsible for de-duping against its // map self.reportInstanceVia <- hostname } func (self *InstanceLocator) Locate() { log.Info().Msg("InstanceLocator starting") x := time.Now() for { log.Info().Msg("InstanceLocator tick") go func() { self.pleromaIndexFetchLock.Lock() if self.pleromaIndexNextRefresh.Before(time.Now()) { self.locatePleroma() } self.pleromaIndexFetchLock.Unlock() }() go func() { self.mastodonIndexFetchLock.Lock() if self.mastodonIndexNextRefresh.Before(time.Now()) { self.locateMastodon() } self.mastodonIndexFetchLock.Unlock() }() time.Sleep(1 * time.Second) if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) { x = time.Now() log.Debug(). Str("nextMastodonIndexFetch", self.mastodonIndexNextRefresh.Format(time.RFC3339)). Send() log.Debug(). Str("nextPleromaIndexFetch", self.pleromaIndexNextRefresh.Format(time.RFC3339)). Send() } } } func (self *InstanceLocator) locateMastodon() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } req, err := http.NewRequest("GET", mastodonIndexUrl, nil) if err != nil { panic(err) } req.Header.Set("User-Agent", USER_AGENT) resp, err := c.Do(req) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } else { log.Info(). Msg("fetched mastodon index") } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } mi := new(MastodonIndexResponse) err = json.Unmarshal(body, &mi) if err != nil { log.Error().Msgf("unable to parse mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } for _, instance := range mi.Instances { self.addInstance(InstanceHostname(instance.Name)) } t := time.Now().Add(INDEX_CHECK_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() } func (self *InstanceLocator) locatePleroma() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } req, err := http.NewRequest("GET", pleromaIndexUrl, nil) if err != nil { panic(err) } req.Header.Set("User-Agent", USER_AGENT) resp, err := c.Do(req) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } // fetch worked pi := new(PleromaIndexResponse) err = json.Unmarshal(body, &pi) if err != nil { log.Warn().Msgf("unable to parse pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } for _, instance := range *pi { self.addInstance(InstanceHostname(instance.Domain)) } t := time.Now().Add(INDEX_CHECK_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() }