package feta import "encoding/json" import "io/ioutil" import "net/http" import "time" import "sync" import "github.com/rs/zerolog/log" import "golang.org/x/sync/semaphore" const INDEX_API_TIMEOUT = time.Second * 60 var USER_AGENT = "https://github.com/sneak/feta indexer bot; sneak@sneak.berlin for feedback" // INDEX_CHECK_INTERVAL defines the interval for downloading new lists from // the index APIs run by mastodon/pleroma (default: 1h) var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 // INDEX_ERROR_INTERVAL is used for when the index fetch/parse fails // (default: 10m) var INDEX_ERROR_INTERVAL = time.Second * 60 * 10 // LOG_REPORT_INTERVAL defines how long between logging internal // stats/reporting for user supervision var LOG_REPORT_INTERVAL = time.Second * 10 const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" type InstanceLocator struct { pleromaIndexNextRefresh *time.Time mastodonIndexNextRefresh *time.Time reportInstanceVia chan InstanceHostname sync.Mutex } func NewInstanceLocator() *InstanceLocator { i := new(InstanceLocator) n := time.Now() i.pleromaIndexNextRefresh = &n i.mastodonIndexNextRefresh = &n return i } func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) { self.Lock() defer self.Unlock() self.reportInstanceVia = via } func (self *InstanceLocator) addInstance(hostname InstanceHostname) { // receiver (InstanceManager) is responsible for de-duping against its // map, we just spray self.reportInstanceVia <- hostname } func (self *InstanceLocator) mastodonIndexRefreshDue() bool { return self.mastodonIndexNextRefresh.Before(time.Now()) } func (self *InstanceLocator) durationUntilNextMastodonIndexRefresh() time.Duration { return (time.Duration(-1) * time.Now().Sub(*self.mastodonIndexNextRefresh)) } func (self *InstanceLocator) pleromaIndexRefreshDue() bool { return self.pleromaIndexNextRefresh.Before(time.Now()) } func (self *InstanceLocator) durationUntilNextPleromaIndexRefresh() time.Duration { return (time.Duration(-1) * time.Now().Sub(*self.pleromaIndexNextRefresh)) } func (self *InstanceLocator) Locate() { log.Info().Msg("InstanceLocator starting") x := time.Now() var pleromaSemaphore = semaphore.NewWeighted(1) var mastodonSemaphore = semaphore.NewWeighted(1) for { log.Info().Msg("InstanceLocator tick") go func() { if self.pleromaIndexRefreshDue() { if !pleromaSemaphore.TryAcquire(1) { return } self.locatePleroma() pleromaSemaphore.Release(1) } }() go func() { if self.mastodonIndexRefreshDue() { if !mastodonSemaphore.TryAcquire(1) { return } self.locateMastodon() mastodonSemaphore.Release(1) } }() time.Sleep(1 * time.Second) if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) { x = time.Now() log.Debug(). Str("nextMastodonIndexRefresh", self.durationUntilNextMastodonIndexRefresh().String()). Msg("refresh countdown") log.Debug(). Str("nextPleromaIndexRefresh", self.durationUntilNextPleromaIndexRefresh().String()). Msg("refresh countdown") } } } func (self *InstanceLocator) locateMastodon() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } req, err := http.NewRequest("GET", mastodonIndexUrl, nil) if err != nil { panic(err) } req.Header.Set("User-Agent", USER_AGENT) resp, err := c.Do(req) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } else { log.Info(). Msg("fetched mastodon index") } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } t := time.Now().Add(INDEX_CHECK_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() mi := new(MastodonIndexResponse) err = json.Unmarshal(body, &mi) if err != nil { log.Error().Msgf("unable to parse mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.mastodonIndexNextRefresh = &t self.Unlock() return } hosts := make(map[string]bool) x := 0 for _, instance := range mi.Instances { hosts[instance.Name] = true x++ } log.Info(). Int("count", x). Msg("received hosts from mastodon index") for k, _ := range hosts { self.addInstance(InstanceHostname(k)) } } func (self *InstanceLocator) locatePleroma() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } req, err := http.NewRequest("GET", pleromaIndexUrl, nil) if err != nil { panic(err) } req.Header.Set("User-Agent", USER_AGENT) resp, err := c.Do(req) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } // fetch worked t := time.Now().Add(INDEX_CHECK_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() pi := new(PleromaIndexResponse) err = json.Unmarshal(body, &pi) if err != nil { log.Warn().Msgf("unable to parse pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) self.Lock() self.pleromaIndexNextRefresh = &t self.Unlock() return } hosts := make(map[string]bool) x := 0 for _, instance := range *pi { hosts[instance.Domain] = true x++ } log.Info(). Int("count", x). Msg("received hosts from pleroma index") for k, _ := range hosts { self.addInstance(InstanceHostname(k)) } }