feta/locator.go

187 lines
4.4 KiB
Go
Raw Normal View History

2019-11-05 02:53:11 +00:00
package feta
2019-10-24 12:14:36 +00:00
2019-11-02 06:56:17 +00:00
import "encoding/json"
import "io/ioutil"
2019-11-02 06:56:17 +00:00
import "net/http"
import "time"
2019-11-03 13:17:00 +00:00
import "sync"
2019-11-02 06:56:17 +00:00
import "github.com/rs/zerolog/log"
2019-10-24 12:14:36 +00:00
const INDEX_API_TIMEOUT = time.Second * 60
2019-11-03 18:00:01 +00:00
var USER_AGENT = "https://github.com/sneak/feta indexer bot; sneak@sneak.berlin for feedback"
2019-10-24 15:23:29 +00:00
// check with indices only hourly
var INDEX_CHECK_INTERVAL = time.Second * 60 * 60
// check with indices after 10 mins if they failed
var INDEX_ERROR_INTERVAL = time.Second * 60 * 10
2019-11-03 13:17:00 +00:00
var LOG_REPORT_INTERVAL = time.Second * 60
const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"
2019-10-24 12:14:36 +00:00
const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi"
type InstanceLocator struct {
pleromaIndexNextRefresh *time.Time
mastodonIndexNextRefresh *time.Time
2019-11-03 13:17:00 +00:00
reportInstanceVia chan InstanceHostname
sync.Mutex
2019-10-24 12:14:36 +00:00
}
func NewInstanceLocator() *InstanceLocator {
i := new(InstanceLocator)
n := time.Now()
i.pleromaIndexNextRefresh = &n
i.mastodonIndexNextRefresh = &n
2019-10-24 12:14:36 +00:00
return i
}
2019-10-24 12:41:05 +00:00
2019-11-03 13:17:00 +00:00
func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) {
self.Lock()
defer self.Unlock()
self.reportInstanceVia = via
}
2019-11-03 18:00:01 +00:00
func (self *InstanceLocator) addInstance(hostname InstanceHostname) {
// receiver (InstanceManager) is responsible for de-duping against its
// map
self.reportInstanceVia <- hostname
}
2019-11-03 13:17:00 +00:00
func (self *InstanceLocator) Locate() {
2019-11-03 18:00:01 +00:00
log.Info().Msg("InstanceLocator starting")
2019-11-03 13:17:00 +00:00
x := time.Now()
for {
2019-11-03 18:00:01 +00:00
log.Info().Msg("InstanceLocator tick")
2019-11-03 13:17:00 +00:00
if self.pleromaIndexNextRefresh.Before(time.Now()) {
self.locatePleroma()
}
2019-11-03 13:17:00 +00:00
if self.mastodonIndexNextRefresh.Before(time.Now()) {
self.locateMastodon()
}
time.Sleep(1 * time.Second)
2019-11-03 13:17:00 +00:00
if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) {
x = time.Now()
log.Debug().
2019-11-03 13:17:00 +00:00
Str("nextMastodonIndexFetch", self.mastodonIndexNextRefresh.Format(time.RFC3339)).
Send()
log.Debug().
2019-11-03 13:17:00 +00:00
Str("nextPleromaIndexFetch", self.pleromaIndexNextRefresh.Format(time.RFC3339)).
Send()
}
}
}
2019-11-03 13:17:00 +00:00
func (self *InstanceLocator) locateMastodon() {
var c = &http.Client{
Timeout: INDEX_API_TIMEOUT,
2019-10-24 12:41:05 +00:00
}
2019-11-03 18:00:01 +00:00
req, err := http.NewRequest("GET", mastodonIndexUrl, nil)
if err != nil {
panic(err)
}
req.Header.Set("User-Agent", USER_AGENT)
resp, err := c.Do(req)
if err != nil {
log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.mastodonIndexNextRefresh = &t
self.Unlock()
return
}
2019-10-24 12:41:05 +00:00
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
2019-10-24 12:41:05 +00:00
if err != nil {
log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.mastodonIndexNextRefresh = &t
self.Unlock()
return
}
mi := new(MastodonIndexResponse)
err = json.Unmarshal(body, &mi)
if err != nil {
log.Error().Msgf("unable to parse mastodon instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.mastodonIndexNextRefresh = &t
self.Unlock()
return
}
for _, instance := range mi.Instances {
2019-11-03 18:00:01 +00:00
self.addInstance(InstanceHostname(instance.Name))
}
t := time.Now().Add(INDEX_CHECK_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.mastodonIndexNextRefresh = &t
self.Unlock()
}
2019-11-03 13:17:00 +00:00
func (self *InstanceLocator) locatePleroma() {
var c = &http.Client{
Timeout: INDEX_API_TIMEOUT,
}
2019-11-03 18:00:01 +00:00
req, err := http.NewRequest("GET", pleromaIndexUrl, nil)
if err != nil {
panic(err)
}
req.Header.Set("User-Agent", USER_AGENT)
resp, err := c.Do(req)
if err != nil {
log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.pleromaIndexNextRefresh = &t
self.Unlock()
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.pleromaIndexNextRefresh = &t
self.Unlock()
return
}
// fetch worked
pi := new(PleromaIndexResponse)
err = json.Unmarshal(body, &pi)
if err != nil {
log.Warn().Msgf("unable to parse pleroma instance list: %s", err)
t := time.Now().Add(INDEX_ERROR_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.pleromaIndexNextRefresh = &t
self.Unlock()
return
}
for _, instance := range *pi {
2019-11-03 18:00:01 +00:00
self.addInstance(InstanceHostname(instance.Domain))
}
t := time.Now().Add(INDEX_CHECK_INTERVAL)
2019-11-03 13:17:00 +00:00
self.Lock()
self.pleromaIndexNextRefresh = &t
self.Unlock()
2019-10-24 12:41:05 +00:00
}