255 lines
6.0 KiB
Go
255 lines
6.0 KiB
Go
package feta
|
|
|
|
import "encoding/json"
|
|
import "io/ioutil"
|
|
import "net/http"
|
|
import "time"
|
|
import "sync"
|
|
|
|
import "github.com/rs/zerolog/log"
|
|
import "golang.org/x/sync/semaphore"
|
|
|
|
const INDEX_API_TIMEOUT = time.Second * 60
|
|
|
|
var USER_AGENT = "https://github.com/sneak/feta indexer bot; sneak@sneak.berlin for feedback"
|
|
|
|
// INDEX_CHECK_INTERVAL defines the interval for downloading new lists from
|
|
// the index APIs run by mastodon/pleroma (default: 1h)
|
|
var INDEX_CHECK_INTERVAL = time.Second * 60 * 60
|
|
|
|
// INDEX_ERROR_INTERVAL is used for when the index fetch/parse fails
|
|
// (default: 10m)
|
|
var INDEX_ERROR_INTERVAL = time.Second * 60 * 10
|
|
|
|
// LOG_REPORT_INTERVAL defines how long between logging internal
|
|
// stats/reporting for user supervision
|
|
var LOG_REPORT_INTERVAL = time.Second * 10
|
|
|
|
const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"
|
|
const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi"
|
|
|
|
type InstanceLocator struct {
|
|
pleromaIndexNextRefresh *time.Time
|
|
mastodonIndexNextRefresh *time.Time
|
|
reportInstanceVia chan InstanceHostname
|
|
sync.Mutex
|
|
}
|
|
|
|
func NewInstanceLocator() *InstanceLocator {
|
|
i := new(InstanceLocator)
|
|
n := time.Now()
|
|
i.pleromaIndexNextRefresh = &n
|
|
i.mastodonIndexNextRefresh = &n
|
|
return i
|
|
}
|
|
|
|
func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) {
|
|
self.Lock()
|
|
defer self.Unlock()
|
|
self.reportInstanceVia = via
|
|
}
|
|
|
|
func (self *InstanceLocator) addInstance(hostname InstanceHostname) {
|
|
// receiver (InstanceManager) is responsible for de-duping against its
|
|
// map, we just spray
|
|
self.reportInstanceVia <- hostname
|
|
}
|
|
|
|
func (self *InstanceLocator) mastodonIndexRefreshDue() bool {
|
|
return self.mastodonIndexNextRefresh.Before(time.Now())
|
|
}
|
|
|
|
func (self *InstanceLocator) durationUntilNextMastodonIndexRefresh() time.Duration {
|
|
return (time.Duration(-1) * time.Now().Sub(*self.mastodonIndexNextRefresh))
|
|
}
|
|
|
|
func (self *InstanceLocator) pleromaIndexRefreshDue() bool {
|
|
return self.pleromaIndexNextRefresh.Before(time.Now())
|
|
}
|
|
|
|
func (self *InstanceLocator) durationUntilNextPleromaIndexRefresh() time.Duration {
|
|
return (time.Duration(-1) * time.Now().Sub(*self.pleromaIndexNextRefresh))
|
|
}
|
|
|
|
func (self *InstanceLocator) Locate() {
|
|
log.Info().Msg("InstanceLocator starting")
|
|
x := time.Now()
|
|
var pleromaSemaphore = semaphore.NewWeighted(1)
|
|
var mastodonSemaphore = semaphore.NewWeighted(1)
|
|
|
|
for {
|
|
|
|
log.Info().Msg("InstanceLocator tick")
|
|
|
|
go func() {
|
|
if self.pleromaIndexRefreshDue() {
|
|
if !pleromaSemaphore.TryAcquire(1) {
|
|
return
|
|
}
|
|
self.locatePleroma()
|
|
pleromaSemaphore.Release(1)
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
if self.mastodonIndexRefreshDue() {
|
|
if !mastodonSemaphore.TryAcquire(1) {
|
|
return
|
|
}
|
|
self.locateMastodon()
|
|
mastodonSemaphore.Release(1)
|
|
}
|
|
}()
|
|
|
|
time.Sleep(1 * time.Second)
|
|
|
|
if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) {
|
|
x = time.Now()
|
|
log.Debug().
|
|
Str("nextMastodonIndexRefresh", self.durationUntilNextMastodonIndexRefresh().String()).
|
|
Msg("refresh countdown")
|
|
log.Debug().
|
|
Str("nextPleromaIndexRefresh", self.durationUntilNextPleromaIndexRefresh().String()).
|
|
Msg("refresh countdown")
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func (self *InstanceLocator) locateMastodon() {
|
|
var c = &http.Client{
|
|
Timeout: INDEX_API_TIMEOUT,
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", mastodonIndexUrl, nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
req.Header.Set("User-Agent", USER_AGENT)
|
|
|
|
resp, err := c.Do(req)
|
|
if err != nil {
|
|
log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
|
|
t := time.Now().Add(INDEX_ERROR_INTERVAL)
|
|
self.Lock()
|
|
self.mastodonIndexNextRefresh = &t
|
|
self.Unlock()
|
|
return
|
|
} else {
|
|
log.Info().
|
|
Msg("fetched mastodon index")
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
if err != nil {
|
|
log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
|
|
t := time.Now().Add(INDEX_ERROR_INTERVAL)
|
|
self.Lock()
|
|
self.mastodonIndexNextRefresh = &t
|
|
self.Unlock()
|
|
return
|
|
}
|
|
|
|
t := time.Now().Add(INDEX_CHECK_INTERVAL)
|
|
self.Lock()
|
|
self.mastodonIndexNextRefresh = &t
|
|
self.Unlock()
|
|
|
|
mi := new(MastodonIndexResponse)
|
|
err = json.Unmarshal(body, &mi)
|
|
if err != nil {
|
|
log.Error().Msgf("unable to parse mastodon instance list: %s", err)
|
|
t := time.Now().Add(INDEX_ERROR_INTERVAL)
|
|
self.Lock()
|
|
self.mastodonIndexNextRefresh = &t
|
|
self.Unlock()
|
|
return
|
|
}
|
|
|
|
hosts := make(map[string]bool)
|
|
x := 0
|
|
for _, instance := range mi.Instances {
|
|
hosts[instance.Name] = true
|
|
x++
|
|
}
|
|
log.Info().
|
|
Int("count", x).
|
|
Msg("received hosts from mastodon index")
|
|
|
|
for k, _ := range hosts {
|
|
self.addInstance(InstanceHostname(k))
|
|
}
|
|
|
|
}
|
|
|
|
func (self *InstanceLocator) locatePleroma() {
|
|
var c = &http.Client{
|
|
Timeout: INDEX_API_TIMEOUT,
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", pleromaIndexUrl, nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
req.Header.Set("User-Agent", USER_AGENT)
|
|
|
|
resp, err := c.Do(req)
|
|
|
|
if err != nil {
|
|
log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
|
|
t := time.Now().Add(INDEX_ERROR_INTERVAL)
|
|
self.Lock()
|
|
self.pleromaIndexNextRefresh = &t
|
|
self.Unlock()
|
|
return
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
if err != nil {
|
|
log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
|
|
t := time.Now().Add(INDEX_ERROR_INTERVAL)
|
|
self.Lock()
|
|
self.pleromaIndexNextRefresh = &t
|
|
self.Unlock()
|
|
return
|
|
}
|
|
|
|
// fetch worked
|
|
|
|
t := time.Now().Add(INDEX_CHECK_INTERVAL)
|
|
self.Lock()
|
|
self.pleromaIndexNextRefresh = &t
|
|
self.Unlock()
|
|
|
|
pi := new(PleromaIndexResponse)
|
|
err = json.Unmarshal(body, &pi)
|
|
if err != nil {
|
|
log.Warn().Msgf("unable to parse pleroma instance list: %s", err)
|
|
t := time.Now().Add(INDEX_ERROR_INTERVAL)
|
|
self.Lock()
|
|
self.pleromaIndexNextRefresh = &t
|
|
self.Unlock()
|
|
return
|
|
}
|
|
|
|
hosts := make(map[string]bool)
|
|
x := 0
|
|
for _, instance := range *pi {
|
|
hosts[instance.Domain] = true
|
|
x++
|
|
}
|
|
log.Info().
|
|
Int("count", x).
|
|
Msg("received hosts from pleroma index")
|
|
|
|
for k, _ := range hosts {
|
|
self.addInstance(InstanceHostname(k))
|
|
}
|
|
|
|
}
|