feta/locator.go

package feta

import "encoding/json"
import "io/ioutil"
import "net/http"
import "time"
import "sync"

import "github.com/rs/zerolog/log"

const INDEX_API_TIMEOUT = time.Second * 60

var USER_AGENT = "https://github.com/sneak/feta indexer bot; sneak@sneak.berlin for feedback"

// INDEX_CHECK_INTERVAL defines the interval for downloading new lists from
// the index APIs run by mastodon/pleroma (default: 1h)
var INDEX_CHECK_INTERVAL = time.Second * 60 * 60

// INDEX_ERROR_INTERVAL is used for when the index fetch/parse fails
// (default: 10m)
var INDEX_ERROR_INTERVAL = time.Second * 60 * 10

// LOG_REPORT_INTERVAL defines how long between logging internal
// stats/reporting for user supervision
var LOG_REPORT_INTERVAL = time.Second * 60

const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"
const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi"

type InstanceLocator struct {
	pleromaIndexNextRefresh  *time.Time
	mastodonIndexNextRefresh *time.Time
	mastodonIndexFetchLock   sync.Mutex
	pleromaIndexFetchLock    sync.Mutex
	reportInstanceVia        chan InstanceHostname
	sync.Mutex
}

func NewInstanceLocator() *InstanceLocator {
	i := new(InstanceLocator)
	n := time.Now()
	i.pleromaIndexNextRefresh = &n
	i.mastodonIndexNextRefresh = &n
	return i
}

func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) {
	self.Lock()
	defer self.Unlock()
	self.reportInstanceVia = via
}

func (self *InstanceLocator) addInstance(hostname InstanceHostname) {
	// receiver (InstanceManager) is responsible for de-duping against its
	// map
	self.reportInstanceVia <- hostname
}

func (self *InstanceLocator) Locate() {
	log.Info().Msg("InstanceLocator starting")
	x := time.Now()
	for {
		log.Info().Msg("InstanceLocator tick")
		go func() {
			self.pleromaIndexFetchLock.Lock()
			if self.pleromaIndexNextRefresh.Before(time.Now()) {
				self.locatePleroma()
			}
			self.pleromaIndexFetchLock.Unlock()
		}()
		go func() {
			self.mastodonIndexFetchLock.Lock()
			if self.mastodonIndexNextRefresh.Before(time.Now()) {
				self.locateMastodon()
			}
			self.mastodonIndexFetchLock.Unlock()
		}()
		time.Sleep(1 * time.Second)
		if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) {
			x = time.Now()
			log.Debug().
				Str("nextMastodonIndexFetch", self.mastodonIndexNextRefresh.Format(time.RFC3339)).
				Send()
			log.Debug().
				Str("nextPleromaIndexFetch", self.pleromaIndexNextRefresh.Format(time.RFC3339)).
				Send()
		}
	}

}

func (self *InstanceLocator) locateMastodon() {
	var c = &http.Client{
		Timeout: INDEX_API_TIMEOUT,
	}

	req, err := http.NewRequest("GET", mastodonIndexUrl, nil)
	if err != nil {
		panic(err)
	}

	req.Header.Set("User-Agent", USER_AGENT)

	resp, err := c.Do(req)
	if err != nil {
		log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.mastodonIndexNextRefresh = &t
		self.Unlock()
		return
	} else {
		log.Info().
			Msg("fetched mastodon index")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.mastodonIndexNextRefresh = &t
		self.Unlock()
		return
	}

	mi := new(MastodonIndexResponse)
	err = json.Unmarshal(body, &mi)
	if err != nil {
		log.Error().Msgf("unable to parse mastodon instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.mastodonIndexNextRefresh = &t
		self.Unlock()
		return
	}

	for _, instance := range mi.Instances {
		self.addInstance(InstanceHostname(instance.Name))
	}

	t := time.Now().Add(INDEX_CHECK_INTERVAL)
	self.Lock()
	self.mastodonIndexNextRefresh = &t
	self.Unlock()
}

func (self *InstanceLocator) locatePleroma() {
	var c = &http.Client{
		Timeout: INDEX_API_TIMEOUT,
	}

	req, err := http.NewRequest("GET", pleromaIndexUrl, nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("User-Agent", USER_AGENT)

	resp, err := c.Do(req)

	if err != nil {
		log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.pleromaIndexNextRefresh = &t
		self.Unlock()
		return
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.pleromaIndexNextRefresh = &t
		self.Unlock()
		return
	}

	// fetch worked
	pi := new(PleromaIndexResponse)
	err = json.Unmarshal(body, &pi)
	if err != nil {
		log.Warn().Msgf("unable to parse pleroma instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.pleromaIndexNextRefresh = &t
		self.Unlock()
		return
	}

	for _, instance := range *pi {
		self.addInstance(InstanceHostname(instance.Domain))
	}
	t := time.Now().Add(INDEX_CHECK_INTERVAL)
	self.Lock()
	self.pleromaIndexNextRefresh = &t
	self.Unlock()
}