feta/locator.go

package feta

import "encoding/json"
import "io/ioutil"
import "net/http"
import "time"
import "sync"

import "github.com/rs/zerolog/log"
import "golang.org/x/sync/semaphore"

const INDEX_API_TIMEOUT = time.Second * 60

var USER_AGENT = "https://github.com/sneak/feta indexer bot; sneak@sneak.berlin for feedback"

// INDEX_CHECK_INTERVAL defines the interval for downloading new lists from
// the index APIs run by mastodon/pleroma (default: 1h)
var INDEX_CHECK_INTERVAL = time.Second * 60 * 60

// INDEX_ERROR_INTERVAL is used for when the index fetch/parse fails
// (default: 10m)
var INDEX_ERROR_INTERVAL = time.Second * 60 * 10

// LOG_REPORT_INTERVAL defines how long between logging internal
// stats/reporting for user supervision
var LOG_REPORT_INTERVAL = time.Second * 60

const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"
const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi"

type InstanceLocator struct {
	pleromaIndexNextRefresh  *time.Time
	mastodonIndexNextRefresh *time.Time
	reportInstanceVia        chan InstanceHostname
	sync.Mutex
}

func NewInstanceLocator() *InstanceLocator {
	i := new(InstanceLocator)
	n := time.Now()
	i.pleromaIndexNextRefresh = &n
	i.mastodonIndexNextRefresh = &n
	return i
}

func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) {
	self.Lock()
	defer self.Unlock()
	self.reportInstanceVia = via
}

func (self *InstanceLocator) addInstance(hostname InstanceHostname) {
	// receiver (InstanceManager) is responsible for de-duping against its
	// map, we just spray
	self.reportInstanceVia <- hostname
}

func (self *InstanceLocator) Locate() {
	log.Info().Msg("InstanceLocator starting")
	x := time.Now()
	var pleromaSemaphore = semaphore.NewWeighted(1)
	var mastodonSemaphore = semaphore.NewWeighted(1)

	for {

		log.Info().Msg("InstanceLocator tick")

		go func() {
			if self.pleromaIndexNextRefresh.Before(time.Now()) {
				if !pleromaSemaphore.TryAcquire(1) {
					return
				}
				self.locatePleroma()
				pleromaSemaphore.Release(1)
			}
		}()

		go func() {
			if self.mastodonIndexNextRefresh.Before(time.Now()) {
				if !mastodonSemaphore.TryAcquire(1) {
					return
				}
				self.locateMastodon()
				mastodonSemaphore.Release(1)
			}
		}()

		time.Sleep(1 * time.Second)

		if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) {
			x = time.Now()
			log.Debug().
				Str("nextMastodonIndexFetch", time.Now().Sub(*self.mastodonIndexNextRefresh).String()).
				Send()
			log.Debug().
				Str("nextMastodonIndexFetch", time.Now().Sub(*self.pleromaIndexNextRefresh).String()).
				Send()
		}
	}

}

func (self *InstanceLocator) locateMastodon() {
	var c = &http.Client{
		Timeout: INDEX_API_TIMEOUT,
	}

	req, err := http.NewRequest("GET", mastodonIndexUrl, nil)
	if err != nil {
		panic(err)
	}

	req.Header.Set("User-Agent", USER_AGENT)

	resp, err := c.Do(req)
	if err != nil {
		log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.mastodonIndexNextRefresh = &t
		self.Unlock()
		return
	} else {
		log.Info().
			Msg("fetched mastodon index")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Error().Msgf("unable to fetch mastodon instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.mastodonIndexNextRefresh = &t
		self.Unlock()
		return
	}

	t := time.Now().Add(INDEX_CHECK_INTERVAL)
	self.Lock()
	self.mastodonIndexNextRefresh = &t
	self.Unlock()

	mi := new(MastodonIndexResponse)
	err = json.Unmarshal(body, &mi)
	if err != nil {
		log.Error().Msgf("unable to parse mastodon instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.mastodonIndexNextRefresh = &t
		self.Unlock()
		return
	}

	hosts := make(map[string]bool)
	x := 0
	for _, instance := range mi.Instances {
		hosts[instance.Name] = true
		x++
	}
	log.Info().
		Int("count", x).
		Msg("received hosts from mastodon index")

	for k, _ := range hosts {
		self.addInstance(InstanceHostname(k))
	}

}

func (self *InstanceLocator) locatePleroma() {
	var c = &http.Client{
		Timeout: INDEX_API_TIMEOUT,
	}

	req, err := http.NewRequest("GET", pleromaIndexUrl, nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set("User-Agent", USER_AGENT)

	resp, err := c.Do(req)

	if err != nil {
		log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.pleromaIndexNextRefresh = &t
		self.Unlock()
		return
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Error().Msgf("unable to fetch pleroma instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.pleromaIndexNextRefresh = &t
		self.Unlock()
		return
	}

	// fetch worked

	t := time.Now().Add(INDEX_CHECK_INTERVAL)
	self.Lock()
	self.pleromaIndexNextRefresh = &t
	self.Unlock()

	pi := new(PleromaIndexResponse)
	err = json.Unmarshal(body, &pi)
	if err != nil {
		log.Warn().Msgf("unable to parse pleroma instance list: %s", err)
		t := time.Now().Add(INDEX_ERROR_INTERVAL)
		self.Lock()
		self.pleromaIndexNextRefresh = &t
		self.Unlock()
		return
	}

	hosts := make(map[string]bool)
	x := 0
	for _, instance := range *pi {
		hosts[instance.Domain] = true
		x++
	}
	log.Info().
		Int("count", x).
		Msg("received hosts from pleroma index")

	for k, _ := range hosts {
		self.addInstance(InstanceHostname(k))
	}

}