feta/locator.go

package main

import "encoding/json"
import "fmt"
import "net/http"
import "sync"
import "time"

import "github.com/rs/zerolog/log"

const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"

var foreverago = time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)

// check with indices only hourly
var INDEX_CHECK_INTERVAL = time.Second * 60 * 60

// thank fuck for https://mholt.github.io/json-to-go/ otherwise
// this would have been a giant pain in the dick
type MastodonIndexResponse struct {
	Instances []struct {
		ID                 string      `json:"_id"`
		AddedAt            time.Time   `json:"addedAt"`
		Name               string      `json:"name"`
		Downchecks         int         `json:"downchecks"`
		Upchecks           int         `json:"upchecks"`
		HTTPSRank          interface{} `json:"https_rank"`
		HTTPSScore         int         `json:"https_score"`
		ObsRank            string      `json:"obs_rank"`
		ObsScore           int         `json:"obs_score"`
		Ipv6               bool        `json:"ipv6"`
		Up                 bool        `json:"up"`
		Users              int         `json:"users"`
		Statuses           string      `json:"statuses"`
		Connections        int         `json:"connections"`
		OpenRegistrations  bool        `json:"openRegistrations"`
		Uptime             float64     `json:"uptime"`
		Version            string      `json:"version"`
		VersionScore       int         `json:"version_score"`
		UpdatedAt          time.Time   `json:"updatedAt"`
		CheckedAt          time.Time   `json:"checkedAt"`
		Dead               bool        `json:"dead"`
		ObsDate            time.Time   `json:"obs_date"`
		Second60           int         `json:"second60"`
		Second             int         `json:"second"`
		ActiveUserCount    interface{} `json:"active_user_count,omitempty"`
		FirstUserCreatedAt interface{} `json:"first_user_created_at,omitempty"`
		Thumbnail          string      `json:"thumbnail"`
		ApUpdatedAt        time.Time   `json:"apUpdatedAt"`
		Second5            int         `json:"second5"`
		RawVersion         string      `json:"raw_version"`
		ActivityPrevw      struct {
			Statuses      int `json:"statuses"`
			Logins        int `json:"logins"`
			Registrations int `json:"registrations"`
		} `json:"activity_prevw,omitempty"`
		Mastodon  bool   `json:"mastodon"`
		UptimeStr string `json:"uptime_str"`
		Score     int    `json:"score"`
		ScoreStr  string `json:"score_str"`
	} `json:"instances"`
}

const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi"

type PleromaIndexResponse []struct {
	Domain                    string `json:"domain"`
	Title                     string `json:"title"`
	Thumbnail                 string `json:"thumbnail"`
	Registration              bool   `json:"registration"`
	Chat                      bool   `json:"chat"`
	Gopher                    bool   `json:"gopher"`
	WhoToFollow               bool   `json:"who_to_follow"`
	MediaProxy                bool   `json:"media_proxy"`
	ScopeOptions              bool   `json:"scope_options"`
	AccountActivationRequired bool   `json:"account_activation_required"`
	TextLimit                 int    `json:"text_limit"`
}

type InstanceLocator struct {
	pleromaIndexLastRefresh  *time.Time
	mastodonIndexLastRefresh *time.Time
	instances                map[string]*Instance
	sync.Mutex
}

func NewInstanceLocator() *InstanceLocator {
	i := new(InstanceLocator)
	i.instances = make(map[string]*Instance)
	i.pleromaIndexLastRefresh = &foreverago
	i.mastodonIndexLastRefresh = &foreverago
	return i
}

func (i *InstanceLocator) addInstance(hostname string) {
	i.Lock()
	defer i.Unlock()
	// only add it if we haven't seen the hostname before
	if i.instances[hostname] == nil {
		log.Debug().Str("hostname", hostname).Msgf("adding discovered instance")
		i.instances[hostname] = NewInstance(hostname)
	}

}

func (i *InstanceLocator) Locate() {
	log.Debug().
		Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)).
		Send()

	log.Debug().
		Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)).
		Send()

	i.locateMastodon()

	i.locatePleroma()

	time.Sleep(120 * time.Second)

	r := i.instanceReport()

	log.Debug().
		Uint("up", r.up).
		Uint("total", r.total).
		Uint("identified", r.identified).
		Msg("instance report")

}

type InstanceLocatorReport struct {
	up         uint
	identified uint
	total      uint
}

func (r *InstanceLocatorReport) String() string {
	return fmt.Sprintf("up=%d identified=%d total=%d", r.up, r.identified, r.total)
}

func (i *InstanceLocator) NumInstances() uint {
	return i.instanceReport().total
}

func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
	i.Lock()
	defer i.Unlock()
	r := new(InstanceLocatorReport)

	r.total = uint(len(i.instances))

	for _, elem := range i.instances {
		if elem.identified == true {
			r.identified = r.identified + 1
		}

		if elem.up == true {
			r.up = r.up + 1
		}
	}

	return r
}

func (i *InstanceLocator) locateMastodon() {
	var netClient = &http.Client{
		Timeout: NODE_TIMEOUT,
	}
	resp, err := netClient.Get(mastodonIndexUrl)
	defer resp.Body.Close()

	if err != nil {
		log.Warn().Msgf("unable to fetch mastodon instance list: %s", err)
	} else {
		// it worked
		mi := new(MastodonIndexResponse)
		err = json.NewDecoder(resp.Body).Decode(&mi)
		if err != nil {
			log.Warn().Msgf("unable to parse mastodon instance list: %s", err)
		} else {
			for _, instance := range mi.Instances {
				i.addInstance(instance.Name)
			}

			t := time.Now()
			i.Lock()
			i.mastodonIndexLastRefresh = &t
			i.Unlock()
		}
	}
}

func (i *InstanceLocator) locatePleroma() {
	var netClient = &http.Client{
		Timeout: NODE_TIMEOUT,
	}
	resp, err := netClient.Get(pleromaIndexUrl)
	if err != nil {
		log.Warn().Msgf("unable to fetch pleroma instance list: %s", err)
	} else {
		// fetch worked
		pi := new(PleromaIndexResponse)
		err = json.NewDecoder(resp.Body).Decode(&pi)
		if err != nil {
			log.Warn().Msgf("unable to parse pleroma instance list: %s", err)
		} else {
			for _, instance := range *pi {
				i.addInstance(instance.Domain)
			}
			t := time.Now()
			i.Lock()
			i.pleromaIndexLastRefresh = &t
			i.Unlock()
		}
	}
}