package feta import "encoding/json" import "fmt" import "io/ioutil" import "net/http" import "strings" import "sync" import "time" import "errors" //import "github.com/gin-gonic/gin" import "github.com/looplab/fsm" import "github.com/rs/zerolog/log" const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" const instanceNodeinfoTimeout = time.Second * 50 const instanceHTTPTimeout = time.Second * 120 const instanceSpiderInterval = time.Second * 120 const instanceErrorInterval = time.Second * 60 * 30 type instanceImplementation int const ( implUnknown instanceImplementation = iota implMastodon implPleroma ) type instance struct { structLock sync.Mutex tootDestination chan *toot errorCount uint successCount uint highestID int hostname string identified bool fetching bool implementation instanceImplementation backend *instanceBackend nextFetch time.Time nodeInfoURL string serverVersionString string serverImplementationString string fetchingLock sync.Mutex fsm *fsm.FSM fsmLock sync.Mutex } func newInstance(options ...func(i *instance)) *instance { i := new(instance) i.setNextFetchAfter(1 * time.Second) i.fsm = fsm.NewFSM( "STATUS_UNKNOWN", fsm.Events{ {Name: "BEGIN_NODEINFO_URL_FETCH", Src: []string{"STATUS_UNKNOWN"}, Dst: "FETCHING_NODEINFO_URL"}, {Name: "GOT_NODEINFO_URL", Src: []string{"FETCHING_NODEINFO_URL"}, Dst: "PRE_NODEINFO_FETCH"}, {Name: "BEGIN_NODEINFO_FETCH", Src: []string{"PRE_NODEINFO_FETCH"}, Dst: "FETCHING_NODEINFO"}, {Name: "GOT_NODEINFO", Src: []string{"FETCHING_NODEINFO"}, Dst: "READY_FOR_TOOTFETCH"}, {Name: "FETCH_TIME_REACHED", Src: []string{"READY_FOR_TOOTFETCH"}, Dst: "READY_AND_DUE_FETCH"}, {Name: "BEGIN_TOOT_FETCH", Src: []string{"READY_AND_DUE_FETCH"}, Dst: "FETCHING"}, {Name: "WEIRD_NODE_RESPONSE", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "WEIRD_NODE"}, {Name: "EARLY_FETCH_ERROR", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "EARLY_ERROR"}, {Name: "TOOT_FETCH_ERROR", Src: []string{"FETCHING"}, Dst: "TOOT_FETCH_ERROR"}, {Name: "TOOTS_FETCHED", Src: []string{"FETCHING"}, Dst: "READY_FOR_TOOTFETCH"}, }, fsm.Callbacks{ "enter_state": func(e *fsm.Event) { i.fsmEnterState(e) }, }, ) for _, opt := range options { opt(i) } return i } func (i *instance) Status() string { i.fsmLock.Lock() defer i.fsmLock.Unlock() return i.fsm.Current() } func (i *instance) setTootDestination(d chan *toot) { i.tootDestination = d } func (i *instance) Event(eventname string) { i.fsmLock.Lock() defer i.fsmLock.Unlock() i.fsm.Event(eventname) } func (i *instance) fsmEnterState(e *fsm.Event) { log.Debug(). Str("hostname", i.hostname). Str("state", e.Dst). Msg("instance changed state") } func (i *instance) Lock() { i.structLock.Lock() } func (i *instance) Unlock() { i.structLock.Unlock() } func (i *instance) bumpFetch() { i.Lock() defer i.Unlock() i.nextFetch = time.Now().Add(120 * time.Second) } func (i *instance) setNextFetchAfter(d time.Duration) { i.Lock() defer i.Unlock() i.nextFetch = time.Now().Add(d) } func (i *instance) Fetch() { i.fetchingLock.Lock() defer i.fetchingLock.Unlock() i.setNextFetchAfter(instanceErrorInterval) err := i.detectNodeTypeIfNecessary() if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msg("unable to fetch instance metadata") return } i.setNextFetchAfter(instanceSpiderInterval) log.Info().Msgf("i (%s) IS NOW READY FOR FETCH", i.hostname) } func (i *instance) dueForFetch() bool { // this just checks FSM state, the ticker must update it and do time // calcs if i.Status() == "READY_AND_DUE_FETCH" { return true } return false } func (i *instance) isNowPastFetchTime() bool { return time.Now().After(i.nextFetch) } // Tick is responsible for pushing idle instance records between states. // The instances will transition between states when doing stuff (e.g. // investigating, fetching, et c) as well. func (i *instance) Tick() { if i.Status() == "READY_FOR_TOOTFETCH" { if i.isNowPastFetchTime() { i.Event("FETCH_TIME_REACHED") } } else if i.Status() == "STATUS_UNKNOWN" { i.Fetch() } else if i.Status() == "READY_AND_DUE_FETCH" { i.fetchRecentToots() } } func (i *instance) nodeIdentified() bool { i.Lock() defer i.Unlock() if i.implementation > implUnknown { return true } return false } func (i *instance) detectNodeTypeIfNecessary() error { if !i.nodeIdentified() { return i.fetchNodeInfo() } return nil } func (i *instance) registerError() { i.Lock() defer i.Unlock() i.errorCount++ } func (i *instance) registerSuccess() { i.Lock() defer i.Unlock() i.successCount++ } func (i *instance) Up() bool { i.Lock() defer i.Unlock() return i.successCount > 0 } func (i *instance) fetchNodeInfoURL() error { url := fmt.Sprintf("https://%s/.well-known/nodeinfo", i.hostname) var c = &http.Client{ Timeout: instanceNodeinfoTimeout, } log.Debug(). Str("url", url). Str("hostname", i.hostname). Msg("fetching nodeinfo reference URL") i.Event("BEGIN_NODEINFO_URL_FETCH") resp, err := c.Get(url) if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msg("unable to fetch nodeinfo, node is down?") i.registerError() i.Event("EARLY_FETCH_ERROR") return err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msg("unable to read nodeinfo") i.registerError() i.Event("EARLY_FETCH_ERROR") return err } nir := new(nodeInfoWellKnownResponse) err = json.Unmarshal(body, &nir) if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msg("unable to parse nodeinfo, node is weird") i.registerError() i.Event("WEIRD_NODE_RESPONSE") return err } for _, item := range nir.Links { if item.Rel == nodeInfoSchemaVersionTwoName { log.Debug(). Str("hostname", i.hostname). Str("nodeinfourl", item.Href). Msg("success fetching url for nodeinfo") i.Lock() i.nodeInfoURL = item.Href i.Unlock() i.registerSuccess() i.Event("GOT_NODEINFO_URL") return nil } log.Debug(). Str("hostname", i.hostname). Str("item-rel", item.Rel). Str("item-href", item.Href). Msg("nodeinfo entry") } log.Error(). Str("hostname", i.hostname). Msg("incomplete nodeinfo") i.registerError() i.Event("WEIRD_NODE_RESPONSE") return errors.New("incomplete nodeinfo") } func (i *instance) fetchNodeInfo() error { err := i.fetchNodeInfoURL() if err != nil { return err } var c = &http.Client{ Timeout: instanceNodeinfoTimeout, } //FIXME make sure the nodeinfourl is on the same domain as the instance //hostname i.Lock() url := i.nodeInfoURL i.Unlock() i.Event("BEGIN_NODEINFO_FETCH") resp, err := c.Get(url) if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msgf("unable to fetch nodeinfo data") i.registerError() i.Event("EARLY_FETCH_ERROR") return err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Error(). Str("hostname", i.hostname). Err(err). Msgf("unable to read nodeinfo data") i.registerError() i.Event("EARLY_FETCH_ERROR") return err } ni := new(nodeInfoVersionTwoSchema) err = json.Unmarshal(body, &ni) if err != nil { log.Error(). Str("hostname", i.hostname). Err(err). Msgf("unable to parse nodeinfo") i.registerError() i.Event("WEIRD_NODE_RESPONSE") return err } log.Debug(). Str("serverVersion", ni.Software.Version). Str("software", ni.Software.Name). Str("hostname", i.hostname). Str("nodeInfoURL", i.nodeInfoURL). Msg("received nodeinfo from instance") i.Lock() i.serverVersionString = ni.Software.Version i.serverImplementationString = ni.Software.Name ni.Software.Name = strings.ToLower(ni.Software.Name) if ni.Software.Name == "pleroma" { log.Debug(). Str("hostname", i.hostname). Str("software", ni.Software.Name). Msg("detected server software") i.identified = true i.implementation = implPleroma i.Unlock() i.registerSuccess() i.Event("GOT_NODEINFO") return nil } else if ni.Software.Name == "mastodon" { log.Debug(). Str("hostname", i.hostname). Str("software", ni.Software.Name). Msg("detected server software") i.identified = true i.implementation = implMastodon i.Unlock() i.registerSuccess() i.Event("GOT_NODEINFO") return nil } else { log.Error(). Str("hostname", i.hostname). Str("software", ni.Software.Name). Msg("FIXME unknown server implementation") i.Unlock() i.registerError() i.Event("WEIRD_NODE_RESPONSE") return errors.New("unknown server implementation") } } func (i *instance) fetchRecentToots() error { // it turns out pleroma supports the mastodon api so we'll just use that // for everything for now url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostname) var c = &http.Client{ Timeout: instanceHTTPTimeout, } i.Event("BEGIN_TOOT_FETCH") // we set the interval now to the error interval regardless here as a // safety against bugs to avoid fetching too frequently by logic // bug. if the fetch is successful, we will conditionally re-update the // next fetch to now+successInterval. i.setNextFetchAfter(instanceErrorInterval) resp, err := c.Get(url) if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msgf("unable to fetch recent toots") i.registerError() i.Event("TOOT_FETCH_ERROR") return err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Debug(). Str("hostname", i.hostname). Err(err). Msgf("unable to read recent toots from response") i.registerError() i.Event("TOOT_FETCH_ERROR") return err } tootList := new(apTootList) err = json.Unmarshal(body, &tootList) if err != nil { log.Error(). Str("hostname", i.hostname). Err(err). Msgf("unable to parse recent toot list") i.registerError() i.Event("TOOT_FETCH_ERROR") fmt.Printf(string(body)) return err } log.Info(). Str("hostname", i.hostname). Int("tootCount", len(*tootList)). Msgf("got and parsed toots") i.registerSuccess() i.Event("TOOTS_FETCHED") for _, x := range *tootList { fmt.Printf("%s\n", x.Content) } panic("unimplemented") } /* func (i *PleromaBackend) fetchRecentToots() ([]byte, error) { //url := //fmt.Sprintf("https://%s/api/statuses/public_and_external_timeline.json?count=100", //i.hostname) return nil, nil } */