From 58954151b5ae60bf0b66abe7d00b5fa5f755a70d Mon Sep 17 00:00:00 2001 From: Jeffrey Paul Date: Sun, 3 Nov 2019 05:17:00 -0800 Subject: [PATCH] checkpoint before major refactoring --- webserver.go => api.go | 10 +-- archiver.go | 27 +++++-- instance.go | 9 ++- locator.go => instancelocator.go | 118 +++++++++++++++++-------------- instancemanager.go | 34 +++++++++ 5 files changed, 129 insertions(+), 69 deletions(-) rename webserver.go => api.go (86%) rename locator.go => instancelocator.go (62%) create mode 100644 instancemanager.go diff --git a/webserver.go b/api.go similarity index 86% rename from webserver.go rename to api.go index f8e2e44..561d429 100644 --- a/webserver.go +++ b/api.go @@ -48,19 +48,19 @@ func (a *TootArchiverAPIServer) getRouter() *gin.Engine { }) r.GET("/", func(c *gin.Context) { + ir := a.archiver.locator.instanceReport() c.JSON(200, gin.H{ // FIXME(sneak) add more stuff here "status": "ok", "now": time.Now().UTC().Format(time.RFC3339), "uptime": a.archiver.Uptime().String(), - "instances": gin.H{ - "total": a.archiver.locator.instanceReport().total, - "up": a.archiver.locator.instanceReport().up, - "identified": a.archiver.locator.instanceReport().identified, + "instanceSummary": gin.H{ + "total": ir.total, + "up": ir.up, + "identified": ir.identified, }, }) }) - // FIXME(sneak) add more status routes here return r } diff --git a/archiver.go b/archiver.go index 3a9cf8c..11c4d5c 100644 --- a/archiver.go +++ b/archiver.go @@ -2,12 +2,12 @@ package main import "time" -type InstanceHostName string +type InstanceHostname string type TootArchiver struct { - locator *InstanceLocator - instances map[InstanceHostName]*Instance - startup *time.Time + locator *InstanceLocator + manager *InstanceManager + startup *time.Time } func NewTootArchiver() *TootArchiver { @@ -19,15 +19,28 @@ func (a *TootArchiver) Uptime() time.Duration { return time.Since(*a.startup) } -func (a *TootArchiver) RunForever() int { +func (a *TootArchiver) RunForever() { t := time.Now() a.startup = &t + + newInstanceHostnameNotifications := make(chan InstanceHostname) + a.locator = NewInstanceLocator() + + a.manager = NewInstanceManager() + + a.locator.AddInstanceNotificationChannel(newInstanceHostnameNotifications) + a.manager.AddInstanceNotificationChannel(newInstanceHostnameNotifications) + + // locator goroutine: go a.locator.Locate() + // manager goroutine: + go a.manager.Manage() + + // this goroutine (main) does nothing until we handle signals + // FIXME(sneak) for { time.Sleep(1 * time.Second) } - - return 0 } diff --git a/instance.go b/instance.go index 96d188a..e4503cd 100644 --- a/instance.go +++ b/instance.go @@ -43,6 +43,7 @@ type Instance struct { hostName string identified bool impl InstanceImplementation + backend *InstanceBackend status InstanceStatus nextCheck *time.Time nodeInfoUrl string @@ -264,6 +265,7 @@ func (i *Instance) fetchNodeInfo() { return } +/* func (i *Instance) fetchRecentToots() ([]byte, error) { i.Lock() impl := i.impl @@ -277,13 +279,16 @@ func (i *Instance) fetchRecentToots() ([]byte, error) { panic("unimplemented") } } +*/ -func (i *Instance) fetchRecentTootsJsonFromPleroma() ([]byte, error) { +/* +func (self *PleromaBackend) fetchRecentToots() ([]byte, error) { //url := fmt.Sprintf("https://%s/api/statuses/public_and_external_timeline.json?count=100", i.hostName) return nil, nil } -func (i *Instance) fetchRecentTootsJsonFromMastodon() ([]byte, error) { +func (self *MastodonBackend) fetchRecentTootsJsonFromMastodon() ([]byte, error) { //url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName) return nil, nil } +*/ diff --git a/locator.go b/instancelocator.go similarity index 62% rename from locator.go rename to instancelocator.go index c53dffd..4f34202 100644 --- a/locator.go +++ b/instancelocator.go @@ -4,8 +4,8 @@ import "encoding/json" import "fmt" import "io/ioutil" import "net/http" -import "sync" import "time" +import "sync" import "github.com/rs/zerolog/log" @@ -17,12 +17,15 @@ var INDEX_CHECK_INTERVAL = time.Second * 60 * 60 // check with indices after 10 mins if they failed var INDEX_ERROR_INTERVAL = time.Second * 60 * 10 +var LOG_REPORT_INTERVAL = time.Second * 60 + const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const pleromaIndexUrl = "https://distsn.org/cgi-bin/distsn-pleroma-instances-api.cgi" type InstanceLocator struct { pleromaIndexNextRefresh *time.Time mastodonIndexNextRefresh *time.Time + reportInstanceVia chan InstanceHostname instances map[string]*Instance sync.Mutex } @@ -36,44 +39,49 @@ func NewInstanceLocator() *InstanceLocator { return i } -func (i *InstanceLocator) addInstance(hostname string) { - i.Lock() - defer i.Unlock() +func (self *InstanceLocator) AddInstanceNotificationChannel(via chan InstanceHostname) { + self.Lock() + defer self.Unlock() + self.reportInstanceVia = via +} + +func (self *InstanceLocator) addInstance(hostname string) { + self.Lock() + defer self.Unlock() // only add it if we haven't seen the hostname before - if i.instances[hostname] == nil { + if self.instances[hostname] == nil { log.Info().Str("hostname", hostname).Msgf("adding discovered instance") - i.instances[hostname] = NewInstance(hostname) + self.instances[hostname] = NewInstance(hostname) } } -func (i *InstanceLocator) Locate() { - x := 0 +func (self *InstanceLocator) Locate() { + x := time.Now() for { - if i.pleromaIndexNextRefresh.Before(time.Now()) { - i.locatePleroma() + if self.pleromaIndexNextRefresh.Before(time.Now()) { + self.locatePleroma() } - if i.mastodonIndexNextRefresh.Before(time.Now()) { - i.locateMastodon() + if self.mastodonIndexNextRefresh.Before(time.Now()) { + self.locateMastodon() } time.Sleep(1 * time.Second) - x++ - if x == 60 { - x = 0 + if time.Now().After(x.Add(LOG_REPORT_INTERVAL)) { + x = time.Now() log.Debug(). - Str("nextmastodonupdate", i.mastodonIndexNextRefresh.Format(time.RFC3339)). + Str("nextMastodonIndexFetch", self.mastodonIndexNextRefresh.Format(time.RFC3339)). Send() log.Debug(). - Str("nextpleromaupdate", i.pleromaIndexNextRefresh.Format(time.RFC3339)). + Str("nextPleromaIndexFetch", self.pleromaIndexNextRefresh.Format(time.RFC3339)). Send() - i.logInstanceReport() + self.logInstanceReport() } } } -func (i *InstanceLocator) logInstanceReport() { - r := i.instanceReport() +func (self *InstanceLocator) logInstanceReport() { + r := self.instanceReport() log.Info(). Uint("up", r.up). Uint("total", r.total). @@ -91,18 +99,18 @@ func (r *InstanceLocatorReport) String() string { return fmt.Sprintf("up=%d identified=%d total=%d", r.up, r.identified, r.total) } -func (i *InstanceLocator) NumInstances() uint { - return i.instanceReport().total +func (self *InstanceLocator) NumInstances() uint { + return self.instanceReport().total } -func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { - i.Lock() - defer i.Unlock() +func (self *InstanceLocator) instanceReport() *InstanceLocatorReport { + self.Lock() + defer self.Unlock() r := new(InstanceLocatorReport) - r.total = uint(len(i.instances)) + r.total = uint(len(self.instances)) - for _, elem := range i.instances { + for _, elem := range self.instances { if elem.identified == true { r.identified = r.identified + 1 } @@ -115,7 +123,7 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { return r } -func (i *InstanceLocator) locateMastodon() { +func (self *InstanceLocator) locateMastodon() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } @@ -124,9 +132,9 @@ func (i *InstanceLocator) locateMastodon() { if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) - i.Lock() - i.mastodonIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.mastodonIndexNextRefresh = &t + self.Unlock() return } @@ -136,9 +144,9 @@ func (i *InstanceLocator) locateMastodon() { if err != nil { log.Error().Msgf("unable to fetch mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) - i.Lock() - i.mastodonIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.mastodonIndexNextRefresh = &t + self.Unlock() return } @@ -147,23 +155,23 @@ func (i *InstanceLocator) locateMastodon() { if err != nil { log.Error().Msgf("unable to parse mastodon instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) - i.Lock() - i.mastodonIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.mastodonIndexNextRefresh = &t + self.Unlock() return } for _, instance := range mi.Instances { - i.addInstance(instance.Name) + self.addInstance(instance.Name) } t := time.Now().Add(INDEX_CHECK_INTERVAL) - i.Lock() - i.mastodonIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.mastodonIndexNextRefresh = &t + self.Unlock() } -func (i *InstanceLocator) locatePleroma() { +func (self *InstanceLocator) locatePleroma() { var c = &http.Client{ Timeout: INDEX_API_TIMEOUT, } @@ -172,9 +180,9 @@ func (i *InstanceLocator) locatePleroma() { if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) - i.Lock() - i.pleromaIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.pleromaIndexNextRefresh = &t + self.Unlock() return } @@ -184,9 +192,9 @@ func (i *InstanceLocator) locatePleroma() { if err != nil { log.Error().Msgf("unable to fetch pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) - i.Lock() - i.pleromaIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.pleromaIndexNextRefresh = &t + self.Unlock() return } @@ -196,17 +204,17 @@ func (i *InstanceLocator) locatePleroma() { if err != nil { log.Warn().Msgf("unable to parse pleroma instance list: %s", err) t := time.Now().Add(INDEX_ERROR_INTERVAL) - i.Lock() - i.pleromaIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.pleromaIndexNextRefresh = &t + self.Unlock() return } for _, instance := range *pi { - i.addInstance(instance.Domain) + self.addInstance(instance.Domain) } t := time.Now().Add(INDEX_CHECK_INTERVAL) - i.Lock() - i.pleromaIndexNextRefresh = &t - i.Unlock() + self.Lock() + self.pleromaIndexNextRefresh = &t + self.Unlock() } diff --git a/instancemanager.go b/instancemanager.go new file mode 100644 index 0000000..daddc53 --- /dev/null +++ b/instancemanager.go @@ -0,0 +1,34 @@ +package main + +import "sync" +import "time" + +//import "github.com/rs/zerolog/log" + +type InstanceBackend interface { + //FIXME +} + +type InstanceManager struct { + sync.Mutex + instances map[InstanceHostname]*Instance + newInstanceNotifications chan InstanceHostname +} + +func NewInstanceManager() *InstanceManager { + i := new(InstanceManager) + return i +} + +func (self *InstanceManager) AddInstanceNotificationChannel(via chan InstanceHostname) { + self.Lock() + defer self.Unlock() + self.newInstanceNotifications = via +} + +func (self *InstanceManager) Manage() { + for { + // FIXME(sneak) + time.Sleep(1 * time.Second) + } +}