renamed

.gitignore

@@ -1,2 +1,2 @@
-fediverse-archive
+feta
 output/

Dockerfile

@@ -1,6 +1,6 @@
 FROM golang:1.13 as builder
 
-WORKDIR /go/src/github.com/sneak/fediverse-archive
+WORKDIR /go/src/github.com/sneak/feta
 COPY . .
 
 RUN go get -v && make build
@@ -13,11 +13,11 @@ RUN tar cvfz go-src.tgz src && du -sh *
 
 FROM alpine
 
-COPY --from=builder /go/src/github.com/sneak/fediverse-archive/fediverse-archive /bin/fediverse-archive
+COPY --from=builder /go/src/github.com/sneak/feta/feta /bin/feta
 
 # put the source in there too for safekeeping
 COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
 
-CMD /bin/fediverse-archive
+CMD /bin/feta
 
 # FIXME add testing

Makefile

@@ -6,7 +6,7 @@ BUILDUSER := $(shell whoami)
 BUILDHOST := $(shell hostname -s)
 BUILDARCH := $(shell uname -m)
 
-FN := fediverse-archive
+FN := feta
 IMAGENAME := sneak/$(FN)
 
 UNAME_S := $(shell uname -s)

instance.go

@@ -4,6 +4,7 @@ import "encoding/json"
 import "fmt"
 import "net/http"
 import "strings"
+import "sync"
 import "time"
 
 import "github.com/rs/zerolog/log"
@@ -11,6 +12,8 @@ import "github.com/rs/zerolog/log"
 const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
 
 const NODE_TIMEOUT = time.Second * 10
+const ONE_HOUR = time.Second * 60 * 60
+const ONE_DAY = time.Second * 60 * 60 * 24
 
 type ServerImplementation int
 
@@ -21,26 +24,23 @@ const (
 )
 
 type Instance struct {
+	sync.Mutex
 	errorCount    uint
 	highestId     int
 	hostName      string
 	impl          ServerImplementation
-	lastFailure   *time.Time
+	lastError     *time.Time
 	lastSuccess   *time.Time
+	nextCheck     *time.Time
 	nodeInfoUrl   string
 	serverVersion string
-	identified    bool
-	up            bool
-	shouldSkip    bool
 }
 
 func NewInstance(hostname string) *Instance {
+	foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
 	i := new(Instance)
 	i.hostName = hostname
-	foreverago := time.Now().Add((-1 * 86400 * 365 * 100) * time.Second)
-	i.lastSuccess = &foreverago
-	i.lastFailure = &foreverago
-	i.identified = false
+	i.nextCheck = &foreverago
 	i.up = false
 	go func() {
 		i.detectNodeType()
@@ -48,10 +48,28 @@ func NewInstance(hostname string) *Instance {
 	return i
 }
 
+func (i *Instance) setNextCheck(d *time.Duration) {
+	i.Lock()
+	defer i.Unlock()
+	i.nextCheck = d
+}
+
+func (i *Instance) dueForCheck() bool {
+	i.Lock()
+	defer i.Unlock()
+	if i.nextCheck <= time.Now() {
+		return true
+	}
+	return false
+}
+
 func (i *Instance) detectNodeType() {
+	i.Lock()
 	if i.impl > ServerUnknown {
+		i.Unlock()
 		return
 	}
+	i.Unlock()
 	i.fetchNodeInfo()
 }
 
@@ -81,12 +99,16 @@ type NodeInfoVersionTwoSchema struct {
 }
 
 func (i *Instance) registerError() {
+	i.Lock()
+	defer i.Unlock()
 	i.errorCount = i.errorCount + 1
 	t := time.Now()
-	i.lastFailure = &t
+	i.lastError = &t
 }
 
 func (i *Instance) registerSuccess() {
+	i.Lock()
+	defer i.Unlock()
 	t := time.Now()
 	i.lastSuccess = &t
 }
@@ -125,7 +147,10 @@ func (i *Instance) fetchNodeInfoURL() {
 					Str("hostname", i.hostName).
 					Str("nodeinfourl", item.Href).
 					Msg("success fetching url for nodeinfo")
+
+				i.Lock()
 				i.nodeInfoUrl = item.Href
+				i.Unlock()
 				i.registerSuccess()
 				return
 			}
@@ -140,10 +165,18 @@ func (i *Instance) fetchNodeInfoURL() {
 
 func (i *Instance) fetchNodeInfo() {
 	i.fetchNodeInfoURL()
+
+	i.Lock()
+	failure := false
 	if i.nodeInfoUrl == "" {
 		log.Error().
 			Str("hostname", i.hostName).
 			Msg("unable to fetch nodeinfo as nodeinfo URL cannot be determined")
+		failure = true
+	}
+	i.Unlock()
+
+	if failure == true {
 		return
 	}
 
@@ -153,7 +186,11 @@ func (i *Instance) fetchNodeInfo() {
 
 	//FIXME make sure the nodeinfourl is on the same domain as the instance
 	//hostname
-	resp, err := c.Get(i.nodeInfoUrl)
+	i.Lock()
+	url := i.nodeInfoUrl
+	i.Unlock()
+
+	resp, err := c.Get(url)
 
 	if err != nil {
 		log.Error().
@@ -178,6 +215,8 @@ func (i *Instance) fetchNodeInfo() {
 			Str("nodeInfoUrl", i.nodeInfoUrl).
 			Msg("received nodeinfo from instance")
 
+		i.Lock()
+		defer i.Unlock()
 		i.serverVersion = ni.Software.Version
 
 		ni.Software.Name = strings.ToLower(ni.Software.Name)
@@ -210,12 +249,16 @@ func (i *Instance) fetchNodeInfo() {
 }
 
 func (i *Instance) fetchRecentToots() ([]byte, error) {
-	if i.impl == ServerMastodon {
+	i.Lock()
+	impl := i.impl
+	i.Unlock()
+
+	if impl == ServerMastodon {
 		return i.fetchRecentTootsJsonFromMastodon()
-	} else if i.impl == ServerPleroma {
+	} else if impl == ServerPleroma {
 		return i.fetchRecentTootsJsonFromPleroma()
 	} else {
-		panic("nope")
+		panic("unimplemented")
 	}
 }
 
@@ -228,8 +271,3 @@ func (i *Instance) fetchRecentTootsJsonFromMastodon() ([]byte, error) {
 	//url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.hostName)
 	return nil, nil
 }
-
-func fetchLatestToots(lastId int) {
-	log.Debug().Msg("This message appears only when log level set to Debug")
-	log.Info().Msg("This message appears when log level set to Debug or Info")
-}

locator.go

@@ -93,12 +93,12 @@ func NewInstanceLocator() *InstanceLocator {
 }
 
 func (i *InstanceLocator) addInstance(hostname string) {
+	i.Lock()
+	defer i.Unlock()
 	// only add it if we haven't seen the hostname before
 	if i.instances[hostname] == nil {
 		log.Debug().Str("hostname", hostname).Msgf("adding discovered instance")
-		i.Lock()
 		i.instances[hostname] = NewInstance(hostname)
-		i.Unlock()
 	}
 
 }
@@ -118,7 +118,14 @@ func (i *InstanceLocator) Locate() {
 
 	time.Sleep(120 * time.Second)
 
-	i.instanceReport()
+	r := i.instanceReport()
+
+	log.Debug().
+		Uint("up", r.up).
+		Uint("total", r.total).
+		Uint("identified", r.identified).
+		Msg("instance report")
+
 }
 
 type InstanceLocatorReport struct {
@@ -136,6 +143,8 @@ func (i *InstanceLocator) NumInstances() uint {
 }
 
 func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
+	i.Lock()
+	defer i.Unlock()
 	r := new(InstanceLocatorReport)
 
 	r.total = uint(len(i.instances))
@@ -150,12 +159,6 @@ func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
 		}
 	}
 
-	log.Debug().
-		Uint("up", r.up).
-		Uint("total", r.total).
-		Uint("identified", r.identified).
-		Msg("instance report")
-
 	return r
 }
 
@@ -179,8 +182,8 @@ func (i *InstanceLocator) locateMastodon() {
 				i.addInstance(instance.Name)
 			}
 
-			i.Lock()
 			t := time.Now()
+			i.Lock()
 			i.mastodonIndexLastRefresh = &t
 			i.Unlock()
 		}
@@ -204,8 +207,8 @@ func (i *InstanceLocator) locatePleroma() {
 			for _, instance := range *pi {
 				i.addInstance(instance.Domain)
 			}
-			i.Lock()
 			t := time.Now()
+			i.Lock()
 			i.pleromaIndexLastRefresh = &t
 			i.Unlock()
 		}

webserver.go

@@ -49,14 +49,18 @@ func (a *TootArchiverAPIServer) getRouter() *gin.Engine {
 
 	r.GET("/", func(c *gin.Context) {
 		c.JSON(200, gin.H{
-			// FIXME add more stuff here
+			// FIXME(sneak) add more stuff here
 			"status": "ok",
 			"now":    time.Now().UTC().Format(time.RFC3339),
 			"uptime": a.archiver.Uptime().String(),
-			"instances": a.archiver.locator.NumInstances(),
+			"instances": gin.H{
+				"total":      a.archiver.locator.instanceReport().total,
+				"up":         a.archiver.locator.instanceReport().up,
+				"identified": a.archiver.locator.instanceReport().identified,
+			},
 		})
 	})
-	// FIXME add more status routes here
+	// FIXME(sneak) add more status routes here
 
 	return r
 }