This commit is contained in:
Jeffrey Paul 2019-11-01 23:56:17 -07:00
parent 92335ead98
commit 3aec504aa1
6 changed files with 182 additions and 61 deletions

View File

@ -9,16 +9,26 @@ BUILDARCH := $(shell uname -m)
FN := fediverse-archive FN := fediverse-archive
IMAGENAME := sneak/$(FN) IMAGENAME := sneak/$(FN)
UNAME_S := $(shell uname -s)
GOLDFLAGS += -X main.Version=$(VERSION) GOLDFLAGS += -X main.Version=$(VERSION)
GOLDFLAGS += -X main.Buildtime=$(BUILDTIME) GOLDFLAGS += -X main.Buildtime=$(BUILDTIME)
GOLDFLAGS += -X main.Builduser=$(BUILDUSER)@$(BUILDHOST) GOLDFLAGS += -X main.Builduser=$(BUILDUSER)@$(BUILDHOST)
GOLDFLAGS += -X main.Buildarch=$(BUILDARCH) GOLDFLAGS += -X main.Buildarch=$(BUILDARCH)
# osx can't statically link apparently?!
ifeq ($(UNAME_S),Darwin)
GOFLAGS := -ldflags "$(GOLDFLAGS)"
endif
ifneq ($(UNAME_S),Darwin)
GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)" GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)"
endif
default: rundebug default: rundebug
rundebug: build rundebug: build
./$(FN) -debug DEBUG=1 ./$(FN)
run: build run: build
./$(FN) ./$(FN)

View File

@ -1,31 +1,25 @@
package main package main
import ( import "time"
"context"
"time"
//"github.com/bitly/go-simplejson"
)
type InstanceHostName string type InstanceHostName string
type Archiver struct { type TootArchiver struct {
locator *InstanceLocator locator *InstanceLocator
instances map[InstanceHostName]*Instance instances map[InstanceHostName]*Instance
startup *time.Time startup *time.Time
ctx context.Context
} }
func NewArchiver(ctx context.Context) *Archiver { func NewTootArchiver() *TootArchiver {
a := new(Archiver) a := new(TootArchiver)
a.ctx = ctx
return a return a
} }
func (a *Archiver) Uptime() time.Duration { func (a *TootArchiver) Uptime() time.Duration {
return time.Since(*a.startup) return time.Since(*a.startup)
} }
func (a *Archiver) RunForever() int { func (a *TootArchiver) RunForever() int {
t := time.Now() t := time.Now()
a.startup = &t a.startup = &t
a.locator = NewInstanceLocator() a.locator = NewInstanceLocator()

View File

@ -1,17 +1,16 @@
package main package main
import ( import "encoding/json"
"encoding/json" import "fmt"
"fmt" import "net/http"
"github.com/rs/zerolog/log" import "strings"
"net/http" import "time"
"strings"
"time" import "github.com/rs/zerolog/log"
)
const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0"
const NODE_TIMEOUT = time.Second * 30 const NODE_TIMEOUT = time.Second * 10
type ServerImplementation int type ServerImplementation int

View File

@ -1,12 +1,12 @@
package main package main
import ( import "encoding/json"
"encoding/json" import "fmt"
"github.com/rs/zerolog/log" import "net/http"
"net/http" import "sync"
"sync" import "time"
"time"
) import "github.com/rs/zerolog/log"
const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false"
@ -104,43 +104,64 @@ func (i *InstanceLocator) addInstance(hostname string) {
} }
func (i *InstanceLocator) Locate() { func (i *InstanceLocator) Locate() {
log.Debug().Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)).Send() log.Debug().
log.Debug().Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)).Send() Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)).
Send()
log.Debug().
Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)).
Send()
i.locateMastodon() i.locateMastodon()
i.locatePleroma() i.locatePleroma()
time.Sleep(120 * time.Second) time.Sleep(120 * time.Second)
i.instanceReport() i.instanceReport()
} }
func (i *InstanceLocator) instanceReport() { type InstanceLocatorReport struct {
var upInstances int = 0 up uint
var identifiedInstances int = 0 identified uint
var totalInstances int = 0 total uint
}
totalInstances = len(i.instances) func (r *InstanceLocatorReport) String() string {
return fmt.Sprintf("up=%d identified=%d total=%d", r.up, r.identified, r.total)
}
func (i *InstanceLocator) NumInstances() uint {
return i.instanceReport().total
}
func (i *InstanceLocator) instanceReport() *InstanceLocatorReport {
r := new(InstanceLocatorReport)
r.total = uint(len(i.instances))
for _, elem := range i.instances { for _, elem := range i.instances {
if elem.identified == true { if elem.identified == true {
identifiedInstances = identifiedInstances + 1 r.identified = r.identified + 1
}
} }
for _, elem := range i.instances {
if elem.up == true { if elem.up == true {
upInstances = upInstances + 1 r.up = r.up + 1
} }
} }
log.Info(). log.Debug().
Int("up", upInstances). Uint("up", r.up).
Int("total", totalInstances). Uint("total", r.total).
Int("identified", identifiedInstances). Uint("identified", r.identified).
Msg("instance report") Msg("instance report")
return r
} }
func (i *InstanceLocator) locateMastodon() { func (i *InstanceLocator) locateMastodon() {
var netClient = &http.Client{ var netClient = &http.Client{
Timeout: time.Second * 20, Timeout: NODE_TIMEOUT,
} }
resp, err := netClient.Get(mastodonIndexUrl) resp, err := netClient.Get(mastodonIndexUrl)
defer resp.Body.Close() defer resp.Body.Close()
@ -168,7 +189,7 @@ func (i *InstanceLocator) locateMastodon() {
func (i *InstanceLocator) locatePleroma() { func (i *InstanceLocator) locatePleroma() {
var netClient = &http.Client{ var netClient = &http.Client{
Timeout: time.Second * 20, Timeout: NODE_TIMEOUT,
} }
resp, err := netClient.Get(pleromaIndexUrl) resp, err := netClient.Get(pleromaIndexUrl)
if err != nil { if err != nil {

43
main.go
View File

@ -1,13 +1,11 @@
package main package main
import ( import "os"
"context" import "sync"
"flag"
"github.com/rs/zerolog" import "github.com/rs/zerolog"
"github.com/rs/zerolog/log" import "github.com/rs/zerolog/log"
"golang.org/x/crypto/ssh/terminal" import "golang.org/x/crypto/ssh/terminal"
"os"
)
func main() { func main() {
os.Exit(app()) os.Exit(app())
@ -19,16 +17,33 @@ func app() int {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
} }
debug := flag.Bool("debug", false, "sets log level to debug")
flag.Parse()
identify() identify()
zerolog.SetGlobalLevel(zerolog.InfoLevel) zerolog.SetGlobalLevel(zerolog.InfoLevel)
if *debug { if os.Getenv("DEBUG") != "" {
zerolog.SetGlobalLevel(zerolog.DebugLevel) zerolog.SetGlobalLevel(zerolog.DebugLevel)
} }
mainContext := context.Background() archiver := NewTootArchiver()
return NewArchiver(mainContext).RunForever()
api := new(TootArchiverAPIServer)
api.archiver = archiver
var wg sync.WaitGroup
// start api webserver goroutine
wg.Add(1)
go func() {
api.Serve()
wg.Done()
}()
wg.Add(1)
go func() {
archiver.RunForever()
wg.Done()
}()
wg.Wait()
return 0
} }

82
webserver.go Normal file
View File

@ -0,0 +1,82 @@
package main
import "fmt"
import "os"
import "time"
import "net/http"
import "github.com/rs/zerolog/log"
import "github.com/gin-gonic/gin"
import "github.com/dn365/gin-zerolog"
type TootArchiverAPIServer struct {
archiver *TootArchiver
}
func (a *TootArchiverAPIServer) Serve() {
if a.archiver == nil {
panic("must have archiver from which to serve stats")
}
s := a.getServer()
err := s.ListenAndServe()
if err != nil {
log.Fatal().Msg("webserver failure: " + err.Error())
return
}
}
func (a *TootArchiverAPIServer) getRouter() *gin.Engine {
if os.Getenv("DEBUG") == "" {
gin.SetMode(gin.ReleaseMode)
}
// empty router
r := gin.New()
// wrap panics:
r.Use(gin.Recovery())
// attach logger middleware
r.Use(ginzerolog.Logger("gin"))
r.GET("/.well-known/healthcheck.json", func(c *gin.Context) {
c.JSON(200, gin.H{
"status": "ok",
"now": time.Now().UTC().Format(time.RFC3339),
"uptime": a.archiver.Uptime().String(),
})
})
r.GET("/", func(c *gin.Context) {
c.JSON(200, gin.H{
// FIXME add more stuff here
"status": "ok",
"now": time.Now().UTC().Format(time.RFC3339),
"uptime": a.archiver.Uptime().String(),
"instances": a.archiver.locator.NumInstances(),
})
})
// FIXME add more status routes here
return r
}
func (a *TootArchiverAPIServer) getServer() *http.Server {
r := a.getRouter()
port := "8080"
if os.Getenv("PORT") != "" {
port = os.Getenv("PORT")
}
log.Info().Str("port", port).Msg("starting webserver")
s := &http.Server{
Addr: fmt.Sprintf(":%s", port),
Handler: r,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
return s
}