From 3aec504aa1edea617d5dbc3f862dd006b1c0c183 Mon Sep 17 00:00:00 2001 From: Jeffrey Paul Date: Fri, 1 Nov 2019 23:56:17 -0700 Subject: [PATCH] latest --- Makefile | 14 +++++++-- archiver.go | 18 ++++-------- instance.go | 17 +++++------ locator.go | 69 ++++++++++++++++++++++++++++--------------- main.go | 43 ++++++++++++++++++--------- webserver.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 182 insertions(+), 61 deletions(-) create mode 100644 webserver.go diff --git a/Makefile b/Makefile index 543dee3..cc70b68 100644 --- a/Makefile +++ b/Makefile @@ -9,16 +9,26 @@ BUILDARCH := $(shell uname -m) FN := fediverse-archive IMAGENAME := sneak/$(FN) +UNAME_S := $(shell uname -s) + GOLDFLAGS += -X main.Version=$(VERSION) GOLDFLAGS += -X main.Buildtime=$(BUILDTIME) GOLDFLAGS += -X main.Builduser=$(BUILDUSER)@$(BUILDHOST) GOLDFLAGS += -X main.Buildarch=$(BUILDARCH) -GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)" + +# osx can't statically link apparently?! +ifeq ($(UNAME_S),Darwin) + GOFLAGS := -ldflags "$(GOLDFLAGS)" +endif + +ifneq ($(UNAME_S),Darwin) + GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)" +endif default: rundebug rundebug: build - ./$(FN) -debug + DEBUG=1 ./$(FN) run: build ./$(FN) diff --git a/archiver.go b/archiver.go index c5c43b9..3a9cf8c 100644 --- a/archiver.go +++ b/archiver.go @@ -1,31 +1,25 @@ package main -import ( - "context" - "time" - //"github.com/bitly/go-simplejson" -) +import "time" type InstanceHostName string -type Archiver struct { +type TootArchiver struct { locator *InstanceLocator instances map[InstanceHostName]*Instance startup *time.Time - ctx context.Context } -func NewArchiver(ctx context.Context) *Archiver { - a := new(Archiver) - a.ctx = ctx +func NewTootArchiver() *TootArchiver { + a := new(TootArchiver) return a } -func (a *Archiver) Uptime() time.Duration { +func (a *TootArchiver) Uptime() time.Duration { return time.Since(*a.startup) } -func (a *Archiver) RunForever() int { +func (a *TootArchiver) RunForever() int { t := time.Now() a.startup = &t a.locator = NewInstanceLocator() diff --git a/instance.go b/instance.go index a762497..3daa7a3 100644 --- a/instance.go +++ b/instance.go @@ -1,17 +1,16 @@ package main -import ( - "encoding/json" - "fmt" - "github.com/rs/zerolog/log" - "net/http" - "strings" - "time" -) +import "encoding/json" +import "fmt" +import "net/http" +import "strings" +import "time" + +import "github.com/rs/zerolog/log" const NodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" -const NODE_TIMEOUT = time.Second * 30 +const NODE_TIMEOUT = time.Second * 10 type ServerImplementation int diff --git a/locator.go b/locator.go index e4c9a6c..e5f1b3c 100644 --- a/locator.go +++ b/locator.go @@ -1,12 +1,12 @@ package main -import ( - "encoding/json" - "github.com/rs/zerolog/log" - "net/http" - "sync" - "time" -) +import "encoding/json" +import "fmt" +import "net/http" +import "sync" +import "time" + +import "github.com/rs/zerolog/log" const mastodonIndexUrl = "https://instances.social/list.json?q%5Busers%5D=&q%5Bsearch%5D=&strict=false" @@ -104,43 +104,64 @@ func (i *InstanceLocator) addInstance(hostname string) { } func (i *InstanceLocator) Locate() { - log.Debug().Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)).Send() - log.Debug().Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)).Send() + log.Debug(). + Str("lastmastodonupdate", i.mastodonIndexLastRefresh.Format(time.RFC3339)). + Send() + + log.Debug(). + Str("lastpleromaupdate", i.pleromaIndexLastRefresh.Format(time.RFC3339)). + Send() + i.locateMastodon() + i.locatePleroma() + time.Sleep(120 * time.Second) + i.instanceReport() } -func (i *InstanceLocator) instanceReport() { - var upInstances int = 0 - var identifiedInstances int = 0 - var totalInstances int = 0 +type InstanceLocatorReport struct { + up uint + identified uint + total uint +} - totalInstances = len(i.instances) +func (r *InstanceLocatorReport) String() string { + return fmt.Sprintf("up=%d identified=%d total=%d", r.up, r.identified, r.total) +} + +func (i *InstanceLocator) NumInstances() uint { + return i.instanceReport().total +} + +func (i *InstanceLocator) instanceReport() *InstanceLocatorReport { + r := new(InstanceLocatorReport) + + r.total = uint(len(i.instances)) for _, elem := range i.instances { if elem.identified == true { - identifiedInstances = identifiedInstances + 1 + r.identified = r.identified + 1 } - } - for _, elem := range i.instances { if elem.up == true { - upInstances = upInstances + 1 + r.up = r.up + 1 } } - log.Info(). - Int("up", upInstances). - Int("total", totalInstances). - Int("identified", identifiedInstances). + log.Debug(). + Uint("up", r.up). + Uint("total", r.total). + Uint("identified", r.identified). Msg("instance report") + + return r } func (i *InstanceLocator) locateMastodon() { var netClient = &http.Client{ - Timeout: time.Second * 20, + Timeout: NODE_TIMEOUT, } resp, err := netClient.Get(mastodonIndexUrl) defer resp.Body.Close() @@ -168,7 +189,7 @@ func (i *InstanceLocator) locateMastodon() { func (i *InstanceLocator) locatePleroma() { var netClient = &http.Client{ - Timeout: time.Second * 20, + Timeout: NODE_TIMEOUT, } resp, err := netClient.Get(pleromaIndexUrl) if err != nil { diff --git a/main.go b/main.go index 30511ea..1b3a747 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,11 @@ package main -import ( - "context" - "flag" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "golang.org/x/crypto/ssh/terminal" - "os" -) +import "os" +import "sync" + +import "github.com/rs/zerolog" +import "github.com/rs/zerolog/log" +import "golang.org/x/crypto/ssh/terminal" func main() { os.Exit(app()) @@ -19,16 +17,33 @@ func app() int { log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) } - debug := flag.Bool("debug", false, "sets log level to debug") - flag.Parse() - identify() zerolog.SetGlobalLevel(zerolog.InfoLevel) - if *debug { + if os.Getenv("DEBUG") != "" { zerolog.SetGlobalLevel(zerolog.DebugLevel) } - mainContext := context.Background() - return NewArchiver(mainContext).RunForever() + archiver := NewTootArchiver() + + api := new(TootArchiverAPIServer) + api.archiver = archiver + + var wg sync.WaitGroup + + // start api webserver goroutine + wg.Add(1) + go func() { + api.Serve() + wg.Done() + }() + + wg.Add(1) + go func() { + archiver.RunForever() + wg.Done() + }() + + wg.Wait() + return 0 } diff --git a/webserver.go b/webserver.go new file mode 100644 index 0000000..453c631 --- /dev/null +++ b/webserver.go @@ -0,0 +1,82 @@ +package main + +import "fmt" +import "os" +import "time" +import "net/http" + +import "github.com/rs/zerolog/log" +import "github.com/gin-gonic/gin" +import "github.com/dn365/gin-zerolog" + +type TootArchiverAPIServer struct { + archiver *TootArchiver +} + +func (a *TootArchiverAPIServer) Serve() { + if a.archiver == nil { + panic("must have archiver from which to serve stats") + } + s := a.getServer() + err := s.ListenAndServe() + if err != nil { + log.Fatal().Msg("webserver failure: " + err.Error()) + return + } +} + +func (a *TootArchiverAPIServer) getRouter() *gin.Engine { + if os.Getenv("DEBUG") == "" { + gin.SetMode(gin.ReleaseMode) + } + + // empty router + r := gin.New() + + // wrap panics: + r.Use(gin.Recovery()) + + // attach logger middleware + r.Use(ginzerolog.Logger("gin")) + + r.GET("/.well-known/healthcheck.json", func(c *gin.Context) { + c.JSON(200, gin.H{ + "status": "ok", + "now": time.Now().UTC().Format(time.RFC3339), + "uptime": a.archiver.Uptime().String(), + }) + }) + + r.GET("/", func(c *gin.Context) { + c.JSON(200, gin.H{ + // FIXME add more stuff here + "status": "ok", + "now": time.Now().UTC().Format(time.RFC3339), + "uptime": a.archiver.Uptime().String(), + "instances": a.archiver.locator.NumInstances(), + }) + }) + // FIXME add more status routes here + + return r +} + +func (a *TootArchiverAPIServer) getServer() *http.Server { + r := a.getRouter() + + port := "8080" + if os.Getenv("PORT") != "" { + port = os.Getenv("PORT") + } + + log.Info().Str("port", port).Msg("starting webserver") + + s := &http.Server{ + Addr: fmt.Sprintf(":%s", port), + Handler: r, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + return s +}