From e6647e47f74ae41b36550fc13d0b0b511ff2c382 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 5 Apr 2020 22:37:22 -0700 Subject: [PATCH 1/9] incorporate postgres patch with minor changes --- Dockerfile | 2 +- README.md | 22 +++++++++++- database/manager.go | 65 ++++++++++++++++++++++++++++-------- database/storageconnector.go | 2 +- docker-compose.yml | 25 ++++++++++++++ go.sum | 1 + process/feta.go | 4 ++- 7 files changed, 103 insertions(+), 18 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 5c0568b..ce5e6d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ FROM alpine # here are the levers ENV FETA_HOSTDISCOVERYPARALLELISM 20 ENV FETA_FSSTORAGELOCATION /state/tootstore -ENV FETA_DBSTORAGELOCATION /state/feta.state.sqlite3 +ENV FETA_DBURL sqlite:///state/feta.state.sqlite3 ENV FETA_TOOTSTODISK false ENV FETA_TOOTSTODB true ENV FETA_DEBUG false diff --git a/README.md b/README.md index 9d2f1b0..e7fc91b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,25 @@ archives the fediverse [![Build Status](https://drone.datavi.be/api/badges/sneak/feta/status.svg)](https://drone.datavi.be/sneak/feta) +# getting started +## sqlite + +*using default file location:* +`./feta` + +*using file:* +`FETA_DBURL=sqlite:// ./feta` + +*using memory:* +`FETA_DBURL=sqlite://:memory: ./feta` + +## postgres + +1. `docker-compose up .` +2. `FETA_DBURL=postgres://feta_user:password@localhost:5432/feta?sslmode=disable ./feta` + +Access the pgweb dashboard at `http://localhost:8081` + # ethics statement It seems that some splinter groups are not well acquainted with the norms of @@ -41,6 +60,7 @@ legitimately-obtained files from the hard drives of other people. # Author -Jeffrey Paul <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)> +Jeffrey Paul <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)> and +others [@sneak@sneak.berlin](https://s.sneak.berlin/@sneak) diff --git a/database/manager.go b/database/manager.go index 385dfeb..988ea18 100644 --- a/database/manager.go +++ b/database/manager.go @@ -1,14 +1,18 @@ package database import ( + "net/url" "path/filepath" + "strings" "sync" - "github.com/jinzhu/gorm" + u "git.eeqj.de/sneak/goutil" + "github.com/golang/groupcache/lru" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/rs/zerolog/log" "github.com/spf13/viper" - "github.com/golang/groupcache/lru" ) const cacheEntries = 1000000 @@ -28,7 +32,7 @@ func New() *Manager { } func (m *Manager) init() { - m.open() + m.open(viper.GetString("DBURL")) // breaks stuff, do not use: //m.db.SingularTable(true) m.db.LogMode(false) @@ -38,22 +42,55 @@ func (m *Manager) init() { m.recentlyInsertedTootHashCache = lru.New(cacheEntries) } -func (m *Manager) open() { +func (m *Manager) open(dbURL string) { log.Info().Msg("opening database") - dirname := filepath.Dir(viper.GetString("DbStorageLocation")) - err := u.Mkdirp(dirname) + dsn, err := url.Parse(dbURL) if err != nil { log.Panic(). Err(err). - Msg("db path erro") + Msg("error parsing dbURL") + } + log.Info(). + Str("scheme", dsn.Scheme). + Str("user", dsn.User.Username()). + Str("host", dsn.Host). + Str("db", dsn.Path). + Str("args", dsn.RawQuery). + Msg("db connection values") + switch { + case strings.HasPrefix(dbURL, "postgres://"): + log.Info().Msg("using postgres db") + db, err := gorm.Open("postgres", dbURL) + if err != nil { + log.Panic(). + Err(err). + Msg("failed to open database") + } + m.db = db + case strings.HasPrefix(dbURL, "sqlite://"): + log.Info().Msg("using sqlite db") + if !strings.HasSuffix(dbURL, ":memory:") { + dirname := filepath.Dir(strings.TrimPrefix(dbURL, "sqlite://")) + err := u.Mkdirp(dirname) + if err != nil { + log.Panic(). + Err(err). + Msg("db path error") + } + } + db, err := gorm.Open("sqlite3", strings.TrimPrefix(dbURL, "sqlite://")) + if err != nil { + log.Panic(). + Err(err). + Str("dbURL", dbURL). + Msg("failed to open database") + } + m.db = db + default: + log.Panic(). + Str("driver", dsn.Scheme). + Msg("unsupported driver in database url, must be 'postgres' or 'sqlite'") } - db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation")) - if err != nil { - log.Panic(). - Err(err). - Msg("failed to open database") - } - m.db = db m.doMigrations() } diff --git a/database/storageconnector.go b/database/storageconnector.go index 600c079..19f5f15 100644 --- a/database/storageconnector.go +++ b/database/storageconnector.go @@ -7,7 +7,7 @@ import ( "git.eeqj.de/sneak/feta/toot" "github.com/google/uuid" hstg "github.com/grokify/html-strip-tags-go" - _ "github.com/jinzhu/gorm/dialects/sqlite" + _ "github.com/jinzhu/gorm/dialects/postgres" ) func (m *Manager) TootInsertHashCacheSize() uint { diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..32e57f1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.1' + +services: + postgres: + image: postgres:12 + restart: always + container_name: postgres + ports: + - "5432:5432" + environment: + POSTGRES_PASSWORD: password + POSTGRES_USER: feta_user + POSTGRES_DB: feta + pgweb: + image: sosedoff/pgweb + restart: always + container_name: pgweb + ports: + - "8081:8081" + links: + - postgres:postgres + environment: + - DATABASE_URL=postgres://feta_user:password@postgres:5432/feta?sslmode=disable + depends_on: + - postgres diff --git a/go.sum b/go.sum index 55dbde5..62aa566 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,7 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20= github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI= diff --git a/process/feta.go b/process/feta.go index 5ffc2f8..f9bea47 100644 --- a/process/feta.go +++ b/process/feta.go @@ -1,6 +1,7 @@ package process import ( + "fmt" "os" "time" @@ -9,6 +10,7 @@ import ( "git.eeqj.de/sneak/feta/locator" "git.eeqj.de/sneak/feta/manager" "git.eeqj.de/sneak/feta/storage" + _ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/k0kubun/pp" "github.com/mattn/go-isatty" @@ -56,7 +58,7 @@ func (f *Feta) configure() { viper.SetDefault("TootsToDB", true) viper.SetDefault("HostDiscoveryParallelism", 5) viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d")) - viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db")) + viper.SetDefault("DBURL", fmt.Sprintf("sqlite://%s", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))) viper.SetDefault("LogReportInterval", time.Second*10) if err := viper.ReadInConfig(); err != nil { -- 2.40.1 From f32deba38fec76178fbb4130786d556b9a77171c Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 00:38:19 -0700 Subject: [PATCH 2/9] should backoff reasonably now --- database/imconnector.go | 6 ++++-- database/model.go | 1 + instance/instance.go | 48 +++++++++++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/database/imconnector.go b/database/imconnector.go index e9af486..fab76f8 100644 --- a/database/imconnector.go +++ b/database/imconnector.go @@ -2,9 +2,8 @@ package database import ( "git.eeqj.de/sneak/feta/instance" - "github.com/rs/zerolog/log" - _ "github.com/jinzhu/gorm/dialects/sqlite" + "github.com/rs/zerolog/log" ) func (m *Manager) SaveInstance(i *instance.Instance) error { @@ -20,6 +19,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error { UUID: i.UUID, Disabled: i.Disabled, ErrorCount: i.ErrorCount, + ConsecutiveErrorCount: i.ConsecutiveErrorCount, FSMState: i.Status(), Fetching: i.Fetching, HighestID: i.HighestID, @@ -46,6 +46,7 @@ func (m *Manager) SaveInstance(i *instance.Instance) error { m.db.Where("UUID = ?", i.UUID).First(&ei) ei.Disabled = i.Disabled ei.ErrorCount = i.ErrorCount + ei.ConsecutiveErrorCount = i.ConsecutiveErrorCount ei.FSMState = i.Status() ei.Fetching = i.Fetching ei.HighestID = i.HighestID @@ -74,6 +75,7 @@ func (m *Manager) ListInstances() ([]*instance.Instance, error) { x.UUID = i.UUID x.Disabled = i.Disabled x.ErrorCount = i.ErrorCount + x.ConsecutiveErrorCount = i.ConsecutiveErrorCount x.InitialFSMState = i.FSMState x.Fetching = i.Fetching x.HighestID = i.HighestID diff --git a/database/model.go b/database/model.go index 93c942d..23cdee2 100644 --- a/database/model.go +++ b/database/model.go @@ -27,6 +27,7 @@ type StoredToot struct { type APInstance struct { gorm.Model UUID uuid.UUID `gorm:"type:uuid;primary_key;"` + ConsecutiveErrorCount uint ErrorCount uint SuccessCount uint HighestID uint diff --git a/instance/instance.go b/instance/instance.go index fd6eed6..6183b9f 100644 --- a/instance/instance.go +++ b/instance/instance.go @@ -17,18 +17,19 @@ import ( "github.com/rs/zerolog/log" ) -//import "github.com/gin-gonic/gin" - const nodeInfoSchemaVersionTwoName = "http://nodeinfo.diaspora.software/ns/schema/2.0" -const instanceNodeinfoTimeout = time.Second * 50 -const instanceHTTPTimeout = time.Second * 120 -const instanceSpiderInterval = time.Second * 120 -const instanceErrorInterval = time.Second * 60 * 30 +const instanceNodeinfoTimeout = time.Second * 60 * 2 // 2m +const instanceHTTPTimeout = time.Second * 60 * 2 // 2m +const instanceSpiderInterval = time.Second * 60 * 2 // 2m +const instanceErrorInterval = time.Second * 60 * 60 // 1h +const instancePersistentErrorInterval = time.Second * 86400 // 1d +const zeroInterval = time.Second * 0 // 0s // Instance stores all the information we know about an instance type Instance struct { Disabled bool ErrorCount uint + ConsecutiveErrorCount uint FSM *fsm.FSM Fetching bool HighestID uint @@ -121,10 +122,26 @@ func (i *Instance) Unlock() { i.structLock.Unlock() } -func (i *Instance) bumpFetch() { +func (i *Instance) bumpFetchError() { i.Lock() - defer i.Unlock() - i.NextFetch = time.Now().Add(120 * time.Second) + probablyDead := i.ConsecutiveErrorCount > 3 + i.Unlock() + + if probablyDead { + // if three consecutive fetch errors happen, only try once per day: + i.setNextFetchAfter(instancePersistentErrorInterval) + } else { + // otherwise give them 1h + i.setNextFetchAfter(instanceErrorInterval) + } +} + +func (i *Instance) bumpFetchSuccess() { + i.setNextFetchAfter(instanceSpiderInterval) +} + +func (i *Instance) scheduleFetchImmediate() { + i.setNextFetchAfter(zeroInterval) } func (i *Instance) setNextFetchAfter(d time.Duration) { @@ -139,8 +156,7 @@ func (i *Instance) Fetch() { i.fetchingLock.Lock() defer i.fetchingLock.Unlock() - i.setNextFetchAfter(instanceErrorInterval) - + i.bumpFetchError() err := i.DetectNodeTypeIfNecessary() if err != nil { log.Debug(). @@ -149,8 +165,7 @@ func (i *Instance) Fetch() { Msg("unable to fetch instance metadata") return } - - i.setNextFetchAfter(instanceSpiderInterval) + i.scheduleFetchImmediate() log.Info(). Str("hostname", i.Hostname). Msg("instance now ready for fetch") @@ -207,12 +222,14 @@ func (i *Instance) registerError() { i.Lock() defer i.Unlock() i.ErrorCount++ + i.ConsecutiveErrorCount++ } func (i *Instance) registerSuccess() { i.Lock() defer i.Unlock() i.SuccessCount++ + i.ConsecutiveErrorCount = 0 } // Up returns true if the success count is >0 @@ -405,9 +422,12 @@ func (i *Instance) fetchRecentToots() error { // it turns out pleroma supports the mastodon api so we'll just use that // for everything for now + + // FIXME would be nice to support non-https url := fmt.Sprintf("https://%s/api/v1/timelines/public?limit=40&local=true", i.Hostname) + // FIXME support broken/expired certs var c = &http.Client{ Timeout: instanceHTTPTimeout, } @@ -461,7 +481,7 @@ func (i *Instance) fetchRecentToots() error { Msgf("got and parsed toots") i.registerSuccess() i.Event("TOOTS_FETCHED") - i.setNextFetchAfter(instanceSpiderInterval) + i.bumpFetchSuccess() // this should go fast as either the channel is buffered bigly or the // ingester receives fast and does its own buffering, but run it in its -- 2.40.1 From b8564b519224a4b3935caa0ddf54dd095e5706e8 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:05:29 -0700 Subject: [PATCH 3/9] ignore debug logfile --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 73722d8..9807680 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ output/ feta.sqlite .lintsetup out +debug.log -- 2.40.1 From 60c00b747adf8bf75055b007f6eea6265f24ce47 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:05:35 -0700 Subject: [PATCH 4/9] log debug output to file --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0e89794..848a29f 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ endif default: build debug: build - GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) + GOTRACEBACK=all FETA_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log run: build ./$(FN) -- 2.40.1 From 95bb0aa30165d2eb5ee12d072b0b4ec8a8a0edf1 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:05:49 -0700 Subject: [PATCH 5/9] little bit of progress on webui, not working yet --- view/index.html | 64 ++++++++++++++++++++++++------------------ view/instance.html | 24 ++++++++++++++++ view/instancelist.html | 36 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 27 deletions(-) create mode 100644 view/instance.html create mode 100644 view/instancelist.html diff --git a/view/index.html b/view/index.html index 6a5c26a..aed4084 100644 --- a/view/index.html +++ b/view/index.html @@ -4,34 +4,44 @@
-

indexer stats

+

feta overview

+ + +
+
Instances
+
+
Tracking {{ instances | length }} instances + across the Fediverse.
+ + + View Instance List +
+
+ +
+
Toots
+
+
I have {{ tootCount }} toots + in my database.
+ View Latest Toots +
+
+ +
+
Recent Events
+
+
Last n System Events
+ +

Discovered instance toot1.example.com

+

Discovered instance toot2.example.com

+

Discovered instance toot3.example.com

+

Discovered instance toot4.example.com

+ +
+
- - - - - - - - - - - -{% for instance in instances %} - - - - - - - -{% endfor %} - -
instance idhostnamestatustootCountDetail
{{instance.uuid}}{{instance.hostname}}{{instance.status}}{{instance.tootCount}} - -
{% endblock %} diff --git a/view/instance.html b/view/instance.html new file mode 100644 index 0000000..fa09213 --- /dev/null +++ b/view/instance.html @@ -0,0 +1,24 @@ +{% extends "page.html" %} + +{% block content %} +
+ + +

instance {{instance.hostname}}

+ +
+
+ {{ instance.hostname }} + ({{instance.tootCount}} toots) +
+
+
{{instance.status}}
+

First Stat

+

Second Stat

+

Third Stat

+ View Instance Website +
+
+ +
+{% endblock %} diff --git a/view/instancelist.html b/view/instancelist.html new file mode 100644 index 0000000..49234b1 --- /dev/null +++ b/view/instancelist.html @@ -0,0 +1,36 @@ +{% extends "page.html" %} + +{% block content %} +
+ +

instance list

+ + + + + + + + + + + + +{% for instance in instances %} + + + + + + + +{% endfor %} + +
hostnamestatustootCountnextFetchDetail
{{instance.hostname}}{{instance.status}}{{instance.tootCount}}{{instance.nextFetch}} + +
+ +
+{% endblock %} -- 2.40.1 From 8dbd92abbd2db3b00265d24bdef5537fb2699084 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:06:30 -0700 Subject: [PATCH 6/9] add TotalTootCount database method --- database/readshortcuts.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/database/readshortcuts.go b/database/readshortcuts.go index 79edb89..92baf35 100644 --- a/database/readshortcuts.go +++ b/database/readshortcuts.go @@ -14,6 +14,16 @@ func (m *Manager) TootCountForHostname(hostname string) (uint, error) { } } +func (m *Manager) TotalTootCount() (uint, error) { + var c uint + e := m.db.Model(&StoredToot{}).Count(&c) + if e.Error != nil { + return 0, e.Error + } else { + return c, nil + } +} + func (m *Manager) GetAPInstanceFromUUID(uuid *uuid.UUID) (*APInstance, error) { var i APInstance e := m.db.Model(&APInstance{}).Where("uuid = ?", uuid).First(&i) -- 2.40.1 From 3d98a37374e5b34f15c210c29e67bf4ea4329a00 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:06:52 -0700 Subject: [PATCH 7/9] instance updates, including: * permdisable instance on 6 failures * restore in proper state on db load --- instance/instance.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/instance/instance.go b/instance/instance.go index 6183b9f..b1585f8 100644 --- a/instance/instance.go +++ b/instance/instance.go @@ -62,6 +62,10 @@ func New(options ...func(i *Instance)) *Instance { opt(i) } + if i.InitialFSMState == "FETCHING" { + i.InitialFSMState = "READY_FOR_TOOTFETCH" + } + i.FSM = fsm.NewFSM( i.InitialFSMState, fsm.Events{ @@ -75,11 +79,13 @@ func New(options ...func(i *Instance)) *Instance { {Name: "EARLY_FETCH_ERROR", Src: []string{"FETCHING_NODEINFO_URL", "PRE_NODEINFO_FETCH", "FETCHING_NODEINFO"}, Dst: "EARLY_ERROR"}, {Name: "TOOT_FETCH_ERROR", Src: []string{"FETCHING"}, Dst: "TOOT_FETCH_ERROR"}, {Name: "TOOTS_FETCHED", Src: []string{"FETCHING"}, Dst: "READY_FOR_TOOTFETCH"}, + {Name: "DISABLEMENT", Src: []string{"WEIRD_NODE", "EARLY_ERROR", "TOOT_FETCH_ERROR"}, Dst: "DISABLED"}, }, fsm.Callbacks{ "enter_state": func(e *fsm.Event) { i.fsmEnterState(e) }, }, ) + return i } @@ -125,8 +131,18 @@ func (i *Instance) Unlock() { func (i *Instance) bumpFetchError() { i.Lock() probablyDead := i.ConsecutiveErrorCount > 3 + shouldDisable := i.ConsecutiveErrorCount > 6 i.Unlock() + if shouldDisable { + // auf wiedersehen, felicia + i.Lock() + i.Disabled = true + i.Unlock() + i.Event("DISABLEMENT") + return + } + if probablyDead { // if three consecutive fetch errors happen, only try once per day: i.setNextFetchAfter(instancePersistentErrorInterval) -- 2.40.1 From ebe241ac3eda23d00759dd7ee0c3c3234cc4d59a Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:08:31 -0700 Subject: [PATCH 8/9] remove gin and begin updating handlers for webui --- process/handlers.go | 64 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/process/handlers.go b/process/handlers.go index 79e46c9..9ccc1b6 100644 --- a/process/handlers.go +++ b/process/handlers.go @@ -9,7 +9,6 @@ import ( u "git.eeqj.de/sneak/goutil" "github.com/flosch/pongo2" - "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/labstack/echo" ) @@ -23,7 +22,12 @@ func (a *Server) instances() []hash { i := make(hash) // TODO move this locking onto a method on Instance that just // returns a new hash - // FIXME figure out why a very short lock here deadlocks + + //this only locks the FSM, not the whole instance struct + i["status"] = v.Status() + + // now do a quick lock of the whole instance just to copy out the + // attrs v.Lock() i["hostname"] = v.Hostname i["uuid"] = v.UUID.String() @@ -31,9 +35,8 @@ func (a *Server) instances() []hash { i["nextCheckAfter"] = (-1 * now.Sub(v.NextFetch)).String() i["successCount"] = v.SuccessCount i["errorCount"] = v.ErrorCount + i["consecutiveErrorCount"] = v.ConsecutiveErrorCount i["identified"] = v.Identified - //this only locks the FSM, not the whole instance struct - i["status"] = v.Status() i["software"] = "unknown" i["version"] = "unknown" if v.Identified { @@ -41,6 +44,7 @@ func (a *Server) instances() []hash { i["version"] = v.ServerVersionString } v.Unlock() + resp = append(resp, i) } @@ -54,7 +58,7 @@ func (a *Server) instances() []hash { return resp } -func (a *Server) instanceSummary() map[string]int { +func (a *Server) instanceStatusSummary() map[string]int { resp := make(map[string]int) for _, v := range a.feta.manager.ListInstances() { v.Lock() @@ -68,26 +72,6 @@ func (a *Server) instanceSummary() map[string]int { return resp } -/* -func (a *Server) getInstanceListHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - - result := &gin.H{ - "instances": a.instances(), - } - - json, err := json.Marshal(result) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.Write(json) - } -} -*/ - func (a *Server) notFoundHandler(c echo.Context) error { return c.String(http.StatusNotFound, "404 not found") } @@ -117,17 +101,33 @@ func (a *Server) instanceHandler(c echo.Context) error { } func (a *Server) indexHandler(c echo.Context) error { + count, err := a.feta.dbm.TotalTootCount() + if err != nil { + count = 0 + } tc := pongo2.Context{ - "time": time.Now().UTC().Format(time.RFC3339Nano), - "gitrev": a.feta.version, - "instances": a.instances(), + "time": time.Now().UTC().Format(time.RFC3339Nano), + "gitrev": a.feta.version, + "tootCount": count, + "instances": a.instances(), + "instanceStatusSummary": a.instanceStatusSummary(), } return c.Render(http.StatusOK, "index.html", tc) } +func (a *Server) instanceListHandler(c echo.Context) error { + il := a.instances() + tc := pongo2.Context{ + "time": time.Now().UTC().Format(time.RFC3339Nano), + "gitrev": a.feta.version, + "instances": il, + } + return c.Render(http.StatusOK, "instancelist.html", tc) +} + func (a *Server) statsHandler(c echo.Context) error { - index := &gin.H{ - "server": &gin.H{ + index := &hash{ + "server": &hash{ "now": time.Now().UTC().Format(time.RFC3339), "uptime": a.feta.uptime().String(), "goroutines": runtime.NumGoroutine(), @@ -135,14 +135,14 @@ func (a *Server) statsHandler(c echo.Context) error { "version": a.feta.version, "buildarch": a.feta.buildarch, }, - "instanceSummary": a.instanceSummary(), + "instanceStatusSummary": a.instanceStatusSummary(), } return c.JSONPretty(http.StatusOK, index, " ") } func (a *Server) healthCheckHandler(c echo.Context) error { - resp := &gin.H{ + resp := &hash{ "status": "ok", "now": time.Now().UTC().Format(time.RFC3339), "uptime": a.feta.uptime().String(), -- 2.40.1 From 23d02b1c9986801b0ef951a1c79fff2ef8ce832c Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 9 Apr 2020 03:08:50 -0700 Subject: [PATCH 9/9] add /instances route --- process/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/server.go b/process/server.go index b117b04..5fa7d84 100644 --- a/process/server.go +++ b/process/server.go @@ -96,6 +96,7 @@ func (s *Server) initRouter() { // Routes s.e.GET("/", s.indexHandler) s.e.GET("/instance/:uuid", s.instanceHandler) + s.e.GET("/instances", s.instanceListHandler) s.e.GET("/stats.json", s.statsHandler) s.e.GET("/.well-known/healthcheck.json", s.healthCheckHandler) //a.e.GET("/about", s.aboutHandler) -- 2.40.1