Compare commits
2
Commits
2c034b4bf1
...
5f1971a293
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
5f1971a293 | ||
 clawbot
|
dc2d240725 |
@@ -266,6 +266,10 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
||||
password as their basic auth credentials. With neither set, there are no
|
||||
metrics and `/metrics` is not found. One set without the other, or a user
|
||||
name containing `:`, stops the container
|
||||
- `SENTRY_DSN`, default empty: set to a Sentry project's DSN, the backend
|
||||
sends its errors to that Sentry project: each request whose handling
|
||||
crashes, which still gets a 500 response. A value Sentry does not accept
|
||||
stops the container. Empty, the backend sends nothing to Sentry
|
||||
- **Health check:** the image's `HEALTHCHECK` requests
|
||||
`/.well-known/healthcheck` through nginx every 30 seconds, so it fails unless
|
||||
both nginx and the backend answer. upaas reads the container's health 60
|
||||
|
||||
@@ -23,6 +23,24 @@ latest run passes.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-04: in `src/main.js` (issue #102), a target's min, max, median and
|
||||
average latency come from one list of its answers, through the same function
|
||||
the summary's figures use, so the median is written once. The latency color
|
||||
limits are one table in `CONFIG`, read by both the figure's and the
|
||||
sparkline's color. The health thresholds, the debug log's length, the gateway
|
||||
check's timeout, the recovery probe's number of hosts and interval, how often
|
||||
the rows are sorted and the delay before the first sparkline resize are
|
||||
`CONFIG` entries too. A unit test checks the summary's figures. Nothing the
|
||||
page does or shows changed; the footer's color legend still writes the limits
|
||||
out as text
|
||||
- 2026-10-04: the backend reports errors to Sentry (issue #95). With
|
||||
`SENTRY_DSN` set, it sets up `sentry-go` with the release `netwatch-server-`
|
||||
and its version, reports each panic in a handler through `sentryhttp`, the
|
||||
last of the middleware every request goes through, which panics again so the
|
||||
request still gets the 500 from the panic recovery, and waits up to 2 seconds
|
||||
on shutdown for Sentry to finish sending. A DSN Sentry refuses stops the start
|
||||
with an error naming `SENTRY_DSN`. With it empty, Sentry is not set up and
|
||||
nothing is sent to it
|
||||
- 2026-10-04: a target's name and URL and a debug log message show as the
|
||||
characters they are and are never read as HTML (issue #29): a host row escapes
|
||||
the name and URL it writes into its markup, and the debug log sets each line
|
||||
|
||||
@@ -90,6 +90,7 @@ project layout:
|
||||
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
||||
| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) |
|
||||
| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) |
|
||||
| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) |
|
||||
|
||||
`TRUSTED_PROXIES` defaults to
|
||||
`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback
|
||||
@@ -198,6 +199,16 @@ is recorded and `/metrics` answers 404. One without the other stops the server
|
||||
from starting, with an error naming both; so does a `METRICS_USERNAME`
|
||||
containing `:`, which basic auth cannot carry, with an error naming it.
|
||||
|
||||
### Sentry
|
||||
|
||||
With `SENTRY_DSN` set, the server sends its errors to that Sentry project: each
|
||||
panic in a handler is reported there, under the release `netwatch-server-`
|
||||
followed by the server's version, and the request still gets 500 from the
|
||||
server's panic recovery. On shutdown the server waits up to 2 seconds for Sentry
|
||||
to finish sending. A DSN Sentry refuses stops the server from starting, with an
|
||||
error naming `SENTRY_DSN`. With it empty, Sentry is not set up, and nothing is
|
||||
sent to it.
|
||||
|
||||
## TODO
|
||||
|
||||
- Add integration test that POSTs a report and verifies the compressed output
|
||||
|
||||
@@ -115,6 +115,28 @@ func TestMalformedConfigFileStopsTheStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefusedSentryDSNStopsTheStart: a SENTRY_DSN that Sentry refuses
|
||||
// stops the start, and the error, naming SENTRY_DSN, is logged as JSON.
|
||||
func TestRefusedSentryDSNStopsTheStart(t *testing.T) {
|
||||
t.Setenv("SENTRY_DSN", "not-a-dsn")
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), childTimeout)
|
||||
defer cancel()
|
||||
|
||||
child, stdout, stderr := startServer(ctx, t, t.TempDir(), freePort(ctx, t))
|
||||
|
||||
err := child.Wait()
|
||||
if child.ProcessState.ExitCode() != 1 {
|
||||
t.Fatalf("server exit = %v, want exit status 1", err)
|
||||
}
|
||||
|
||||
requireJSONLines(t, stdout, stderr)
|
||||
|
||||
if !strings.Contains(stdout.String(), "SENTRY_DSN") {
|
||||
t.Fatalf("no error naming SENTRY_DSN in stdout:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
// startServer runs main() in a child process listening on
|
||||
// 127.0.0.1:port, with home as its HOME and working directory and its
|
||||
// data directory in home, so it touches nothing outside home. Its
|
||||
|
||||
@@ -4,6 +4,7 @@ go 1.25.5
|
||||
|
||||
require (
|
||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||
github.com/getsentry/sentry-go v0.49.0
|
||||
github.com/go-chi/chi/v5 v5.2.5
|
||||
github.com/go-chi/cors v1.2.2
|
||||
github.com/go-chi/httprate v0.16.0
|
||||
|
||||
+16
-8
@@ -4,18 +4,22 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/getsentry/sentry-go v0.49.0 h1:Ehejknu1l023Ub7QoRBVLAI7g3Jnhqku4oWx4B4Sh5s=
|
||||
github.com/getsentry/sentry-go v0.49.0/go.mod h1:nuMJAoCfe1u0Bts2ocyNI+TW8HT84vRMqwA5Qq/SKUI=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||
github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
|
||||
github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@@ -36,8 +40,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
@@ -46,8 +54,8 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8=
|
||||
@@ -93,7 +101,7 @@ golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
package server
|
||||
|
||||
import "github.com/go-chi/chi/v5"
|
||||
|
||||
// Router exposes the router to the external tests, which add routes
|
||||
// of their own to it after SetupRoutes.
|
||||
func (s *Server) Router() *chi.Mux {
|
||||
return s.router
|
||||
}
|
||||
|
||||
// MaxRequestBodyBytes exposes the router-wide body limit to the
|
||||
// external tests.
|
||||
const MaxRequestBodyBytes = maxRequestBodyBytes
|
||||
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"time"
|
||||
|
||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -32,6 +33,12 @@ func (s *Server) SetupRoutes() {
|
||||
s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
|
||||
s.router.Use(middleware.Timeout(requestTimeout))
|
||||
|
||||
// Sentry reports a panic, then panics again, so that s.mw.Recoverer
|
||||
// still answers 500.
|
||||
if s.params.Config.SentryDSN != "" {
|
||||
s.router.Use(sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle)
|
||||
}
|
||||
|
||||
// The metrics go in a registry of this server's own, not in
|
||||
// Prometheus' default one, which takes them only once per process.
|
||||
registry := prometheus.NewRegistry()
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package server_test
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
@@ -15,6 +17,7 @@ import (
|
||||
"sneak.berlin/go/netwatch/internal/reportbuf"
|
||||
"sneak.berlin/go/netwatch/internal/server"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxtest"
|
||||
)
|
||||
@@ -196,6 +199,70 @@ func TestMetricsInTwoServers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSentry: with SENTRY_DSN empty there is no Sentry client. With it
|
||||
// pointing at a local server standing in for Sentry, a panic in a
|
||||
// handler reaches that server, and the request still gets the 500 from
|
||||
// the panic recovery.
|
||||
func TestSentry(t *testing.T) {
|
||||
const panicMessage = "handler panic for TestSentry"
|
||||
|
||||
// sentry.Init sets the client for the whole process; take it away
|
||||
// again so that no other test reports to Sentry.
|
||||
t.Cleanup(func() { sentry.CurrentHub().BindClient(nil) })
|
||||
|
||||
t.Setenv("SENTRY_DSN", "")
|
||||
newServer(t)
|
||||
|
||||
if sentry.CurrentHub().Client() != nil {
|
||||
t.Fatal("a Sentry client exists with SENTRY_DSN empty")
|
||||
}
|
||||
|
||||
// The body of the first request the stand-in for Sentry receives.
|
||||
received := make(chan string, 1)
|
||||
|
||||
sentryServer := httptest.NewServer(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
|
||||
select {
|
||||
case received <- string(body):
|
||||
default:
|
||||
}
|
||||
},
|
||||
))
|
||||
defer sentryServer.Close()
|
||||
|
||||
t.Setenv("SENTRY_DSN",
|
||||
"http://key@"+sentryServer.Listener.Addr().String()+"/1")
|
||||
|
||||
srv := newServer(t)
|
||||
srv.SetupRoutes()
|
||||
srv.Router().Get("/panic", func(http.ResponseWriter, *http.Request) {
|
||||
panic(panicMessage)
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequestWithContext(t.Context(),
|
||||
http.MethodGet, "/panic", http.NoBody)
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status = %d, want %d",
|
||||
rec.Code, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
// Sentry sends from a goroutine of its own.
|
||||
select {
|
||||
case body := <-received:
|
||||
if !strings.Contains(body, panicMessage) {
|
||||
t.Fatalf("the Sentry server received no report of the panic:\n%s",
|
||||
body)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("nothing reached the Sentry server")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHealthCheckRejectsOversizeBody sends the health check, which
|
||||
// never reads its body, a body one byte over the limit. Only the
|
||||
// router-wide body limit can reject it.
|
||||
|
||||
@@ -7,8 +7,10 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
@@ -16,10 +18,15 @@ import (
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
"sneak.berlin/go/netwatch/internal/middleware"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// sentryFlushTimeout is how long shutdown waits for Sentry to send
|
||||
// what it still holds.
|
||||
const sentryFlushTimeout = 2 * time.Second
|
||||
|
||||
// Params defines the dependencies for Server.
|
||||
type Params struct {
|
||||
fx.In
|
||||
@@ -56,6 +63,11 @@ func New(
|
||||
s.log = params.Logger.Get()
|
||||
s.shutdowner = params.Shutdowner
|
||||
|
||||
err := s.enableSentry()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
// Build the router and http.Server synchronously
|
||||
@@ -87,10 +99,37 @@ func (s *Server) ServeHTTP(
|
||||
s.router.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// enableSentry sets Sentry up when SENTRY_DSN is set, so that
|
||||
// SetupRoutes can report panics to it. With SENTRY_DSN empty it does
|
||||
// nothing. A DSN Sentry refuses stops the start.
|
||||
func (s *Server) enableSentry() error {
|
||||
if s.params.Config.SentryDSN == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: s.params.Config.SentryDSN,
|
||||
Release: s.params.Globals.Appname + "-" + s.params.Globals.Version,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("SENTRY_DSN: %w", err)
|
||||
}
|
||||
|
||||
s.log.Info("sentry error reporting activated")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// shutdown gracefully stops the HTTP server within the
|
||||
// deadline of the context fx provides for OnStop.
|
||||
// deadline of the context fx provides for OnStop, then gives
|
||||
// Sentry, if set up, time to send what it still holds.
|
||||
func (s *Server) shutdown(ctx context.Context) error {
|
||||
err := s.httpServer.Shutdown(ctx)
|
||||
|
||||
if s.params.Config.SentryDSN != "" {
|
||||
sentry.Flush(sentryFlushTimeout)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
s.log.Error("server clean shutdown failed", "error", err)
|
||||
|
||||
|
||||
+93
-82
@@ -30,6 +30,39 @@ export const CONFIG = {
|
||||
return [0, 1, 2, 3, 4, 5].map((i) => Math.round((d * i) / 5));
|
||||
},
|
||||
canvasHeight: 96,
|
||||
// A latency figure and its sparkline take the color of the first entry
|
||||
// whose limit, in ms, the latency is below.
|
||||
latencyColors: [
|
||||
{ below: 50, hex: "#22c55e", className: "text-green-500" },
|
||||
{ below: 100, hex: "#84cc16", className: "text-lime-500" },
|
||||
{ below: 200, hex: "#eab308", className: "text-yellow-500" },
|
||||
{ below: 500, hex: "#f97316", className: "text-orange-500" },
|
||||
{ below: Infinity, hex: "#ef4444", className: "text-red-500" },
|
||||
],
|
||||
// The health is offline when more than offlineTimeouts WAN hosts timed
|
||||
// out or were unreachable and at most offlineReachable answered;
|
||||
// otherwise degraded when more than degradedTimeouts timed out or were
|
||||
// unreachable; otherwise slow when more than slowHosts answered after
|
||||
// more than slowLatency ms.
|
||||
offlineTimeouts: 10,
|
||||
offlineReachable: 4,
|
||||
degradedTimeouts: 4,
|
||||
slowHosts: 3,
|
||||
slowLatency: 1000,
|
||||
// The debug log keeps its last maxLogEntries lines.
|
||||
maxLogEntries: 1000,
|
||||
// A gateway candidate that has not answered after gatewayTimeout ms is
|
||||
// passed over.
|
||||
gatewayTimeout: 1500,
|
||||
// When no WAN host answers, the recovery probe checks recoveryProbeHosts
|
||||
// random ones every recoveryProbeInterval ms.
|
||||
recoveryProbeHosts: 4,
|
||||
recoveryProbeInterval: 500,
|
||||
// The rows are sorted after the first round that is not discarded, then
|
||||
// every roundsPerSort rounds.
|
||||
roundsPerSort: 10,
|
||||
// The sparklines are sized and drawn again resizeDelay ms after start.
|
||||
resizeDelay: 100,
|
||||
};
|
||||
|
||||
// WAN endpoints to monitor. These are used for the aggregate health/stats
|
||||
@@ -114,7 +147,8 @@ const debugLog = [];
|
||||
const log = (() => {
|
||||
function append(level, message) {
|
||||
debugLog.push({ timestamp: new Date(), level, message });
|
||||
if (debugLog.length > 1000) debugLog.splice(0, debugLog.length - 1000);
|
||||
if (debugLog.length > CONFIG.maxLogEntries)
|
||||
debugLog.splice(0, debugLog.length - CONFIG.maxLogEntries);
|
||||
const panel = document.getElementById("debug-panel");
|
||||
if (panel && !panel.classList.contains("hidden")) renderDebugLog();
|
||||
}
|
||||
@@ -174,7 +208,10 @@ async function detectGateway() {
|
||||
const result = await Promise.any(
|
||||
GATEWAY_CANDIDATES.map(async (url) => {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 1500);
|
||||
const timeoutId = setTimeout(
|
||||
() => controller.abort(),
|
||||
CONFIG.gatewayTimeout,
|
||||
);
|
||||
try {
|
||||
await fetch(url, {
|
||||
method: "GET",
|
||||
@@ -199,6 +236,27 @@ async function detectGateway() {
|
||||
|
||||
// --- App State ---------------------------------------------------------------
|
||||
|
||||
// The min, max, median and average of latencies, a list of numbers, or all
|
||||
// null when it is empty. The median of an even count is the mean of the
|
||||
// middle two; it and the average are rounded.
|
||||
function latencyStats(latencies) {
|
||||
if (latencies.length === 0)
|
||||
return { min: null, max: null, med: null, avg: null };
|
||||
const sorted = [...latencies].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
return {
|
||||
min: sorted[0],
|
||||
max: sorted[sorted.length - 1],
|
||||
med:
|
||||
sorted.length % 2
|
||||
? sorted[mid]
|
||||
: Math.round((sorted[mid - 1] + sorted[mid]) / 2),
|
||||
avg: Math.round(
|
||||
latencies.reduce((a, b) => a + b, 0) / latencies.length,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export class HostState {
|
||||
constructor(host, pinned = false) {
|
||||
this.name = host.name;
|
||||
@@ -230,38 +288,16 @@ export class HostState {
|
||||
this._trim();
|
||||
}
|
||||
|
||||
averageLatency() {
|
||||
const valid = this.history.filter((p) => p.latency !== null);
|
||||
if (valid.length === 0) return null;
|
||||
return Math.round(
|
||||
valid.reduce((s, p) => s + p.latency, 0) / valid.length,
|
||||
// The min, max, median and average latency of the checks in the history
|
||||
// that got an answer.
|
||||
historyStats() {
|
||||
return latencyStats(
|
||||
this.history
|
||||
.filter((p) => p.latency !== null)
|
||||
.map((p) => p.latency),
|
||||
);
|
||||
}
|
||||
|
||||
minLatency() {
|
||||
const valid = this.history.filter((p) => p.latency !== null);
|
||||
if (valid.length === 0) return null;
|
||||
return Math.min(...valid.map((p) => p.latency));
|
||||
}
|
||||
|
||||
maxLatency() {
|
||||
const valid = this.history.filter((p) => p.latency !== null);
|
||||
if (valid.length === 0) return null;
|
||||
return Math.max(...valid.map((p) => p.latency));
|
||||
}
|
||||
|
||||
medianLatency() {
|
||||
const sorted = this.history
|
||||
.filter((p) => p.latency !== null)
|
||||
.map((p) => p.latency)
|
||||
.sort((a, b) => a - b);
|
||||
if (sorted.length === 0) return null;
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2
|
||||
? sorted[mid]
|
||||
: Math.round((sorted[mid - 1] + sorted[mid]) / 2);
|
||||
}
|
||||
|
||||
_trim() {
|
||||
while (this.history.length > CONFIG.maxHistoryPoints)
|
||||
this.history.shift();
|
||||
@@ -288,33 +324,13 @@ export class AppState {
|
||||
|
||||
/** WAN-only stats from latest sample (excludes local) */
|
||||
wanStats() {
|
||||
const reachable = this.wan.filter((h) => h.lastLatency !== null);
|
||||
const latencies = reachable.map((h) => h.lastLatency);
|
||||
const total = this.wan.length;
|
||||
if (latencies.length === 0)
|
||||
return {
|
||||
reachable: 0,
|
||||
total,
|
||||
min: null,
|
||||
max: null,
|
||||
med: null,
|
||||
avg: null,
|
||||
};
|
||||
const sorted = [...latencies].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
const med =
|
||||
sorted.length % 2
|
||||
? sorted[mid]
|
||||
: Math.round((sorted[mid - 1] + sorted[mid]) / 2);
|
||||
const latencies = this.wan
|
||||
.filter((h) => h.lastLatency !== null)
|
||||
.map((h) => h.lastLatency);
|
||||
return {
|
||||
reachable: latencies.length,
|
||||
total,
|
||||
min: Math.min(...latencies),
|
||||
max: Math.max(...latencies),
|
||||
med,
|
||||
avg: Math.round(
|
||||
latencies.reduce((a, b) => a + b, 0) / latencies.length,
|
||||
),
|
||||
total: this.wan.length,
|
||||
...latencyStats(latencies),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -340,12 +356,16 @@ export class AppState {
|
||||
const timeouts = this.wan.filter(
|
||||
(h) => h.status === "error" || h.status === "offline",
|
||||
).length;
|
||||
if (timeouts > 10 && reachable <= 4) return "offline";
|
||||
if (timeouts > 4) return "degraded";
|
||||
if (
|
||||
timeouts > CONFIG.offlineTimeouts &&
|
||||
reachable <= CONFIG.offlineReachable
|
||||
)
|
||||
return "offline";
|
||||
if (timeouts > CONFIG.degradedTimeouts) return "degraded";
|
||||
const slow = this.wan.filter(
|
||||
(h) => h.lastLatency !== null && h.lastLatency > 1000,
|
||||
(h) => h.lastLatency !== null && h.lastLatency > CONFIG.slowLatency,
|
||||
).length;
|
||||
if (slow > 3) return "slow";
|
||||
if (slow > CONFIG.slowHosts) return "slow";
|
||||
return "healthy";
|
||||
}
|
||||
|
||||
@@ -557,21 +577,13 @@ export async function measureLatency(url, signal) {
|
||||
|
||||
export function latencyHex(latency) {
|
||||
if (latency === null) return "#6b7280";
|
||||
if (latency < 50) return "#22c55e";
|
||||
if (latency < 100) return "#84cc16";
|
||||
if (latency < 200) return "#eab308";
|
||||
if (latency < 500) return "#f97316";
|
||||
return "#ef4444";
|
||||
return CONFIG.latencyColors.find((c) => latency < c.below).hex;
|
||||
}
|
||||
|
||||
export function latencyClass(latency, status) {
|
||||
if (status === "offline" || status === "error" || latency === null)
|
||||
return "text-gray-500";
|
||||
if (latency < 50) return "text-green-500";
|
||||
if (latency < 100) return "text-lime-500";
|
||||
if (latency < 200) return "text-yellow-500";
|
||||
if (latency < 500) return "text-orange-500";
|
||||
return "text-red-500";
|
||||
return CONFIG.latencyColors.find((c) => latency < c.below).className;
|
||||
}
|
||||
|
||||
// --- Sparkline Renderer ------------------------------------------------------
|
||||
@@ -912,10 +924,7 @@ function updateHostRow(host, index) {
|
||||
latencyEl.innerHTML = `<span class="text-gray-500">---</span>`;
|
||||
}
|
||||
|
||||
const avg = host.averageLatency();
|
||||
const med = host.medianLatency();
|
||||
const min = host.minLatency();
|
||||
const max = host.maxLatency();
|
||||
const { min, med, avg, max } = host.historyStats();
|
||||
if (host.status === "online" && avg !== null) {
|
||||
statusEl.innerHTML = statusStatsHTML([
|
||||
["min", min],
|
||||
@@ -1182,8 +1191,9 @@ export async function tick(state, signal, onOffline) {
|
||||
// rows whose check ended before the resume still read "paused"
|
||||
state.allHosts.forEach((host, i) => updateHostRow(host, i));
|
||||
|
||||
// Sort after the first real check, then every 10 ticks thereafter
|
||||
if (state.tickCount === 2 || state.tickCount % 10 === 1) {
|
||||
// Sort after the first real check, then every CONFIG.roundsPerSort
|
||||
// ticks thereafter
|
||||
if (state.tickCount === 2 || state.tickCount % CONFIG.roundsPerSort === 1) {
|
||||
sortAndRebuildWAN(state);
|
||||
}
|
||||
|
||||
@@ -1206,9 +1216,10 @@ export async function tick(state, signal, onOffline) {
|
||||
|
||||
// --- Recovery Probe ----------------------------------------------------------
|
||||
|
||||
// When offline, check 4 random WAN hosts every 500ms, giving up the checks
|
||||
// started 500ms before, so at most 4 are ever waiting. As soon as one
|
||||
// answers, stop probing and start a new round at once.
|
||||
// When offline, check CONFIG.recoveryProbeHosts random WAN hosts every
|
||||
// CONFIG.recoveryProbeInterval ms, giving up the checks started one interval
|
||||
// before, so at most that many are ever waiting. As soon as one answers,
|
||||
// stop probing and start a new round at once.
|
||||
function startRecoveryProbe(state, startRounds) {
|
||||
if (state._recoveryProbeId) return; // already running
|
||||
const candidates = [...state.wan];
|
||||
@@ -1216,7 +1227,7 @@ function startRecoveryProbe(state, startRounds) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[candidates[i], candidates[j]] = [candidates[j], candidates[i]];
|
||||
}
|
||||
const canaries = candidates.slice(0, 4);
|
||||
const canaries = candidates.slice(0, CONFIG.recoveryProbeHosts);
|
||||
log.notice(
|
||||
`Recovery probe started (${canaries.map((h) => h.name).join(", ")})`,
|
||||
);
|
||||
@@ -1233,7 +1244,7 @@ function startRecoveryProbe(state, startRounds) {
|
||||
startRounds();
|
||||
});
|
||||
}
|
||||
}, 500);
|
||||
}, CONFIG.recoveryProbeInterval);
|
||||
}
|
||||
|
||||
function stopRecoveryProbe(state) {
|
||||
@@ -1497,7 +1508,7 @@ async function init() {
|
||||
});
|
||||
|
||||
window.addEventListener("resize", () => handleResize(state));
|
||||
setTimeout(() => handleResize(state), 100);
|
||||
setTimeout(() => handleResize(state), CONFIG.resizeDelay);
|
||||
}
|
||||
|
||||
// Bootstrap only when loaded as the page: a real DOM containing the #app
|
||||
|
||||
+26
-10
@@ -347,19 +347,35 @@ for (const { history, latencies, statistics } of [
|
||||
},
|
||||
]) {
|
||||
test(`a target's min, max, average and median latency over ${history}`, () => {
|
||||
const host = hostAfter(latencies);
|
||||
assert.deepEqual(
|
||||
{
|
||||
min: host.minLatency(),
|
||||
max: host.maxLatency(),
|
||||
average: host.averageLatency(),
|
||||
median: host.medianLatency(),
|
||||
},
|
||||
statistics,
|
||||
);
|
||||
const { min, max, avg, med } = hostAfter(latencies).historyStats();
|
||||
assert.deepEqual({ min, max, average: avg, median: med }, statistics);
|
||||
});
|
||||
}
|
||||
|
||||
// The summary's figures come from each WAN target's last check, by the same
|
||||
// rules as a target's own: here four answered, one was found unreachable
|
||||
// and the rest have not been checked yet. The median, 22.5, and the
|
||||
// average, 21.25, are rounded.
|
||||
test("the summary's min, max, median and average latency over the WAN targets' last checks", () => {
|
||||
const state = new AppState([]);
|
||||
[30, 10, null, 25, 20].forEach((latency, i) =>
|
||||
state.wan[i].pushSample(
|
||||
Date.now(),
|
||||
latency === null
|
||||
? { latency: null, error: "unreachable" }
|
||||
: { latency, error: null },
|
||||
),
|
||||
);
|
||||
assert.deepEqual(state.wanStats(), {
|
||||
reachable: 4,
|
||||
total: state.wan.length,
|
||||
min: 10,
|
||||
max: 30,
|
||||
med: 23,
|
||||
avg: 21,
|
||||
});
|
||||
});
|
||||
|
||||
// An app state in which, of the WAN targets, the first timedOut timed out,
|
||||
// the next unreachable were found unreachable, the next answered answered
|
||||
// after latency ms, and the rest have not been checked yet.
|
||||
|
||||
Reference in New Issue
Block a user