Serve Prometheus metrics at /metrics behind basic auth (closes #94)

With METRICS_USERNAME and METRICS_PASSWORD both set, the backend
records request metrics through go-http-metrics in a registry of its
own, with Go's runtime and process metrics, and serves them at
GET /metrics behind basic auth with those credentials; nginx passes
/metrics to it. With neither set there is no such route; one alone
stops the start with an error naming both.

Only requests that reach the health check or POST /api/v1/reports are
recorded, not every request as the conventions show: the labels are
path and method, which clients can make up without end. So
POST /api/v1/reports is registered by its full path, not inside a
route group.

Deviation: go get and go mod tidy ran directly; no entrypoint adds a Go
dependency yet (#45).

Model: opus-5-5

backend/internal/config/config.go

@@ -44,6 +44,11 @@ var (
 	errNotPort = errors.New("must be a port number, 1 to 65535")
 	errNotBool = errors.New("must be true or false")
 	errNotIP   = errors.New("must be an IP address, or empty")
+
+	errMetricsCredentials = errors.New(
+		"METRICS_USERNAME and METRICS_PASSWORD must be set together, " +
+			"or neither",
+	)
 )
 
 // Params defines the dependencies for Config.
@@ -178,6 +183,12 @@ func (s *Config) check() error {
 		}
 	}
 
+	// The server records and serves metrics only with both set, so
+	// one alone is a mistake that would otherwise go unnoticed.
+	if (s.MetricsUsername == "") != (s.MetricsPassword == "") {
+		return errMetricsCredentials
+	}
+
 	return checkOrigins(s.CORSAllowedOrigins)
 }
 

backend/internal/config/config_test.go

@@ -103,6 +103,22 @@ func TestDataDirMaxBytesMustBeANumber(t *testing.T) {
 	requireConfigError(t, "DATA_DIR_MAX_BYTES")
 }
 
+// TestMetricsCredentialsGoTogether: with only one of the two set, the
+// server would quietly serve no metrics, so the start fails, naming
+// both.
+func TestMetricsCredentialsGoTogether(t *testing.T) {
+	for _, set := range []string{"METRICS_USERNAME", "METRICS_PASSWORD"} {
+		t.Run(set, func(t *testing.T) {
+			t.Setenv("METRICS_USERNAME", "")
+			t.Setenv("METRICS_PASSWORD", "")
+			t.Setenv(set, "prometheus")
+
+			requireConfigError(t, "METRICS_USERNAME")
+			requireConfigError(t, "METRICS_PASSWORD")
+		})
+	}
+}
+
 // TestCORSAllowedOriginsMustBeOrigins: "*" would let every origin in,
 // and an entry that is not a plain origin would match no page.
 func TestCORSAllowedOriginsMustBeOrigins(t *testing.T) {

backend/internal/middleware/middleware.go

@@ -18,9 +18,14 @@ import (
 	"sneak.berlin/go/netwatch/internal/globals"
 	"sneak.berlin/go/netwatch/internal/logger"
 
+	basicauth "github.com/99designs/basicauth-go"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
 	"github.com/go-chi/httprate"
+	"github.com/prometheus/client_golang/prometheus"
+	metrics "github.com/slok/go-http-metrics/metrics/prometheus"
+	ghmm "github.com/slok/go-http-metrics/middleware"
+	"github.com/slok/go-http-metrics/middleware/std"
 	"go.uber.org/fx"
 )
 
@@ -364,3 +369,25 @@ func (s *Middleware) RateLimit(
 		),
 	)
 }
+
+// Metrics returns middleware that records each request's duration and
+// response size, and the requests in progress, in registry. They are
+// labelled by the request path.
+func (s *Middleware) Metrics(
+	registry prometheus.Registerer,
+) func(http.Handler) http.Handler {
+	mdlw := ghmm.New(ghmm.Config{
+		Recorder: metrics.NewRecorder(metrics.Config{Registry: registry}),
+	})
+
+	return std.HandlerProvider("", mdlw)
+}
+
+// MetricsAuth returns middleware that lets a request through only with
+// METRICS_USERNAME and METRICS_PASSWORD as its basic auth credentials,
+// and answers any other with 401.
+func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
+	return basicauth.New("metrics", map[string][]string{
+		s.params.Config.MetricsUsername: {s.params.Config.MetricsPassword},
+	})
+}

backend/internal/server/routes.go

@@ -5,6 +5,9 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
@@ -29,13 +32,37 @@ func (s *Server) SetupRoutes() {
 	s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
 	s.router.Use(middleware.Timeout(requestTimeout))
 
-	s.router.Get(
-		"/.well-known/healthcheck",
-		s.h.HandleHealthCheck(),
+	// The metrics go in a registry of this server's own, not in
+	// Prometheus' default one, which takes them only once per process.
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
 	)
 
-	s.router.Route("/api/v1", func(r chi.Router) {
+	// Requests are measured only once chi has matched them to one of
+	// these routes, by path and method. The metrics are labelled with
+	// both, which any client can make up, so measuring every request
+	// would let clients add labels without bound. A Route here would
+	// be matched by its path prefix alone, so each path is given in
+	// full.
+	s.router.Group(func(r chi.Router) {
+		// config.New refuses one of the two credentials without the
+		// other.
+		if s.params.Config.MetricsUsername != "" {
+			r.Use(s.mw.Metrics(registry))
+		}
+
+		r.Get("/.well-known/healthcheck", s.h.HandleHealthCheck())
+
 		r.With(s.mw.RateLimit(s.params.Config.ReportsPerMinute)).
-			Post("/reports", s.h.HandleReport())
+			Post("/api/v1/reports", s.h.HandleReport())
 	})
+
+	if s.params.Config.MetricsUsername != "" {
+		s.router.With(s.mw.MetricsAuth()).
+			Get("/metrics", promhttp.HandlerFor(
+				registry, promhttp.HandlerOpts{},
+			).ServeHTTP)
+	}
 }

backend/internal/server/routes_test.go

@@ -107,6 +107,95 @@ func TestCORSAllowedOriginsReachTheRouter(t *testing.T) {
 	}
 }
 
+// TestNoMetricsWithoutCredentials: with neither metrics setting set,
+// there is no /metrics.
+func TestNoMetricsWithoutCredentials(t *testing.T) {
+	t.Setenv("METRICS_USERNAME", "")
+	t.Setenv("METRICS_PASSWORD", "")
+
+	srv := newServer(t)
+	srv.SetupRoutes()
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(t.Context(),
+		http.MethodGet, "/metrics", http.NoBody)
+	srv.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
+	}
+}
+
+// TestMetricsBehindBasicAuth: with both metrics settings set, /metrics
+// answers only with them as basic auth credentials, and shows a
+// request to a route but not one to a path no route has.
+func TestMetricsBehindBasicAuth(t *testing.T) {
+	t.Setenv("METRICS_USERNAME", "prometheus")
+	t.Setenv("METRICS_PASSWORD", "right")
+
+	srv := newServer(t)
+	srv.SetupRoutes()
+
+	get := func(path, username, password string) *httptest.ResponseRecorder {
+		rec := httptest.NewRecorder()
+		req := httptest.NewRequestWithContext(t.Context(),
+			http.MethodGet, path, http.NoBody)
+
+		if username != "" {
+			req.SetBasicAuth(username, password)
+		}
+
+		srv.ServeHTTP(rec, req)
+
+		return rec
+	}
+
+	get("/.well-known/healthcheck", "", "")
+	get("/api/v1/no-such-route", "", "")
+
+	for _, creds := range [][2]string{
+		{"", ""},
+		{"prometheus", "wrong"},
+		{"someone", "right"},
+	} {
+		rec := get("/metrics", creds[0], creds[1])
+		if rec.Code != http.StatusUnauthorized {
+			t.Errorf("credentials %q: status = %d, want %d",
+				creds, rec.Code, http.StatusUnauthorized)
+		}
+	}
+
+	rec := get("/metrics", "prometheus", "right")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("right credentials: status = %d, want %d",
+			rec.Code, http.StatusOK)
+	}
+
+	body := rec.Body.String()
+	if !strings.Contains(body, `handler="/.well-known/healthcheck"`) {
+		t.Errorf("metrics show no health check request:\n%s", body)
+	}
+
+	if strings.Contains(body, "no-such-route") {
+		t.Errorf("metrics show a request to a path no route has:\n%s", body)
+	}
+
+	if !strings.Contains(body, "go_goroutines") {
+		t.Errorf("metrics show no Go runtime metrics:\n%s", body)
+	}
+}
+
+// TestMetricsInTwoServers: two servers in one process can both have
+// metrics on.
+func TestMetricsInTwoServers(t *testing.T) {
+	t.Setenv("METRICS_USERNAME", "prometheus")
+	t.Setenv("METRICS_PASSWORD", "right")
+
+	for range 2 {
+		newServer(t).SetupRoutes()
+	}
+}
+
 // TestHealthCheckRejectsOversizeBody sends the health check, which
 // never reads its body, a body one byte over the limit. Only the
 // router-wide body limit can reject it.