Serve Prometheus metrics behind SWWAF_METRICS_TOKEN (closes #23)

GET /_smallwebwaf/metrics answers in the Prometheus text format for a
request carrying SWWAF_METRICS_TOKEN, 401 without it and 404 while it is
unset. Every request under /_smallwebwaf/ but the health check now goes
through the checks and is answered where it would be forwarded, 404 for
any path but the metrics, so none reaches the app. SWWAF_METRICS_TOP_N
bounds the series by country, the rest counted as other.

Judgement call: a request answered at smallwebwaf's own endpoints is
neither forwarded nor refused in the client's history.
Deviation: go.mod and go.sum written by hand from the module proxy and
sum.golang.org, as go runs only through make.
Deviation: no metrics yet for state files read again after an edit or
edits set aside; that work is not merged.

Model: opus-5-5

internal/proxy/admin.go

@@ -0,0 +1,40 @@
+package proxy
+
+import (
+	"crypto/subtle"
+	"net/http"
+	"strings"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// answerAdmin answers a request for smallwebwaf itself, under
+// /_smallwebwaf/, once it has passed the checks: GET MetricsPath with
+// SWWAF_METRICS_TOKEN gets the metrics, and without it 401. Any other
+// request gets 404, as the metrics do while SWWAF_METRICS_TOKEN is unset.
+func (rq *request) answerAdmin() {
+	rq.line.Action = requestlog.ActionAdmin
+	rq.startClientResponseTimeout()
+
+	token := rq.h.config.MetricsToken
+
+	switch {
+	case token == "" || rq.in.Method != http.MethodGet || rq.in.URL.Path != MetricsPath:
+		http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+	case !hasToken(rq.in, token):
+		rq.out.Header().Set("WWW-Authenticate", "Bearer")
+		http.Error(rq.out, http.StatusText(http.StatusUnauthorized),
+			http.StatusUnauthorized)
+	default:
+		rq.h.metrics.ServeHTTP(rq.out, rq.in)
+	}
+}
+
+// hasToken reports whether r carries token, as Authorization: Bearer
+// <token>.
+func hasToken(r *http.Request, token string) bool {
+	scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
+
+	return strings.EqualFold(scheme, "Bearer") &&
+		subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
+}

internal/proxy/bans_test.go

@@ -402,36 +402,54 @@ func (s *sender) get(from string, status int, action string) logLine {
 func (s *sender) request(from, path string, status int, action string) logLine {
 	s.t.Helper()
 
+	line, _ := s.requestWithHeader(from, path, "", status, action)
+
+	return line
+}
+
+// requestWithHeader is request with header, such as "Authorization:
+// Bearer x", added to the request unless it is "". It returns the body of
+// the answer too.
+func (s *sender) requestWithHeader(
+	from, path, header string, status int, action string,
+) (logLine, string) {
+	s.t.Helper()
+
+	if header != "" {
+		header += "\r\n"
+	}
+
 	conn := dial(s.t, s.addr)
 	send(s.t, conn, "GET "+path+" HTTP/1.1\r\nHost: "+appHost+
-		"\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n\r\n")
+		"\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n"+
+		header+"\r\n")
 
 	err := conn.SetReadDeadline(time.Now().Add(waitLimit))
 	if err != nil {
 		s.t.Fatalf("set read deadline: %v", err)
 	}
 
-	got := 0
+	var got answer
 
 	res, err := http.ReadResponse(bufio.NewReader(conn), nil)
 
 	switch {
 	case err == nil:
-		got = readAnswer(res).status
+		got = readAnswer(res)
 	case !errors.Is(err, io.ErrUnexpectedEOF):
 		s.t.Fatalf("read response: %v", err)
 	}
 
 	_ = conn.Close()
 
-	if got != status {
-		s.t.Errorf("request %d, from %s: status %d, want %d", s.sent+1, from, got,
-			status)
+	if got.status != status {
+		s.t.Errorf("request %d, from %s: status %d, want %d", s.sent+1, from,
+			got.status, status)
 	}
 
 	line := s.out.requestLines(s.t, s.sent+1)[s.sent]
 	s.sent++
 	wantLine(s.t, line, status, action)
 
-	return line
+	return line, string(got.body)
 }

internal/proxy/bodies.go

@@ -46,6 +46,7 @@ func (b *requestBody) Read(p []byte) (int, error) {
 		b.rq.refuse(refusal{
 			status: http.StatusRequestEntityTooLarge,
 			action: requestlog.ActionTooLarge,
+			limit:  "SWWAF_REQUEST_MAX_BYTES",
 		})
 	}
 
@@ -81,6 +82,7 @@ func (b *responseBody) Read(p []byte) (int, error) {
 		b.rq.refuse(refusal{
 			status: http.StatusBadGateway,
 			action: requestlog.ActionTooLarge,
+			limit:  "SWWAF_RESPONSE_MAX_BYTES",
 		})
 
 		return n, errResponseTooLarge

internal/proxy/history_test.go

@@ -86,6 +86,24 @@ func TestHealthEndpointIsNotInTheHistory(t *testing.T) {
 	}
 }
 
+func TestRequestForSmallwebwafIsNeitherForwardedNorRefused(t *testing.T) {
+	t.Parallel()
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	addr, out, server := startProxyWithClock(t, app.URL, "", time.Now,
+		map[string]string{metricsToken: token})
+
+	scrape(t, addr)
+	wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
+	out.requestLines(t, 2)
+
+	history := historyOf(t, server, localhost)
+	if history.Requests != 2 || history.Forwarded != 0 || history.Refused != 0 {
+		t.Errorf("history counts %d requests, %d forwarded and %d refused, "+
+			"want 2, 0 and 0", history.Requests, history.Forwarded, history.Refused)
+	}
+}
+
 // historyOf returns the history of the client at addr.
 func historyOf(t *testing.T, server *proxy.Server, addr string) ratelimit.History {
 	t.Helper()

internal/proxy/metrics_test.go

@@ -0,0 +1,466 @@
+package proxy_test
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+const (
+	metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
+	metricsTopN  = "SWWAF_METRICS_TOP_N"
+	// token is the SWWAF_METRICS_TOKEN the tests set, and bearer how a
+	// request carries it.
+	token  = "0123456789abcdef0123456789abcdef"
+	bearer = "Bearer " + token
+)
+
+func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
+	t.Parallel()
+
+	var calls atomic.Int32
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {
+		calls.Add(1)
+	})
+	addr, out := startProxy(t, app.URL, nil)
+
+	// An empty token does not match the unset one either.
+	for i, authorization := range []string{bearer, "Bearer ", ""} {
+		req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
+		if authorization != "" {
+			req.Header.Set("Authorization", authorization)
+		}
+
+		wantStatus(t, do(t, req), http.StatusNotFound)
+		wantLine(t, out.requestLines(t, i+1)[i], http.StatusNotFound,
+			requestlog.ActionAdmin)
+	}
+
+	if calls.Load() != 0 {
+		t.Errorf("the app was called %d times, want never", calls.Load())
+	}
+}
+
+func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
+	t.Parallel()
+
+	var calls atomic.Int32
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {
+		calls.Add(1)
+	})
+	addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})
+
+	for i, tc := range []struct {
+		method, path, authorization string
+		status                      int
+	}{
+		{http.MethodGet, proxy.MetricsPath, "", http.StatusUnauthorized},
+		{
+			http.MethodGet, proxy.MetricsPath, "Bearer " + strings.ToUpper(token),
+			http.StatusUnauthorized,
+		},
+		{http.MethodGet, proxy.MetricsPath, "Basic " + token, http.StatusUnauthorized},
+		{http.MethodGet, proxy.MetricsPath, bearer, http.StatusOK},
+		{http.MethodGet, proxy.MetricsPath, "bearer " + token, http.StatusOK},
+		{http.MethodPost, proxy.MetricsPath, bearer, http.StatusNotFound},
+		{http.MethodGet, proxy.MetricsPath + "/", bearer, http.StatusNotFound},
+		{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
+		{http.MethodPost, proxy.HealthPath, "", http.StatusNotFound},
+	} {
+		req := newRequest(t, tc.method, addr, tc.path, http.NoBody)
+		if tc.authorization != "" {
+			req.Header.Set("Authorization", tc.authorization)
+		}
+
+		got := do(t, req)
+		wantStatus(t, got, tc.status)
+		wantLine(t, out.requestLines(t, i+1)[i], tc.status, requestlog.ActionAdmin)
+
+		if tc.status == http.StatusUnauthorized &&
+			got.header.Get("WWW-Authenticate") != "Bearer" {
+			t.Errorf("%q was answered without WWW-Authenticate: Bearer",
+				tc.authorization)
+		}
+
+		if tc.status == http.StatusOK &&
+			!strings.Contains(string(got.body), "# TYPE smallwebwaf_requests_total counter") {
+			t.Errorf("the metrics are\n%s", got.body)
+		}
+	}
+
+	if calls.Load() != 0 {
+		t.Errorf("the app was called %d times, want never", calls.Load())
+	}
+}
+
+func TestMetricsAreAskedForThroughTheChecks(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{
+		metricsToken:       token,
+		rateLimitPerMinute: "1",
+	})
+
+	// Asking for the metrics counts toward the client's limit of one
+	// request a minute, so its next request breaks it, and bans it. A
+	// banned client is refused the metrics too.
+	s.scrape(client)
+	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
+	s.requestWithHeader(client, proxy.MetricsPath, "Authorization: "+bearer,
+		http.StatusForbidden, requestlog.ActionBanned)
+}
+
+func TestMetricsCountTheTraffic(t *testing.T) {
+	t.Parallel()
+
+	arrived, release := make(chan struct{}), make(chan struct{})
+	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.Copy(io.Discard, r.Body)
+
+		if r.URL.Path == "/held" {
+			close(arrived)
+			<-release
+		}
+
+		_, _ = io.WriteString(w, "hello")
+	})
+	releaseApp := sync.OnceFunc(func() { close(release) })
+	t.Cleanup(releaseApp)
+
+	addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})
+
+	got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
+	wantStatus(t, got, http.StatusOK)
+	wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
+	out.requestLines(t, 2)
+
+	forward := `{action="forward",status_class="2xx"}`
+	notFound := `{action="admin",status_class="4xx"}`
+
+	// The request for the metrics is itself under way.
+	metrics := scrape(t, addr)
+	wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
+	wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
+	wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
+	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5)
+	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound,
+		float64(len("Not Found\n")))
+	wantMetric(t, metrics, "smallwebwaf_request_duration_seconds_count", 2)
+	wantMetric(t, metrics, "smallwebwaf_upstream_duration_seconds_count", 1)
+	wantMetric(t, metrics, "smallwebwaf_requests_in_flight", 1)
+	metric(t, metrics, "go_goroutines")
+	metric(t, metrics, "process_start_time_seconds")
+
+	// A request the app holds is under way until it ends.
+	httpClient := newClient(t)
+	held := newRequest(t, http.MethodGet, addr, "/held", http.NoBody)
+	ended := make(chan error, 1)
+
+	go func() {
+		res, err := httpClient.Do(held)
+		if err == nil {
+			err = readAnswer(res).err
+		}
+
+		ended <- err
+	}()
+
+	<-arrived
+	wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 2)
+	releaseApp()
+
+	err := <-ended
+	if err != nil {
+		t.Fatalf("held request: %v", err)
+	}
+
+	out.requestLines(t, 5)
+	wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 1)
+}
+
+func TestMetricsCountLimitsAndBans(t *testing.T) {
+	t.Parallel()
+
+	const (
+		scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
+		denied  = "192.0.2.50"  // in SWWAF_DENY_NETS
+	)
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		metricsToken:        token,
+		rateLimitPerMinute:  "1",
+		rateLimitExemptNets: scraper,
+		denyNets:            denied,
+		banResponse:         "close",
+		limitBanDuration:    "1h",
+		maxBanDuration:      "2h",
+	})
+
+	// SWWAF_BAN_RESPONSE=close sends no status at all.
+	s.get(denied, 0, requestlog.ActionDenied)
+
+	// A first broken limit bans for an hour.
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, 0, requestlog.ActionRateLimited)
+
+	metrics := s.scrape(scraper)
+	wantMetric(t, metrics,
+		`smallwebwaf_requests_total{action="denied",status_class="none"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 1)
+	wantMetric(t, metrics, "smallwebwaf_active_bans", 1)
+	wantMetric(t, metrics, "smallwebwaf_permanent_bans", 0)
+
+	clk.advance(time.Hour)
+	wantMetric(t, s.scrape(scraper), "smallwebwaf_active_bans", 0)
+
+	// A limit broken again right after would ban for three hours, longer
+	// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, 0, requestlog.ActionRateLimited)
+
+	metrics = s.scrape(scraper)
+	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 2)
+	wantMetric(t, metrics, "smallwebwaf_active_bans", 1)
+	wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1)
+	// denied, client, and the scraper as of its earlier requests.
+	wantMetric(t, metrics, "smallwebwaf_tracked_clients", 3)
+}
+
+func TestMetricsCountSizeAndTimeLimits(t *testing.T) {
+	t.Parallel()
+
+	// The app never answers /hang, so the timeout runs out however slowly
+	// the test runs.
+	app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/hang" {
+			<-r.Context().Done()
+		}
+	})
+	addr, out := startProxy(t, app.URL, map[string]string{
+		metricsToken:            token,
+		requestMaxBytes:         sizeLimitSetting,
+		upstreamResponseTimeout: "100ms",
+	})
+
+	body := bytes.NewReader(make([]byte, 2*sizeLimit))
+	wantStatus(t, do(t, newRequest(t, http.MethodPost, addr, "/", body)),
+		http.StatusRequestEntityTooLarge)
+	wantStatus(t, get(t, addr, "/hang"), http.StatusGatewayTimeout)
+	out.requestLines(t, 2)
+
+	metrics := scrape(t, addr)
+	hits := "smallwebwaf_size_and_time_limit_hits_total"
+	wantMetric(t, metrics, hits+`{limit="SWWAF_REQUEST_MAX_BYTES"}`, 1)
+	wantMetric(t, metrics, hits+`{limit="SWWAF_UPSTREAM_RESPONSE_TIMEOUT"}`, 1)
+}
+
+func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
+	t.Parallel()
+
+	const fromFR = "198.51.100.20"
+
+	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = io.WriteString(w, "hello")
+	})
+	env := map[string]string{
+		trustedProxies:  trustLocalhost,
+		metricsToken:    token,
+		metricsTopN:     "2",
+		deniedCountries: "kp",
+	}
+	addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, env)
+
+	// The answers are kept before the requests, so that none waits for
+	// GeoJS.
+	server.GeoJS.Load([]lookup.Answer{
+		keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"), keptAnswer(fromFR, "FR"),
+	})
+
+	lines := 0
+	send := func(from string, times, status int) {
+		t.Helper()
+
+		for range times {
+			req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))
+			req.Header.Set(forwardedFor, from)
+			wantStatus(t, do(t, req), status)
+
+			// Each is counted before the next is sent, so that the
+			// countries are ranked in the order sent.
+			lines++
+			out.requestLines(t, lines)
+		}
+	}
+
+	// With two countries of their own, the third is counted as other.
+	send(fromKP, 3, http.StatusForbidden)
+	send(fromDE, 2, http.StatusOK)
+	send(fromFR, 1, http.StatusOK)
+
+	metrics := scrape(t, addr)
+	lines++
+
+	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3)
+	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_country_list_refusals_total{country="KP"}`, 3)
+	wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="KP"}`, 0)
+	wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`, 6)
+	wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="KP"}`,
+		float64(3*len("Forbidden\n")))
+	wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="other"}`,
+		float64(len("hello")))
+	wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`)
+
+	// Once FR is busier than DE, it takes DE's place: its series counts
+	// from then on, and DE's is gone.
+	send(fromFR, 3, http.StatusOK)
+
+	metrics = scrape(t, addr)
+	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3)
+	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 2)
+	wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`)
+	wantNoSeries(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`)
+}
+
+func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
+	t.Parallel()
+
+	geojs := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}))
+	t.Cleanup(geojs.Close)
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{
+		trustedProxies:  trustLocalhost,
+		metricsToken:    token,
+		deniedCountries: "kp",
+	})
+
+	// GeoJS fails, so the client counts as coming from an unknown country,
+	// which SWWAF_DENIED_COUNTRIES does not refuse.
+	req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
+	req.Header.Set(forwardedFor, fromDE)
+	wantStatus(t, do(t, req), http.StatusOK)
+
+	// The client stops waiting for GeoJS after a second, so GeoJS's
+	// failure can come after its request has ended.
+	deadline := time.Now().Add(waitLimit)
+	metrics := scrape(t, addr)
+
+	for metric(t, metrics, "smallwebwaf_geojs_failures_total") == 0 &&
+		time.Now().Before(deadline) {
+		time.Sleep(pollInterval)
+
+		metrics = scrape(t, addr)
+	}
+
+	wantMetric(t, metrics, "smallwebwaf_geojs_requests_total", 1)
+	wantMetric(t, metrics, "smallwebwaf_geojs_failures_total", 1)
+	wantMetric(t, metrics, "smallwebwaf_geojs_unanswered_total", 1)
+}
+
+// keptAnswer returns GeoJS's answer that the client at addr is in
+// country, given now.
+func keptAnswer(addr, country string) lookup.Answer {
+	now := time.Now()
+
+	return lookup.Answer{
+		Client: netip.MustParsePrefix(addr + "/32"), Country: country,
+		Answered: now, Used: now,
+	}
+}
+
+// scrape asks smallwebwaf at addr for the metrics, with the token, and
+// returns them.
+func scrape(t *testing.T, addr string) string {
+	t.Helper()
+
+	req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
+	req.Header.Set("Authorization", bearer)
+
+	got := do(t, req)
+	if got.status != http.StatusOK {
+		t.Fatalf("the metrics were answered %d", got.status)
+	}
+
+	return string(got.body)
+}
+
+// scrape asks for the metrics, with the token, from the client at from,
+// and returns them.
+func (s *sender) scrape(from string) string {
+	s.t.Helper()
+
+	_, metrics := s.requestWithHeader(from, proxy.MetricsPath, "Authorization: "+bearer,
+		http.StatusOK, requestlog.ActionAdmin)
+
+	return metrics
+}
+
+// metric returns the value of series in metrics, which are in the
+// Prometheus text format. series is a name and its labels in the order of
+// their names, such as smallwebwaf_offences_total{kind="limit"}. It fails
+// the test if there is no such series.
+func metric(t *testing.T, metrics, series string) float64 {
+	t.Helper()
+
+	for line := range strings.Lines(metrics) {
+		value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ")
+		if !found {
+			continue
+		}
+
+		number, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			t.Fatalf("%s has the value %q", series, value)
+		}
+
+		return number
+	}
+
+	t.Fatalf("no series %s in the metrics:\n%s", series, metrics)
+
+	return 0
+}
+
+// wantMetric checks the value of series in metrics, as metric reads it.
+func wantMetric(t *testing.T, metrics, series string, want float64) {
+	t.Helper()
+
+	got := metric(t, metrics, series)
+	if got != want {
+		t.Errorf("%s is %v, want %v", series, got, want)
+	}
+}
+
+// wantNoSeries checks that metrics have no series series.
+func wantNoSeries(t *testing.T, metrics, series string) {
+	t.Helper()
+
+	if strings.Contains(metrics, "\n"+series+" ") {
+		t.Errorf("there is a series %s", series)
+	}
+}

internal/proxy/proxy.go

@@ -8,11 +8,13 @@ import (
 	"log"
 	"log/slog"
 	"net/http"
+	"strings"
 	"time"
 
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
@@ -23,10 +25,18 @@ const (
 	appIdleConnTimeout = 90 * time.Second
 )
 
+// adminPrefix starts the path of every request for smallwebwaf itself,
+// which never reaches the app.
+const adminPrefix = "/_smallwebwaf/"
+
 // HealthPath is smallwebwaf's health endpoint, which the container's
 // health check asks.
 const HealthPath = "/_smallwebwaf/healthz"
 
+// MetricsPath is where the metrics are, for a request that carries
+// SWWAF_METRICS_TOKEN.
+const MetricsPath = "/_smallwebwaf/metrics"
+
 // Params are what New needs.
 type Params struct {
 	Config *config.Config
@@ -44,13 +54,14 @@ type Params struct {
 }
 
 // Server is the server smallwebwaf runs, with the parts of the proxy
-// whose state the state files keep.
+// whose state the state files keep, and the metrics.
 type Server struct {
 	*http.Server
 
 	Ledger  *bans.Ledger
 	Limiter *ratelimit.Limiter
 	GeoJS   *lookup.GeoJS
+	Metrics *metrics.Metrics
 }
 
 // New returns the server smallwebwaf runs: each request it reads passes
@@ -61,6 +72,7 @@ type Server struct {
 // applies the timeouts and size limits from then on.
 func New(params Params) *Server {
 	errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
+	m := metrics.New(params.Config.MetricsTopN)
 	h := &handler{
 		config:     params.Config,
 		requestLog: params.RequestLog,
@@ -68,6 +80,7 @@ func New(params Params) *Server {
 		errorLog:   errorLog,
 		transport:  newTransport(),
 		now:        params.Now,
+		metrics:    m,
 		limiter: ratelimit.New(ratelimit.Limits{
 			PerMinute: params.Config.RateLimitPerMinute,
 			PerHour:   params.Config.RateLimitPerHour,
@@ -83,8 +96,10 @@ func New(params Params) *Server {
 			URL:        params.GeoJSURL,
 			Now:        params.Now,
 			ProcessLog: params.ProcessLog,
+			Metrics:    m,
 		}),
 	}
+	m.AddBansAndClients(h.ledger, h.limiter, params.Now)
 
 	return &Server{
 		Server: &http.Server{
@@ -102,6 +117,7 @@ func New(params Params) *Server {
 		Ledger:  h.ledger,
 		Limiter: h.limiter,
 		GeoJS:   h.geojs,
+		Metrics: m,
 	}
 }
 
@@ -114,6 +130,7 @@ type handler struct {
 	errorLog   *log.Logger
 	transport  http.RoundTripper
 	now        func() time.Time
+	metrics    *metrics.Metrics
 	limiter    *ratelimit.Limiter
 	ledger     *bans.Ledger
 	geojs      *lookup.GeoJS
@@ -133,7 +150,8 @@ func newTransport() *http.Transport {
 
 // ServeHTTP handles one request: it works out the client, runs the
 // checks, passes the request to the app and the answer back within the
-// limits, and writes the request's log line.
+// limits, or answers it itself if it is for smallwebwaf, and writes the
+// request's log line.
 func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	rq := h.newRequest(w, r)
 	defer rq.finish()
@@ -157,5 +175,13 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// A request for smallwebwaf itself is answered where another would be
+	// passed to the app, so that it goes through every check first.
+	if strings.HasPrefix(r.URL.Path, adminPrefix) {
+		rq.answerAdmin()
+
+		return
+	}
+
 	rq.forward(r.Context())
 }

internal/proxy/request.go

@@ -22,11 +22,13 @@ const flushAfterEachWrite time.Duration = -1
 
 // refusal is smallwebwaf refusing a request, or refusing to go on with it:
 // the status the client is answered if the response has not started yet,
-// 0 to close the connection without an answer, and the action the log
-// line names.
+// 0 to close the connection without an answer, the action the log line
+// names, and the setting whose size or time limit the request passed, if
+// that is why.
 type refusal struct {
 	status int
 	action string
+	limit  string
 }
 
 // request is one request on its way through smallwebwaf, from the moment
@@ -65,9 +67,11 @@ type request struct {
 	requestSent time.Time
 }
 
-// newRequest starts handling r: it notes the time and works out the
-// client.
+// newRequest starts handling r: it notes the time, counts the request as
+// under way, and works out the client.
 func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
+	h.metrics.RequestStarted()
+
 	start := time.Now()
 	peer := peerAddress(r)
 	trusted := h.config.TrustedProxies
@@ -141,6 +145,7 @@ func (rq *request) check(ctx context.Context) *refusal {
 		return &refusal{
 			status: http.StatusRequestEntityTooLarge,
 			action: requestlog.ActionTooLarge,
+			limit:  "SWWAF_REQUEST_MAX_BYTES",
 		}
 	}
 
@@ -206,7 +211,11 @@ func (rq *request) modifyResponse(res *http.Response) error {
 
 	maxBytes := rq.h.config.ResponseMaxBytes
 	if maxBytes > 0 && res.Body != http.NoBody && res.ContentLength > maxBytes {
-		rq.refuse(refusal{status: http.StatusBadGateway, action: requestlog.ActionTooLarge})
+		rq.refuse(refusal{
+			status: http.StatusBadGateway,
+			action: requestlog.ActionTooLarge,
+			limit:  "SWWAF_RESPONSE_MAX_BYTES",
+		})
 
 		return errResponseTooLarge
 	}
@@ -278,7 +287,8 @@ func (rq *request) refuse(r refusal) {
 	rq.cancel()
 }
 
-// finish ends the request's timeouts and writes its log line.
+// finish ends the request's timeouts, counts it in the metrics and writes
+// its log line.
 func (rq *request) finish() {
 	rq.stopTimers()
 
@@ -295,24 +305,37 @@ func (rq *request) finish() {
 		line.RequestBytes = rq.body.bytes.Load()
 	}
 
+	// limit is the setting whose size or time limit the request passed.
+	var limit string
+
 	switch {
 	case refused != nil:
 		line.Action = refused.action
+		limit = refused.limit
 	case errors.Is(rq.out.err, os.ErrDeadlineExceeded):
 		// The client took longer than SWWAF_CLIENT_RESPONSE_TIMEOUT to
 		// take the response.
 		line.Action = requestlog.ActionTimedOut
+		limit = "SWWAF_CLIENT_RESPONSE_TIMEOUT"
 	case !rq.complete && (rq.out.err != nil || rq.in.Context().Err() != nil):
 		line.Aborted = true
 	}
 
 	now := time.Now()
-	line.DurationTotal = requestlog.Milliseconds(now.Sub(rq.start))
+	duration := now.Sub(rq.start)
+	line.DurationTotal = requestlog.Milliseconds(duration)
+
+	var upstreamDuration time.Duration
 
 	if !rq.upstreamStart.IsZero() {
-		line.DurationUpstreamTotal = requestlog.Milliseconds(now.Sub(rq.upstreamStart))
+		upstreamDuration = now.Sub(rq.upstreamStart)
+		line.DurationUpstreamTotal = requestlog.Milliseconds(upstreamDuration)
 	}
 
+	// Counted before the log line is written, so that the metrics count
+	// every request whose line is out.
+	rq.h.metrics.RequestEnded(line, limit, duration, upstreamDuration)
+
 	err := requestlog.Write(rq.h.requestLog, line)
 	if err != nil {
 		rq.h.processLog.Error("writing the request log failed", "error", err.Error())
@@ -327,9 +350,12 @@ func (rq *request) addToHistory() {
 		requestBytes = rq.body.bytes.Load()
 	}
 
+	forwarded := !rq.upstreamStart.IsZero()
+
 	rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{
 		Country:       rq.line.Country,
-		Forwarded:     !rq.upstreamStart.IsZero(),
+		Forwarded:     forwarded,
+		Refused:       !forwarded && rq.refused.Load() != nil,
 		Status:        rq.out.status,
 		RequestBytes:  requestBytes,
 		ResponseBytes: rq.out.bytes,
@@ -369,21 +395,26 @@ func (rq *request) startRequestTimers() {
 
 	if rq.body != nil && rq.h.config.ClientRequestTimeout > 0 {
 		rq.clientRequestTimer = time.AfterFunc(
-			time.Until(rq.clientRequestDeadline()), rq.requestTimedOut)
+			time.Until(rq.clientRequestDeadline()), func() {
+				rq.requestTimedOut("SWWAF_CLIENT_REQUEST_TIMEOUT")
+			})
 	}
 
 	timeout := rq.h.config.UpstreamRequestTimeout
 	if timeout > 0 {
-		rq.upstreamRequestTimer = time.AfterFunc(timeout, rq.requestTimedOut)
+		rq.upstreamRequestTimer = time.AfterFunc(timeout, func() {
+			rq.requestTimedOut("SWWAF_UPSTREAM_REQUEST_TIMEOUT")
+		})
 	}
 }
 
-// requestTimedOut is called when a request timeout runs out while the
-// request is still on its way to the app. The answer names the side
-// smallwebwaf was waiting on at that moment: 408 when it was waiting for
-// the client to send more of its body, 504 when it was waiting for the
-// app to be reached or to take what it had.
-func (rq *request) requestTimedOut() {
+// requestTimedOut is called when limit, SWWAF_CLIENT_REQUEST_TIMEOUT or
+// SWWAF_UPSTREAM_REQUEST_TIMEOUT, runs out while the request is still on
+// its way to the app. The answer names the side smallwebwaf was waiting
+// on at that moment: 408 when it was waiting for the client to send more
+// of its body, 504 when it was waiting for the app to be reached or to
+// take what it had.
+func (rq *request) requestTimedOut(limit string) {
 	rq.mu.Lock()
 	defer rq.mu.Unlock()
 
@@ -395,6 +426,7 @@ func (rq *request) requestTimedOut() {
 		rq.refuse(refusal{
 			status: http.StatusGatewayTimeout,
 			action: requestlog.ActionTimedOut,
+			limit:  limit,
 		})
 
 		return
@@ -403,6 +435,7 @@ func (rq *request) requestTimedOut() {
 	rq.refuse(refusal{
 		status: http.StatusRequestTimeout,
 		action: requestlog.ActionTimedOut,
+		limit:  limit,
 	})
 	// The transport gives up on the app only once its Read of the
 	// client's body returns, so that Read is ended now. The lock keeps
@@ -452,6 +485,7 @@ func (rq *request) responseTimedOut() {
 		rq.refuse(refusal{
 			status: http.StatusGatewayTimeout,
 			action: requestlog.ActionTimedOut,
+			limit:  "SWWAF_UPSTREAM_RESPONSE_TIMEOUT",
 		})
 	}
 }