rproxy and servewaf replace the one-process smallwebwaf (closes #172, closes #165)

`smallwebwaf` with no command now prints the commands and exits with status 2.
The image runs rproxy, or servewaf with SMALLWEBWAF_COMMAND=servewaf. The
passing-on code, its settings and its log fields and metric go. internal/proxy
keeps the checks, answers, reports and own endpoints; its tests run through the
real rproxy. The documents describe the pair.

Judgement call: clients.json keeps token_refused, since state files refuse unknown fields.
Deviation: script/gitea-requests sets SWWAF_LOG_REQUEST_HEADERS, to label requests.
Partially verified: make example-app and make gitea-requests never built the image, as Ubuntu's snapshot service answered 5xx.

Model: opus-5-5

internal/proxy/metrics_test.go

@@ -7,8 +7,6 @@ import (
 	"net/netip"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -30,41 +28,21 @@ const (
 func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
 	t.Parallel()
 
-	var calls atomic.Int32
-
-	app := startApp(t, func(http.ResponseWriter, *http.Request) {
-		calls.Add(1)
-	})
-	addr, out := startProxy(t, app.URL, nil)
+	s, _, _ := startWithClock(t, "", nil)
 
 	// An empty token does not match the unset one either.
-	for i, authorization := range []string{bearer, "Bearer ", ""} {
-		req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
-		if authorization != "" {
-			req.Header.Set("Authorization", authorization)
-		}
-
-		wantStatus(t, do(t, req), http.StatusNotFound)
-		wantLine(t, out.requestLines(t, i+1)[i], http.StatusNotFound,
-			requestlog.ActionAdmin)
-	}
-
-	if calls.Load() != 0 {
-		t.Errorf("the app was called %d times, want never", calls.Load())
+	for _, authorization := range []string{bearer, "Bearer ", ""} {
+		s.adminRequest(authorization, http.MethodGet, proxy.MetricsPath, "",
+			http.StatusNotFound)
 	}
 }
 
 func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
 	t.Parallel()
 
-	var calls atomic.Int32
+	s, _, _ := startWithClock(t, "", map[string]string{metricsToken: token})
 
-	app := startApp(t, func(http.ResponseWriter, *http.Request) {
-		calls.Add(1)
-	})
-	addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})
-
-	for i, tc := range []struct {
+	for _, tc := range []struct {
 		method, path, authorization string
 		status                      int
 	}{
@@ -81,14 +59,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
 		{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
 		{http.MethodPost, proxy.HealthPath, "", http.StatusNotFound},
 	} {
-		req := newRequest(t, tc.method, addr, tc.path, http.NoBody)
-		if tc.authorization != "" {
-			req.Header.Set("Authorization", tc.authorization)
-		}
-
-		got := do(t, req)
-		wantStatus(t, got, tc.status)
-		wantLine(t, out.requestLines(t, i+1)[i], tc.status, requestlog.ActionAdmin)
+		got := s.adminRequest(tc.authorization, tc.method, tc.path, "", tc.status)
 
 		if tc.status == http.StatusUnauthorized &&
 			got.header.Get("WWW-Authenticate") != "Bearer" {
@@ -101,58 +72,28 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
 			t.Errorf("the metrics are\n%s", got.body)
 		}
 	}
-
-	if calls.Load() != 0 {
-		t.Errorf("the app was called %d times, want never", calls.Load())
-	}
-}
-
-func TestMetricsAreAskedForThroughTheChecks(t *testing.T) {
-	t.Parallel()
-
-	s, _, _ := startWithClock(t, "", map[string]string{
-		metricsToken:       token,
-		rateLimitPerMinute: "1",
-	})
-
-	// Asking for the metrics counts toward the client's limit of one
-	// request a minute, so its next request breaks it, and bans it. A
-	// banned client is refused the metrics too.
-	s.scrape(client)
-	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
-	s.requestWithHeader(client, proxy.MetricsPath, "Authorization: "+bearer,
-		http.StatusForbidden, requestlog.ActionBanned)
 }
 
 func TestMetricsCountTheTraffic(t *testing.T) {
 	t.Parallel()
 
-	arrived, release := make(chan struct{}), make(chan struct{})
 	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
 		_, _ = io.Copy(io.Discard, r.Body)
-
-		if r.URL.Path == "/held" {
-			close(arrived)
-			<-release
-		}
-
 		_, _ = io.WriteString(w, "hello")
 	})
-	releaseApp := sync.OnceFunc(func() { close(release) })
-	t.Cleanup(releaseApp)
-
-	addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})
+	addr, out, server := startProxyWithClock(t, app.URL, "", time.Now,
+		map[string]string{metricsToken: token})
 
 	got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
 	wantStatus(t, got, http.StatusOK)
 	wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
-	out.requestLines(t, 2)
+	out.endedLines(t, 2)
 
 	forward := `{action="forward",instance="app",status_class="2xx"}`
 	notFound := `{action="admin",instance="app",status_class="4xx"}`
 
 	// The request for the metrics is itself under way.
-	metrics := scrape(t, addr)
+	metrics := scrape(t, server.Addr)
 	wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
 	wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
 	wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
@@ -166,50 +107,20 @@ func TestMetricsCountTheTraffic(t *testing.T) {
 	wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
 	metric(t, metrics, `go_goroutines{instance="app"}`)
 	metric(t, metrics, `process_start_time_seconds{instance="app"}`)
-
-	// A request the app holds is under way until it ends.
-	httpClient := newClient(t)
-	held := newRequest(t, http.MethodGet, addr, "/held", http.NoBody)
-	ended := make(chan error, 1)
-
-	go func() {
-		res, err := httpClient.Do(held)
-		if err == nil {
-			err = readAnswer(res).err
-		}
-
-		ended <- err
-	}()
-
-	<-arrived
-	wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 2)
-	releaseApp()
-
-	err := <-ended
-	if err != nil {
-		t.Fatalf("held request: %v", err)
-	}
-
-	out.requestLines(t, 5)
-	wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 1)
 }
 
 func TestMetricsCountLimitsAndBans(t *testing.T) {
 	t.Parallel()
 
-	const (
-		scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
-		denied  = "192.0.2.50"  // in SWWAF_DENY_NETS
-	)
+	const denied = "192.0.2.50" // in SWWAF_DENY_NETS
 
 	s, clk, _ := startWithClock(t, "", map[string]string{
-		metricsToken:        token,
-		rateLimitPerMinute:  "1",
-		rateLimitExemptNets: scraper,
-		denyNets:            denied,
-		banResponse:         banResponseClose,
-		limitBanDuration:    "1h",
-		maxBanDuration:      "2h",
+		metricsToken:       token,
+		rateLimitPerMinute: "1",
+		denyNets:           denied,
+		banResponse:        banResponseClose,
+		limitBanDuration:   "1h",
+		maxBanDuration:     "2h",
 	})
 
 	// SWWAF_BAN_RESPONSE=close sends no status at all.
@@ -219,7 +130,7 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
 	s.get(client, http.StatusOK, requestlog.ActionForward)
 	s.get(client, 0, requestlog.ActionRateLimited)
 
-	metrics := s.scrape(scraper)
+	metrics := s.scrape()
 	wantMetric(t, metrics,
 		`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
 	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
@@ -230,38 +141,35 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
 	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)
 
 	clk.advance(time.Hour)
-	wantMetric(t, s.scrape(scraper), `smallwebwaf_active_bans{instance="app"}`, 0)
+	wantMetric(t, s.scrape(), `smallwebwaf_active_bans{instance="app"}`, 0)
 
 	// A limit broken again right after would ban for three hours, longer
 	// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
 	s.get(client, http.StatusOK, requestlog.ActionForward)
 	s.get(client, 0, requestlog.ActionRateLimited)
 
-	metrics = s.scrape(scraper)
+	metrics = s.scrape()
 	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
 		`kind="requests",window="minute"}`, 2)
 	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
 	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
 	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
 	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
-	// denied, client, and the scraper as of its earlier requests.
-	wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 3)
+	// denied and client.
+	wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 2)
 }
 
 func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
 	t.Parallel()
 
-	const scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
-
 	s, clk, server := startWithClock(t, "", map[string]string{
-		metricsToken:        token,
-		adminToken:          adminSecret,
-		rateLimitExemptNets: scraper,
+		metricsToken: token,
+		adminToken:   adminSecret,
 	})
 
 	const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`
 
-	wantMetric(t, s.scrape(scraper), admins, 0)
+	wantMetric(t, s.scrape(), admins, 0)
 
 	// As an admin's edit of bans.json that adds a ban is taken in.
 	server.Ledger.LoadEdit([]bans.Ban{{
@@ -269,11 +177,11 @@ func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
 		Start:    clk.Now(),
 	}})
 
-	wantMetric(t, s.scrape(scraper), admins, 1)
+	wantMetric(t, s.scrape(), admins, 1)
 
 	// And a ban made through the endpoint.
 	s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
-	wantMetric(t, s.scrape(scraper), admins, 2)
+	wantMetric(t, s.scrape(), admins, 2)
 }
 
 func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
@@ -312,7 +220,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
 			// Each is counted before the next is sent, so that the
 			// countries are ranked in the order sent.
 			lines++
-			out.requestLines(t, lines)
+			out.endedLines(t, lines)
 		}
 	}
 
@@ -321,7 +229,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
 	send(fromDE, 2, http.StatusOK)
 	send(fromFR, 1, http.StatusOK)
 
-	metrics := scrape(t, addr)
+	metrics := scrape(t, server.Addr)
 	lines++
 
 	wantMetric(t, metrics,
@@ -349,7 +257,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
 	// from then on, and DE's is gone.
 	send(fromFR, 3, http.StatusOK)
 
-	metrics = scrape(t, addr)
+	metrics = scrape(t, server.Addr)
 	wantMetric(t, metrics,
 		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
 	wantMetric(t, metrics,
@@ -383,13 +291,12 @@ func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
 		answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
 	})
 
-	// With one AS number of its own, the other is counted as other. The
-	// metrics are asked for from a private address, which has no AS number.
+	// With one AS number of its own, the other is counted as other.
 	s.get(fromDE, http.StatusOK, requestlog.ActionForward)
 	s.get(fromDE, http.StatusOK, requestlog.ActionForward)
 	s.get(fromKP, http.StatusOK, requestlog.ActionForward)
 
-	metrics := s.scrape("10.0.0.9")
+	metrics := s.scrape()
 	wantMetric(t, metrics,
 		`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
 	wantMetric(t, metrics,
@@ -412,11 +319,12 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
 	t.Cleanup(geojs.Close)
 
 	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
-	addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{
-		trustedProxies:  trustLocalhost,
-		metricsToken:    token,
-		deniedCountries: "kp",
-	})
+	addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
+		map[string]string{
+			trustedProxies:  trustLocalhost,
+			metricsToken:    token,
+			deniedCountries: "kp",
+		})
 
 	// GeoJS fails, so the client counts as coming from an unknown country,
 	// which SWWAF_DENIED_COUNTRIES does not refuse.
@@ -427,13 +335,13 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
 	// The client stops waiting for GeoJS once SWWAF_LOOKUP_TIMEOUT is up, so
 	// GeoJS's failure can come after its request has ended.
 	deadline := time.Now().Add(waitLimit)
-	metrics := scrape(t, addr)
+	metrics := scrape(t, server.Addr)
 
 	for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
 		time.Now().Before(deadline) {
 		time.Sleep(pollInterval)
 
-		metrics = scrape(t, addr)
+		metrics = scrape(t, server.Addr)
 	}
 
 	wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
@@ -452,8 +360,8 @@ func keptAnswer(addr, country string) lookup.Answer {
 	}
 }
 
-// scrape asks smallwebwaf at addr for the metrics, with the token, and
-// returns them.
+// scrape asks servewaf at addr for the metrics, with the token, and returns
+// them.
 func scrape(t *testing.T, addr string) string {
 	t.Helper()
 
@@ -468,15 +376,12 @@ func scrape(t *testing.T, addr string) string {
 	return string(got.body)
 }
 
-// scrape asks for the metrics, with the token, from the client at from,
-// and returns them.
-func (s *sender) scrape(from string) string {
+// scrape asks for the metrics, with the token, and returns them.
+func (s *sender) scrape() string {
 	s.t.Helper()
 
-	_, metrics := s.requestWithHeader(from, proxy.MetricsPath, "Authorization: "+bearer,
-		http.StatusOK, requestlog.ActionAdmin)
-
-	return metrics
+	return string(s.adminRequest(bearer, http.MethodGet, proxy.MetricsPath, "",
+		http.StatusOK).body)
 }
 
 // metric returns the value of series in metrics, which are in the
@@ -524,22 +429,3 @@ func wantNoSeries(t *testing.T, metrics, series string) {
 		t.Errorf("there is a series %s", series)
 	}
 }
-
-// wantLimitHits checks that the metrics of smallwebwaf at addr count hits
-// requests that passed the size or time limit of the setting limit, with
-// no series for it when hits is 0.
-func wantLimitHits(t *testing.T, addr, limit string, hits int) {
-	t.Helper()
-
-	series := `smallwebwaf_size_and_time_limit_hits_total{instance="app",limit="` +
-		limit + `"}`
-	metrics := scrape(t, addr)
-
-	if hits == 0 {
-		wantNoSeries(t, metrics, series)
-
-		return
-	}
-
-	wantMetric(t, metrics, series, float64(hits))
-}