Core Rule Set tests: check the recorded requests without the race detector (closes #290)

The tests that check the apps' recorded requests move to
internal/coreruleset/recorded_test.go, behind the recorded build tag. The
test phase runs only those tests in a second go test, without -race. The
lint phase reads the same file, with the same tag.

The Gitea and WordPress tests are split in two. Their recorded-request loops
become TestOrdinaryGiteaRequestsPass and TestOrdinaryWordPressRequestsPass;
their attack checks stay in the race pass as TestGiteaAttacksAreRefused and
TestWordPressAttacksAreRefused.

Model: opus-5-5

internal/coreruleset/recorded_test.go

@@ -0,0 +1,204 @@
+//go:build recorded
+
+// The tests that check the apps' recorded requests in testdata. The recorded
+// tag keeps them out of the test phase's pass with the race detector, and
+// its second pass, without the race detector, runs them.
+
+package coreruleset_test
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"sneak.berlin/go/smallwebwaf/internal/coreruleset"
+)
+
+// giteaRequests returns the recorded ordinary gitea requests in testdata.
+// Each has the Referer of a page that names the same, as a page opened
+// from it has.
+func giteaRequests(t *testing.T) []request {
+	t.Helper()
+
+	text, err := os.ReadFile(filepath.Join("testdata", "gitea-requests.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var requests []request
+
+	for line := range strings.Lines(string(text)) {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		target, header, _ := strings.Cut(line, " ")
+
+		headers := []string{"Referer: http://git.example" + target}
+		if header != "" {
+			headers = append(headers, header)
+		}
+
+		requests = append(requests, get(target, headers...))
+	}
+
+	return requests
+}
+
+func TestOrdinaryGiteaRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	const threshold = 5 // SWW_WAF_ANOMALY_THRESHOLD's default
+
+	crs := atDefaults(t)
+
+	for _, r := range giteaRequests(t) {
+		if got := inspect(t, crs, r); got.Score >= threshold {
+			t.Errorf("%s %q refused: %+v", r.target, r.headers[1:], got)
+		}
+	}
+}
+
+// recordedRequest is a line of a file of recorded requests in testdata.
+type recordedRequest struct {
+	Method  string   `json:"method"`
+	Target  string   `json:"target"`
+	Headers []string `json:"headers"`
+	Body    string   `json:"body"`
+}
+
+// recordedRequests returns the recorded ordinary requests in the file name
+// in testdata, each with its body.
+func recordedRequests(t *testing.T, name string) []recordedRequest {
+	t.Helper()
+
+	path := filepath.Join("testdata", name)
+
+	text, err := os.ReadFile(path) //nolint:gosec // a file in testdata
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var requests []recordedRequest
+
+	for line := range strings.Lines(string(text)) {
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		var r recordedRequest
+
+		err := json.Unmarshal([]byte(line), &r)
+		if err != nil {
+			t.Fatalf("%q: %v", line, err)
+		}
+
+		requests = append(requests, r)
+	}
+
+	return requests
+}
+
+// wantRecordedRequestsPass checks that none of the recorded ordinary
+// requests in the file name in testdata is refused, at the defaults and
+// while bodies are read. A request with neither a body nor a Content-Type
+// header is checked only at the defaults: with neither, the set that reads
+// bodies has nothing more to check. Each request runs as its own parallel
+// subtest.
+func wantRecordedRequestsPass(t *testing.T, name string) {
+	t.Helper()
+
+	const threshold = 5 // SWW_WAF_ANOMALY_THRESHOLD's default
+
+	defaults, bodies := atDefaults(t), readingBodies(t)
+
+	for _, recorded := range recordedRequests(t, name) {
+		t.Run(recorded.Method+" "+recorded.Target, func(t *testing.T) {
+			t.Parallel()
+
+			r := request{recorded.Method, recorded.Target, recorded.Headers}
+
+			got, _ := inspectBody(t, defaults, r, recorded.Body)
+			if got.Score >= threshold {
+				t.Errorf("%s %s refused: %+v", r.method, r.target, got)
+			}
+
+			typed := slices.ContainsFunc(recorded.Headers, func(header string) bool {
+				name, _, _ := strings.Cut(header, ":")
+
+				return strings.EqualFold(name, "Content-Type")
+			})
+			if recorded.Body == "" && !typed {
+				return
+			}
+
+			got, _ = inspectBody(t, bodies, r, recorded.Body)
+			if got.Score >= threshold {
+				t.Errorf("%s %s %.200q refused while bodies are read: %+v",
+					r.method, r.target, recorded.Body, got)
+			}
+		})
+	}
+}
+
+func TestOrdinaryDrupalRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	wantRecordedRequestsPass(t, "drupal-requests.txt")
+}
+
+func TestOrdinaryMastodonRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	wantRecordedRequestsPass(t, "mastodon-requests.txt")
+}
+
+func TestOrdinaryMediaWikiRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	wantRecordedRequestsPass(t, "mediawiki-requests.txt")
+}
+
+func TestOrdinaryNextcloudRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	wantRecordedRequestsPass(t, "nextcloud-requests.txt")
+}
+
+func TestOrdinaryVaultwardenRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	wantRecordedRequestsPass(t, "vaultwarden-requests.txt")
+}
+
+func TestOrdinaryWordPressRequestsPass(t *testing.T) {
+	t.Parallel()
+
+	const threshold = 5 // SWW_WAF_ANOMALY_THRESHOLD's default
+
+	defaults, bodies := atDefaults(t), readingBodiesUpTo64K(t)
+
+	for _, recorded := range recordedRequests(t, "wordpress-requests.txt") {
+		t.Run(recorded.Method+" "+recorded.Target, func(t *testing.T) {
+			t.Parallel()
+
+			r := request{recorded.Method, recorded.Target, recorded.Headers}
+
+			crsSets := []*coreruleset.CoreRuleSet{defaults}
+			if recorded.Body != "" {
+				crsSets = append(crsSets, bodies)
+			}
+
+			for _, crs := range crsSets {
+				got, _ := inspectBody(t, crs, r, recorded.Body)
+				if got.Score >= threshold {
+					t.Errorf("%s %s %.200q refused: %+v", r.method, r.target, recorded.Body, got)
+				}
+			}
+		})
+	}
+}