waf: the header limits are the first check (closes #311)

The header limits (more than 50 cookies or 100 header fields, answered `431`) move from just before the Core Rule Set to the first check after `SWW_ALLOW_NETS`. A request over them causes no lookup, no query to RouteWatch, a DNS blocklist or AbuseIPDB, no rule file match and no Core Rule Set. Requests within the limits go through the same checks in the same order.

For a request over the limits: the rate limits no longer count it; a banned client or one in `SWW_DENY_NETS` gets `431` instead of its refusal; it makes no ban, makes no ban permanent and is not counted in a ban's notes; in `observe` mode it is answered `allow` with no lookup. `SPEC.md` and `README.md` say so.

Model: opus-5-5

internal/waf/answertime_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
@@ -16,6 +17,7 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/reputation"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -216,93 +218,199 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
 	}
 }
 
-func TestRequestsOverTheHeaderLimitsCountTowardTheRateLimits(t *testing.T) {
+func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
 	t.Parallel()
 
 	s := startWAF(t, "", map[string]string{rateLimitPerMinute: "1"})
-	ask := func() answer {
+
+	for range 2 {
 		req := s.question(siteA, client, "/")
 		req.Header.Set(cookieHeader, cookies(cookieLimit+1))
-
-		return do(t, req)
+		wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
 	}
 
-	wantDisposition(t, ask(), tooLarge, waf.DispositionDeny)
-	wantDisposition(t, ask(), http.StatusForbidden, waf.DispositionDeny)
-
-	lines := s.out.requestLines(t, 2)
-	wantLine(t, lines[1], http.StatusForbidden, requestlog.ActionClientIsRateLimited)
+	wantDisposition(t, s.ask(client, "/"), http.StatusOK, waf.DispositionAllow)
 }
 
-func TestTheHeaderLimitsComeAfterTheTrapPathsAndTheRuleFiles(t *testing.T) {
+func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
 	t.Parallel()
 
+	// The lookup database places client in Germany.
+	lookupDB := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, lookupDB, map[string]lookuptest.Network{
+		client + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"},
+	})
+
 	for _, tc := range []struct {
-		name, path, action string
+		name string
+		env  map[string]string
+		path string
+		// prepare gets the server ready, at now, before the question.
+		prepare func(t *testing.T, server *waf.Server, now time.Time)
 	}{
-		{"a trap path", "/wp-login.php", requestlog.ActionTemporarilyBanned},
-		{"a block rule", "/blocked", requestlog.ActionBlockedByBlockRule},
+		{"SWW_DENY_NETS", map[string]string{denyNets: client}, "/", leaveAsItIs},
+		{"a ban", nil, "/", banClient},
+		{
+			"the country list", map[string]string{
+				lookupSource: fileSource, lookupDBPath: lookupDB, allowedCountries: "FR",
+			},
+			"/", leaveAsItIs,
+		},
+		{"a blocklist", map[string]string{blocklistURLs: dropURL}, "/", listClient},
+		{
+			"the CrowdSec decision list",
+			map[string]string{crowdSecURL: lapi, crowdSecKey: bouncerKey}, "/", decideOnClient,
+		},
+		{
+			"a DNS blocklist", map[string]string{
+				dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionDeny,
+			},
+			"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
+				loadVerdicts(server, map[string][]string{client: {dnsblZone}})
+			},
+		},
+		{
+			"AbuseIPDB",
+			map[string]string{abuseIPDBKey: accountKey, reputationAction: actionDeny},
+			"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
+				// Over SWW_ABUSEIPDB_MIN_SCORE, 75 by default.
+				loadScores(server, map[string]int64{client: 80})
+			},
+		},
+		{
+			"SWW_TAG_DENY", map[string]string{tagLists: "vpn=" + vpnList, tagDeny: "vpn"},
+			"/", func(t *testing.T, server *waf.Server, _ time.Time) {
+				t.Helper()
+				loadLists(t, server, map[string][]string{vpnList: {client}})
+			},
+		},
+		{"the rate limits", map[string]string{rateLimitPerMinute: "1"}, "/", countClient},
+		{
+			"a trap path", map[string]string{trapPaths: trapPathList}, "/wp-login.php",
+			leaveAsItIs,
+		},
+		{
+			"a block rule", map[string]string{rulesDir: writeRules(t, testRules)}, "/blocked",
+			leaveAsItIs,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			s := startWAF(t, "", map[string]string{
-				trapPaths: trapPathList, rulesDir: writeRules(t, testRules),
-			})
+			s := startWAF(t, "", tc.env)
+			tc.prepare(t, s.server, s.clock.Now())
+
 			req := s.question(siteA, client, tc.path)
 			req.Header.Set(cookieHeader, cookies(cookieLimit+1))
 
-			wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
-			wantLine(t, s.out.requestLine(t), http.StatusForbidden, tc.action)
+			wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
+			wantLine(t, s.out.requestLine(t), tooLarge, requestlog.ActionRequestHeaderTooLarge)
 		})
 	}
 }
 
+func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
+	t.Parallel()
+
+	// Each check after the header limits leaves a mark on the log line of a
+	// request it reads: a tag, routewatch_tags_missing, a rule id, the
+	// counts, or the Core Rule Set's time.
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		client + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"},
+	})
+	s := startWAF(t, "", map[string]string{
+		lookupSource: fileSource, lookupDBPath: path,
+		tagLists:      "vpn=" + vpnList,
+		routeWatchURL: startRouteWatch(t, map[string]standInRoute{client: {prefixDE, 64496}}),
+		dnsblZones:    dnsblZone, reputationAction: actionLog,
+		rulesDir: writeRules(t, "noted path log ^/\n"),
+		wafMode:  block,
+	})
+	loadLists(t, s.server, map[string][]string{vpnList: {client}})
+	loadVerdicts(s.server, map[string][]string{client: {dnsblZone}})
+
+	over := s.question(siteA, client, "/")
+	over.Header.Set(cookieHeader, cookies(cookieLimit+1))
+	wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny)
+
+	within := s.question(siteA, client, "/")
+	within.Header.Set(cookieHeader, cookies(cookieLimit))
+	wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow)
+
+	lines := s.out.requestLines(t, 2)
+
+	wantTags(t, lines[0])
+	wantTagsMissing(t, lines[0], "")
+
+	if lines[0].RuleIDs != nil || lines[0].Counts != (ratelimit.Counts{}) ||
+		lines[0].DurationCoreRuleSetMS != nil {
+		t.Errorf("rule ids %v, counts %+v and Core Rule Set time %v, want none",
+			lines[0].RuleIDs, lines[0].Counts, lines[0].DurationCoreRuleSetMS)
+	}
+
+	// The request within the limits shows that each check leaves its mark.
+	wantTags(t, lines[1],
+		"asn=64496", "country=DE", "dnsbl="+dnsblZone, "exploitsignature", "vpn")
+	wantTagsMissing(t, lines[1], "no_answer_yet")
+
+	if lines[1].RuleIDs == nil || lines[1].Counts.Minute != 1 ||
+		lines[1].DurationCoreRuleSetMS == nil {
+		t.Errorf("rule ids %v, counts %+v and Core Rule Set time %v, want each",
+			lines[1].RuleIDs, lines[1].Counts, lines[1].DurationCoreRuleSetMS)
+	}
+}
+
 func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
 	t.Parallel()
 
+	// The client's next request takes it over the whole limit, so it does
+	// not wait.
 	geojsURL, _ := startGeoJS(t)
 	s := startWAF(t, geojsURL, map[string]string{
-		unknownLimitPercent: "0", lookupTimeout: "1h",
+		unknownLimitPercent: "0", lookupTimeout: "1h", rateLimitPerMinute: "1",
 	})
-	req := s.question(siteA, fromDE, "/")
-	req.Header.Set(cookieHeader, cookies(cookieLimit+1))
+	countClient(t, s.server, s.clock.Now())
 
-	wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
+	wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden, waf.DispositionDeny)
 
 	line := s.out.requestLine(t)
-	wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
+	wantLine(t, line, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
 
 	if line.RateLimitPercent != nil {
 		t.Errorf("limit percentage %d, want none", *line.RateLimitPercent)
 	}
-
-	// The client is under no ban, and GeoJS places it in a country.
-	wantDisposition(t, s.ask(fromDE, "/"), http.StatusOK, waf.DispositionAllow)
 }
 
 func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(t *testing.T) {
 	t.Parallel()
 
+	// The client's next request takes it over the whole limit, so it does
+	// not wait.
 	geojsURL, _ := startGeoJS(t)
-	s := startWAF(t, geojsURL, map[string]string{unknownLimitPercent: "0"})
+	s := startWAF(t, geojsURL, map[string]string{
+		unknownLimitPercent: "0", rateLimitPerMinute: "1",
+	})
 	s.server.GeoJS.Load([]lookup.Answer{{
 		Client: netip.MustParsePrefix(client + "/32"), ASN: "AS64512",
 		Answered: s.clock.Now(), Used: s.clock.Now(),
 	}})
+	countClient(t, s.server, s.clock.Now())
 
-	req := s.question(siteA, client, "/")
-	req.Header.Set(cookieHeader, cookies(cookieLimit+1))
+	wantDisposition(t, s.ask(client, "/"), http.StatusForbidden, waf.DispositionDeny)
 
-	wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
-	wantLine(t, s.out.requestLine(t), http.StatusForbidden,
-		requestlog.ActionClientIsRateLimited)
+	line := s.out.requestLine(t)
+	wantLine(t, line, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
+
+	if line.RateLimitPercent == nil || *line.RateLimitPercent != 0 {
+		t.Errorf("limit percentage %v, want 0", line.RateLimitPercent)
+	}
 }
 
-func TestObserveModeWaitsForGeoJSForARequestOverTheHeaderLimits(t *testing.T) {
+func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(t *testing.T) {
 	t.Parallel()
 
-	geojsURL, _ := startGeoJS(t)
+	geojsURL, asked := startGeoJS(t)
 	s := startWAF(t, geojsURL, map[string]string{
 		mode: observe, addLookupHeaders: on, lookupTimeout: "1h",
 	})
@@ -312,10 +420,9 @@ func TestObserveModeWaitsForGeoJSForARequestOverTheHeaderLimits(t *testing.T) {
 	got := do(t, req)
 	wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
 
-	asn, country := got.header.Get("X-Client-Asn"), got.header.Get("X-Client-Country")
-	if asn != asnDE || country != "DE" {
-		t.Errorf("answered with the AS number %q and the country %q, want %q and DE",
-			asn, country, asnDE)
+	if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
+		t.Errorf("GeoJS was asked about %v, and the answer gave the AS number %q, "+
+			"want neither", asked(), got.header.Get("X-Client-Asn"))
 	}
 }
 

internal/waf/headerlimits.go

@@ -10,22 +10,23 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
-// The header limits: the most cookies and header fields a request may have
-// when the checks reach the Core Rule Set. Real clients never reach them.
+// The header limits: the most cookies and header fields a request may have.
+// Real clients never reach them.
 const (
 	maxCookies      = 50
 	maxHeaderFields = 100
 )
 
-// headerLimits refuses a request over the header limits, with 431. Its figures
-// are the request's header fields and cookies, and the limit of each.
+// headerLimits refuses a request with more than maxHeaderFields header fields
+// or more than maxCookies cookies, with 431. Its figures are the request's
+// header fields and cookies, and the limit of each.
 type headerLimits struct{}
 
 func (headerLimits) name() string { return "header_limits" }
 
 func (headerLimits) check(_ context.Context, rq *request, _ time.Time) *verdict {
 	fields, cookies := rq.headerCounts()
-	if !overHeaderLimits(fields, cookies) {
+	if fields <= maxHeaderFields && cookies <= maxCookies {
 		return nil
 	}
 
@@ -38,13 +39,6 @@ func (headerLimits) check(_ context.Context, rq *request, _ time.Time) *verdict
 	}
 }
 
-// overHeaderLimits reports whether a request with these counts of header fields
-// and cookies, as headerCounts counts them, is over the header limits: it has
-// more than maxHeaderFields fields or more than maxCookies cookies.
-func overHeaderLimits(fields, cookies int) bool {
-	return fields > maxHeaderFields || cookies > maxCookies
-}
-
 // headerCounts returns the request's header fields, each value of a header sent
 // twice being one, and its cookies, each part of a Cookie header between
 // semicolons being one unless it is empty or only spaces and tabs. The Core

internal/waf/heuristics.go

@@ -41,8 +41,11 @@ const (
 
 // beforeLookup are the heuristics that run before the client's AS number and
 // country are looked up, so that a client they refuse is not looked up.
+// header_limits comes first: it only counts, and a request it refuses causes no
+// other work.
 func beforeLookup() []heuristic {
 	return []heuristic{
+		headerLimits{},
 		tagLists{},
 		routeWatchTags{},
 		denyNets{},
@@ -64,7 +67,6 @@ func afterLookup() []heuristic {
 		rateLimits{},
 		trapPaths{},
 		ruleFiles{},
-		headerLimits{},
 		coreRuleSet{},
 		tagDeny{},
 	}

internal/waf/waf.go

@@ -143,10 +143,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // refusedWhateverTheAnswer reports whether the request is one that does not wait
 // for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
-// its client: the heuristic crowdsec, header_limits, blocklists or rate_limits
-// will refuse it, as a function in that heuristic's file says. In observe mode,
-// which refuses nothing, none is. rproxy lets a request through when
-// waf answers too late.
+// its client: the heuristic crowdsec, blocklists or rate_limits will refuse it,
+// as a function in that heuristic's file says. In observe mode, which refuses
+// nothing, none is. rproxy lets a request through when waf answers too late.
 func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
 	if rq.h.config.Observe {
 		return false
@@ -154,8 +153,7 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
 
 	_, decided := rq.crowdSecDecisionInForce(now)
 
-	return decided || overHeaderLimits(rq.headerCounts()) ||
-		rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
+	return decided || rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
 		rq.overAWholeRateLimit(now)
 }