discriminator: each decision heuristic is its own module (closes #195)

Each discriminator check is now a type in its own file behind one small interface, listed in `heuristics.go`. Adding a heuristic is one file and one line. Behaviour does not change.

Refusal and ban log lines gain `heuristic` and `heuristic_figures`. A ban after the answer, by `refusal_limit` or `byte_limits`, logs `after_answer_heuristic` and its figures, so one line can name both.

Deviation: four lists, not the planned three, so `refusal_limit` still bans at the decision.
Rule suppressed: `//nolint:lll` on one struct tag.

Model: opus-5-5

internal/discriminator/abuseipdb.go

@@ -0,0 +1,55 @@
+package discriminator
+
+// The heuristic abuseipdb. It reads SWW_ABUSEIPDB_KEY,
+// SWW_ABUSEIPDB_MIN_SCORE and SWW_REPUTATION_ACTION.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/reputation"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// abuseIPDB notes AbuseIPDB, as noteHit does, with the score, when its score
+// of the client is a hit, and refuses the request when SWW_REPUTATION_ACTION
+// is deny, as dnsbl does for a zone. While SWW_ABUSEIPDB_KEY is unset it does
+// nothing. A client without a score is checked in the background, by the
+// request's address, if its history counts an offence, and the request does
+// not wait for the answer. The score is then used for each address of the
+// client. Its figures are the score and SWW_ABUSEIPDB_MIN_SCORE.
+type abuseIPDB struct{}
+
+func (abuseIPDB) name() string { return "abuseipdb" }
+
+func (abuseIPDB) check(ctx context.Context, rq *request, _ time.Time) *verdict {
+	if rq.h.config.AbuseIPDBKey == "" {
+		return nil
+	}
+
+	client := rq.h.clientGroup(rq.client)
+	held, _ := rq.h.limiter.Client(client)
+	offender := held.History.Offences != ratelimit.Offences{}
+
+	score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
+	if !hit {
+		return nil
+	}
+
+	rq.abuseIPDBHit = true
+	rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
+		"scored by AbuseIPDB at or over SWW_ABUSEIPDB_MIN_SCORE")
+
+	if rq.h.config.ReputationAction != deny {
+		return nil
+	}
+
+	return &verdict{
+		action: requestlog.ActionDenied,
+		figures: map[string]any{
+			figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
+		},
+	}
+}

internal/discriminator/answertime_test.go

@@ -91,6 +91,18 @@ func TestSixteenThousandCookiesAreRefusedWellWithinTheTimeLimit(t *testing.T) {
 	}
 }
 
+func TestHeaderLimitRefusalNamesItsHeuristicCountsAndLimits(t *testing.T) {
+	t.Parallel()
+
+	s := startDiscriminator(t, "", nil)
+	req := s.question(siteA, client, "/")
+	fillHeaders(req, headerFieldLimit+1)
+	do(t, req)
+
+	wantHeuristic(t, s.out.requestLine(t), "header_limits",
+		`{"fields": 101, "fields_limit": 100, "cookies": 0, "cookies_limit": 50}`)
+}
+
 func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/bans.go

@@ -1,7 +1,10 @@
 package discriminator
 
+// The heuristic active_ban, and the code that makes bans, which several
+// heuristics share. active_ban reads SWW_MODE; the bans are the ban ledger's.
+
 import (
-	"net/http"
+	"context"
 	"net/netip"
 	"time"
 
@@ -18,11 +21,15 @@ func (rq *request) banResponse(action string) *refusal {
 	return &refusal{status: rq.h.config.BanResponse, action: action}
 }
 
-// banned reports whether a ban on a netblock the client is in covers the
-// request at now, and notes for the log line when that ban ends. A
-// request that makes the ban permanent, or in observe mode would have,
-// raises the alert for it.
-func (rq *request) banned(now time.Time) bool {
+// activeBan refuses a client a ban covers: a ban on a netblock the client is
+// in, in force at now. It notes for the log line when that ban ends. A request
+// that makes the ban permanent, or in observe mode would have, raises the alert
+// for it. Its figures are the ban's netblock and when it ends.
+type activeBan struct{}
+
+func (activeBan) name() string { return "active_ban" }
+
+func (activeBan) check(_ context.Context, rq *request, now time.Time) *verdict {
 	check := rq.h.ledger.Check
 	if rq.h.config.Observe {
 		check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
@@ -38,85 +45,24 @@ func (rq *request) banned(now time.Time) bool {
 		rq.alertBan(ban)
 	}
 
-	return banned
-}
-
-// limitBroken counts the request for the rate limits at now, notes the
-// client's counts for the log line, and reports whether the request takes
-// the client over a rate limit, as its limit percentage lowers it, which
-// breaks it.
-func (rq *request) limitBroken(now time.Time) bool {
-	counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
-		rq.limitPercent.percent)
-	rq.line.Counts = counts
-
-	if over {
-		rq.banForLimit(now, hit, rq.h.config.BanResponse)
+	if !banned {
+		return nil
 	}
 
-	return over
-}
-
-// countBytes counts the request's bytes, as countedBytes gives them, for
-// the byte limits, once its report has come, and notes the client's byte totals
-// for the log line; its requests stay there as the rate limits counted them.
-// Only an allowed request has them counted, and only one the rate limits
-// counted; in observe mode, not one that enforce mode would have refused. Bytes
-// that take the client over a byte limit, as its limit percentage for the byte
-// limits lowers it, break it; the response was passed on whole.
-func (rq *request) countBytes() {
-	if !rq.counted || rq.line.WouldAction != "" {
-		return
-	}
-
-	now := rq.h.now()
-
-	counts, hit, over := rq.h.limiter.CountBytes(rq.h.clientGroup(rq.client), now,
-		rq.countedBytes(), rq.bytesPercent.percent)
-	rq.line.Counts.MinuteBytes = counts.MinuteBytes
-	rq.line.Counts.HourBytes = counts.HourBytes
-	rq.line.Counts.DayBytes = counts.DayBytes
-
-	if over {
-		rq.banForLimit(now, hit, rq.out.status)
+	return &verdict{
+		action: requestlog.ActionBanned,
+		figures: map[string]any{
+			figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires,
+		},
 	}
 }
 
-// countRefusal counts the request for the error burst once it has been
-// answered, if discriminator refused it after a rule file match, a trap path or a
-// Core Rule Set match, and in observe mode if enforce mode would have: more than
-// SWW_ERROR_BURST_THRESHOLD such refusals of the client within a minute
-// break a limit. A client in SWW_ALLOW_NETS, which the checks skip, is
-// not counted, and nothing is while the threshold is off.
-func (rq *request) countRefusal() {
-	cfg := rq.h.config
-	if cfg.ErrorBurstThreshold == 0 {
-		return
+// limitFigures are the figures of a heuristic that broke hit, a limit: its
+// window, the count and the limit.
+func limitFigures(hit ratelimit.Hit) map[string]any {
+	return map[string]any{
+		figureWindow: hit.Window, figureCount: hit.Count, figureLimit: hit.Limit,
 	}
-
-	if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked {
-		return
-	}
-
-	now := rq.h.now()
-
-	hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
-		cfg.ErrorBurstThreshold)
-	if !over {
-		return
-	}
-
-	// What the client was sent, or in observe mode would have been.
-	status := rq.out.status
-
-	switch rq.line.WouldAction {
-	case requestlog.ActionRuleBlocked, requestlog.ActionWAFBlocked:
-		status = http.StatusForbidden
-	case requestlog.ActionBanned:
-		status = cfg.BanResponse
-	}
-
-	rq.banForLimit(now, hit, status)
 }
 
 // countedBytes returns the request's bytes, once it has ended, as the

internal/discriminator/bans_test.go

@@ -228,6 +228,22 @@ func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) {
 	}
 }
 
+func TestRefusalUnderABanNamesItsHeuristicNetblockAndEnd(t *testing.T) {
+	t.Parallel()
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		rateLimitPerMinute: "1", banScopeV4Prefix: "24",
+	})
+	expires := requestlog.FormatTime(clk.Now().Add(time.Hour))
+
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
+
+	line := s.get(otherClient, http.StatusForbidden, requestlog.ActionBanned)
+	wantHeuristic(t, line, "active_ban",
+		`{"netblock": "`+clientsNet+`", "ban_expires": "`+expires+`"}`)
+}
+
 func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/blocklists.go

@@ -0,0 +1,41 @@
+package discriminator
+
+// The heuristic blocklists. It reads SWW_BLOCKLIST_ACTION, and the blocklists
+// through the reputation lists.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// blocklists notes the blocklists that list the client, as noteListed does,
+// and refuses the request when SWW_BLOCKLIST_ACTION is deny. Being limit, it
+// lowers the client's limits instead (see limitPercentages), and being log, it
+// does nothing more. Its figure is the blocklists that list the client.
+type blocklists struct{}
+
+func (blocklists) name() string { return "blocklists" }
+
+func (blocklists) check(_ context.Context, rq *request, _ time.Time) *verdict {
+	listedBy := rq.h.lists.ListedBy(rq.client)
+	rq.blocklisted = len(listedBy) > 0
+	rq.noteListed(listedBy, "listed by a blocklist")
+
+	if !rq.refusedByBlocklists(listedBy) {
+		return nil
+	}
+
+	return &verdict{
+		action:  requestlog.ActionDenied,
+		figures: map[string]any{"lists": listedBy},
+	}
+}
+
+// refusedByBlocklists reports whether the heuristic blocklists refuses a
+// client, given listedBy, the blocklists that list it: it does when one lists
+// it and SWW_BLOCKLIST_ACTION is deny.
+func (rq *request) refusedByBlocklists(listedBy []string) bool {
+	return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
+}

internal/discriminator/bytelimits.go

@@ -0,0 +1,43 @@
+package discriminator
+
+// The heuristic byte_limits. It reads SWW_BYTES_COUNT, and the byte limits
+// and biased thresholds through the limiter and limitPercentages.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// byteLimits counts the request's bytes, as countedBytes gives them, for the
+// byte limits, once its report has come, and notes the client's byte totals for
+// the log line; its requests stay there as the rate limits counted them. Only
+// an allowed request has them counted, and only one the rate limits counted; in
+// observe mode, not one that enforce mode would have refused. Bytes that take
+// the client over a byte limit, as its limit percentage for the byte limits
+// lowers it, break it, which bans the client; the response was passed on whole.
+// Its figures are the window, the count and the limit.
+type byteLimits struct{}
+
+func (byteLimits) name() string { return "byte_limits" }
+
+func (byteLimits) check(_ context.Context, rq *request, now time.Time) *verdict {
+	if !rq.counted || rq.line.WouldAction != "" {
+		return nil
+	}
+
+	counts, hit, over := rq.h.limiter.CountBytes(rq.h.clientGroup(rq.client), now,
+		rq.countedBytes(), rq.bytesPercent.percent)
+	rq.line.Counts.MinuteBytes = counts.MinuteBytes
+	rq.line.Counts.HourBytes = counts.HourBytes
+	rq.line.Counts.DayBytes = counts.DayBytes
+
+	if !over {
+		return nil
+	}
+
+	rq.banForLimit(now, hit, rq.out.status)
+
+	return &verdict{action: requestlog.ActionBanned, figures: limitFigures(hit)}
+}

internal/discriminator/bytelimits_test.go

@@ -98,6 +98,21 @@ func TestEachByteLimitBansOnceTheResponseHasEnded(t *testing.T) {
 	}
 }
 
+func TestByteLimitBanNamesItsHeuristicWindowCountAndLimit(t *testing.T) {
+	t.Parallel()
+
+	s, _ := startWithAnswers(t, map[string]string{bytesLimitPerMinute: byteLimit})
+
+	line, _ := s.download()
+	wantAfterAnswerHeuristic(t, line, "", "null")
+
+	// Two answers of 70 bytes are over the limit of 99.
+	line, _ = s.download()
+	wantHeuristic(t, line, "", "null")
+	wantAfterAnswerHeuristic(t, line, "byte_limits",
+		`{"window": "minute", "count": 140, "limit": 99}`)
+}
+
 func TestResponseOverAByteLimitByItselfIsPassedOnWhole(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/coreruleset.go

@@ -1,6 +1,10 @@
 package discriminator
 
+// The heuristic core_rule_set. It reads SWW_WAF_MODE, SWW_WAF_EXEMPT_PATHS
+// and SWW_WAF_ANOMALY_THRESHOLD.
+
 import (
+	"context"
 	"time"
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
@@ -9,18 +13,21 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/waf"
 )
 
-// checkCoreRuleSet inspects the request with the Core Rule Set, unless
+// coreRuleSet inspects the request with the Core Rule Set, unless
 // SWW_WAF_MODE is off or SWW_WAF_EXEMPT_PATHS exempts its path, as
 // pathExempt decides, and notes the rules it matched and its score in the
 // log line, and the rules in the metrics. A score at or over
 // SWW_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert,
-// and in block mode refuses the request, which is an offence its client's
-// history counts, and so returns ActionWAFBlocked. It returns "" for a
-// request it does not refuse.
-func (rq *request) checkCoreRuleSet() string {
+// and in block mode refuses the request with 403, which is an offence its
+// client's history counts. Its figures are the score and the threshold.
+type coreRuleSet struct{}
+
+func (coreRuleSet) name() string { return "core_rule_set" }
+
+func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
 	cfg := rq.h.config
 	if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in, cfg.WAFExemptPaths) {
-		return ""
+		return nil
 	}
 
 	start := time.Now()
@@ -40,18 +47,21 @@ func (rq *request) checkCoreRuleSet() string {
 
 	threshold := cfg.WAFAnomalyThreshold
 	if threshold == 0 || result.Score < threshold {
-		return ""
+		return nil
 	}
 
 	rq.alertWAFBlock(result)
 
 	if cfg.WAFMode == config.WAFModeDetect {
-		return ""
+		return nil
 	}
 
 	rq.wafBlocked = true
 
-	return requestlog.ActionWAFBlocked
+	return &verdict{
+		action:  requestlog.ActionWAFBlocked,
+		figures: map[string]any{figureScore: result.Score, "threshold": threshold},
+	}
 }
 
 // alertWAFBlock raises the waf_block alert for the request, which the Core

internal/discriminator/coreruleset_test.go

@@ -110,6 +110,16 @@ func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *test
 	}
 }
 
+func TestCoreRuleSetRefusalNamesItsHeuristicScoreAndThreshold(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{wafMode: block})
+
+	line := s.request(client, sqlInjection, http.StatusForbidden,
+		requestlog.ActionWAFBlocked)
+	wantHeuristic(t, line, "core_rule_set", `{"score": 5, "threshold": 5}`)
+}
+
 func TestOrdinaryRequestIsInspectedAndPassed(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/countries.go

@@ -1,21 +1,42 @@
 package discriminator
 
+// The heuristic countries. It reads SWW_DENIED_COUNTRIES and
+// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
+
 import (
+	"context"
 	"slices"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
-// countryDenied reports whether the country lists refuse the request, by
-// the client's country as it was looked up. A client without a country,
-// or whose country cannot be found, is refused only by
-// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
-func (rq *request) countryDenied() bool {
+// countries refuses a request by the client's country as it was looked up. A
+// client without a country, or whose country cannot be found, is refused only
+// by SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. Its figures are the country and the
+// setting that refused it.
+type countries struct{}
+
+func (countries) name() string { return "countries" }
+
+func (countries) check(_ context.Context, rq *request, _ time.Time) *verdict {
 	denied := rq.h.config.DeniedCountries
 	allowed := rq.h.config.ExclusivelyAllowedCountries
 	country := rq.line.Country
 
-	if slices.Contains(denied, country) {
-		return true
+	var setting string
+
+	switch {
+	case slices.Contains(denied, country):
+		setting = "SWW_DENIED_COUNTRIES"
+	case len(allowed) > 0 && !slices.Contains(allowed, country):
+		setting = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
+	default:
+		return nil
 	}
 
-	return len(allowed) > 0 && !slices.Contains(allowed, country)
+	return &verdict{
+		action:  requestlog.ActionCountryDenied,
+		figures: map[string]any{"country": country, "setting": setting},
+	}
 }

internal/discriminator/countries_test.go

@@ -90,6 +90,23 @@ func TestCountryLists(t *testing.T) {
 	}
 }
 
+func TestCountryRefusalNamesItsHeuristicCountryAndSetting(t *testing.T) {
+	t.Parallel()
+
+	geojsURL, _ := startGeoJS(t)
+	s, _, _ := startWithClock(t, geojsURL, map[string]string{
+		lookupTimeout: "1h", deniedCountries: "kp", allowedCountries: "de",
+	})
+
+	line := s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied)
+	wantHeuristic(t, line, "countries",
+		`{"country": "KP", "setting": "SWW_DENIED_COUNTRIES"}`)
+
+	line = s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied)
+	wantHeuristic(t, line, "countries",
+		`{"country": "", "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"}`)
+}
+
 func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/crowdsec.go

@@ -0,0 +1,49 @@
+package discriminator
+
+// The heuristic crowdsec. It reads SWW_CROWDSEC_LAPI_URL, and the decisions
+// through the reputation lists.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/reputation"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// crowdSec refuses a request when a decision of the CrowdSec decision list on
+// the client is in force at now. It then notes the list, as noteHit does, and
+// bans the client until that decision ends. Its figures are the decision's
+// scenario and when it ends.
+type crowdSec struct{}
+
+func (crowdSec) name() string { return "crowdsec" }
+
+func (crowdSec) check(_ context.Context, rq *request, now time.Time) *verdict {
+	decision, listed := rq.crowdSecDecisionInForce(now)
+	if !listed {
+		return nil
+	}
+
+	rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
+		"listed by the CrowdSec decision list")
+	rq.banForCrowdSec(now, decision)
+
+	return &verdict{
+		action: requestlog.ActionBanned,
+		figures: map[string]any{
+			"scenario": decision.Scenario,
+			"expires":  requestlog.FormatTime(decision.Expires),
+		},
+	}
+}
+
+// crowdSecDecisionInForce returns the decision of the CrowdSec decision list on
+// the client that is in force at now, and reports whether there is one, which
+// the heuristic crowdsec refuses the request for.
+func (rq *request) crowdSecDecisionInForce(
+	now time.Time,
+) (reputation.Decision, bool) {
+	return rq.h.lists.CrowdSecDecision(rq.client, now)
+}

internal/discriminator/crowdsec_test.go

@@ -95,6 +95,21 @@ func TestClientTheCrowdSecDecisionListListsIsBannedUntilTheDecisionEnds(t *testi
 		float64(fetched.Unix()))
 }
 
+func TestCrowdSecBanNamesItsHeuristicScenarioAndEnd(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, _ := startWithAlerts(t, map[string]string{
+		crowdSecURL: lapi, crowdSecKey: bouncerKey,
+	})
+	loadDecisions(t, server, clk.Now(), `[{"duration": "4h0m0s", `+
+		`"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+
+		`"value": "`+client+`"}]`)
+
+	line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+	wantHeuristic(t, line, "crowdsec", `{"scenario": "crowdsecurity/ssh-bf", `+
+		`"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`)
+}
+
 func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/denynets.go

@@ -0,0 +1,29 @@
+package discriminator
+
+// The heuristic deny_nets. It reads SWW_DENY_NETS.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// denyNets refuses a client in SWW_DENY_NETS. Its figure is the netblock of
+// the setting the client is in.
+type denyNets struct{}
+
+func (denyNets) name() string { return "deny_nets" }
+
+func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
+	for _, netblock := range rq.h.config.DenyNets {
+		if netblock.Contains(rq.client) {
+			return &verdict{
+				action:  requestlog.ActionDenied,
+				figures: map[string]any{figureNetblock: netblock.String()},
+			}
+		}
+	}
+
+	return nil
+}

internal/discriminator/discriminator.go

@@ -2,6 +2,7 @@ package discriminator
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -52,13 +53,6 @@ const ReportPath = "/_sww/report"
 // QuestionBodyMaxBytes is the most of a visitor's body a question carries.
 const QuestionBodyMaxBytes = 64 << 10
 
-// The header limits: the most cookies and header fields a request may have
-// when the checks reach the Core Rule Set. Real clients never reach them.
-const (
-	maxCookies      = 50
-	maxHeaderFields = 100
-)
-
 const (
 	// maxAwaitingReport is the most answered requests held until their
 	// reports come. Past it, the request answered longest ago is dropped.
@@ -148,48 +142,20 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // refusedWhateverTheAnswer reports whether the request is one that does not wait
 // for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
-// its client: it is over the header limits, a blocklist lists its client while
-// SWW_BLOCKLIST_ACTION is deny, the CrowdSec decision list lists it, or it is
-// over a whole rate limit. In observe mode, which refuses nothing, none is.
-// rproxy lets a request through when discriminator answers too late.
+// its client: the heuristic crowdsec, header_limits, blocklists or rate_limits
+// will refuse it, as a function in that heuristic's file says. In observe mode,
+// which refuses nothing, none is. rproxy lets a request through when
+// discriminator answers too late.
 func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
 	if rq.h.config.Observe {
 		return false
 	}
 
-	lists := rq.h.lists
-	_, decided := lists.CrowdSecDecision(rq.client, now)
+	_, decided := rq.crowdSecDecisionInForce(now)
 
-	return decided || rq.overHeaderLimits() ||
-		(rq.h.config.BlocklistAction == deny && len(lists.ListedBy(rq.client)) > 0) ||
-		(rq.countedByRateLimits() &&
-			rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now))
-}
-
-// overHeaderLimits reports whether discriminator refuses the request for its
-// headers: it has more than maxHeaderFields fields, each value of a header sent
-// twice being one, or more than maxCookies cookies, each part of a Cookie header
-// between semicolons being one unless it is empty or only spaces and tabs. The
-// Core Rule Set runs each of its rules on each cookie, so that thousands of small
-// cookies keep it busy for over a second. It trims only spaces and control
-// characters, of which a header value can hold only tabs, so a part of Unicode
-// spaces is a cookie to it.
-func (rq *request) overHeaderLimits() bool {
-	fields, cookies := 0, 0
-
-	for _, values := range rq.in.Header {
-		fields += len(values)
-	}
-
-	for _, value := range rq.in.Header.Values("Cookie") {
-		for part := range strings.SplitSeq(value, ";") {
-			if strings.Trim(part, " \t") != "" {
-				cookies++
-			}
-		}
-	}
-
-	return fields > maxHeaderFields || cookies > maxCookies
+	return decided || overHeaderLimits(rq.headerCounts()) ||
+		rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
+		rq.overAWholeRateLimit(now)
 }
 
 // answerQuestion runs the checks on the visitor's request a question
@@ -220,18 +186,18 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
 
 	if refused == nil && !underOwnPrefix {
 		// In observe mode, what enforce mode would have refused counts here.
-		defer rq.countRefusal()
+		defer rq.runAfterAnswer(r.Context())
 
 		rq.allow()
 
 		return
 	}
 
-	// The last deferred runs first: countRefusal before addToHistory, so that a
-	// broken error burst is in the client's history.
+	// The last deferred runs first: runAfterAnswer before addToHistory, so that
+	// a broken refusal limit is in the client's history.
 	defer rq.addToHistory()
 	defer rq.countAnomalies()
-	defer rq.countRefusal()
+	defer rq.runAfterAnswer(r.Context())
 
 	if refused != nil {
 		rq.deny(*refused)
@@ -413,7 +379,7 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
 	// Remove reports true to one report only, of several that name the id. A
 	// question still being handled is held as nil.
 	if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
-		h.finishWith(held, report)
+		h.finishWith(r.Context(), held, report)
 	}
 
 	w.WriteHeader(http.StatusNoContent)
@@ -476,11 +442,11 @@ func (report *Report) checkRanges() error {
 
 // finishWith finishes a request with what its report says, and writes the
 // request's last log line, which holds its decision's fields and the report's.
-// For an allowed request it first counts the request's bytes, and adds it to its
-// client's history, the anomaly counters and the metrics, with the status and
-// bytes the report gives. A denied request's counting was finished at its
-// decision.
-func (h *handler) finishWith(held *decision, report Report) {
+// For an allowed request it first runs the heuristics of onReport, and adds the
+// request to its client's history, the anomaly counters and the metrics, with
+// the status and bytes the report gives. A denied request's counting was
+// finished at its decision. ctx is the report's own context.
+func (h *handler) finishWith(ctx context.Context, held *decision, report Report) {
 	rq := &request{
 		decision:     *held,
 		h:            h,
@@ -497,7 +463,7 @@ func (h *handler) finishWith(held *decision, report Report) {
 	if rq.allowed {
 		rq.line.DurationUpstreamTotal = &report.DurationUpstreamTotal
 
-		rq.countBytes()
+		rq.runBanningHeuristics(ctx, onReport())
 		rq.countAnomalies()
 		rq.addToHistory()
 

internal/discriminator/dnsbl.go

@@ -0,0 +1,36 @@
+package discriminator
+
+// The heuristic dnsbl. It reads SWW_REPUTATION_ACTION, and the DNSBL zones'
+// verdicts through the DNSBL checker.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed
+// does, and refuses the request when SWW_REPUTATION_ACTION is deny. Being
+// limit, it lowers the client's limits instead (see limitPercentages), and
+// being log, it does nothing more. A zone without a verdict on the client is
+// asked about it in the background, and the request does not wait for the
+// answer. Its figure is the zones that list the client, their keys masked.
+type dnsbl struct{}
+
+func (dnsbl) name() string { return "dnsbl" }
+
+func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
+	listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
+	rq.dnsblListed = len(listedBy) > 0
+	rq.noteListed(listedBy, "listed by a DNSBL zone")
+
+	if !rq.dnsblListed || rq.h.config.ReputationAction != deny {
+		return nil
+	}
+
+	return &verdict{
+		action:  requestlog.ActionDenied,
+		figures: map[string]any{"zones": listedBy},
+	}
+}

internal/discriminator/headerlimits.go

@@ -0,0 +1,71 @@
+package discriminator
+
+// The heuristic header_limits. It reads no setting: its limits are fixed.
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// The header limits: the most cookies and header fields a request may have
+// when the checks reach the Core Rule Set. Real clients never reach them.
+const (
+	maxCookies      = 50
+	maxHeaderFields = 100
+)
+
+// headerLimits refuses a request over the header limits, with 431. Its figures
+// are the request's header fields and cookies, and the limit of each.
+type headerLimits struct{}
+
+func (headerLimits) name() string { return "header_limits" }
+
+func (headerLimits) check(_ context.Context, rq *request, _ time.Time) *verdict {
+	fields, cookies := rq.headerCounts()
+	if !overHeaderLimits(fields, cookies) {
+		return nil
+	}
+
+	return &verdict{
+		action: requestlog.ActionTooLarge,
+		figures: map[string]any{
+			"fields": fields, "fields_limit": maxHeaderFields,
+			"cookies": cookies, "cookies_limit": maxCookies,
+		},
+	}
+}
+
+// overHeaderLimits reports whether a request with these counts of header fields
+// and cookies, as headerCounts counts them, is over the header limits: it has
+// more than maxHeaderFields fields or more than maxCookies cookies.
+func overHeaderLimits(fields, cookies int) bool {
+	return fields > maxHeaderFields || cookies > maxCookies
+}
+
+// headerCounts returns the request's header fields, each value of a header sent
+// twice being one, and its cookies, each part of a Cookie header between
+// semicolons being one unless it is empty or only spaces and tabs. The Core
+// Rule Set runs each of its rules on each cookie, so that thousands of small
+// cookies keep it busy for over a second. It trims only spaces and control
+// characters, of which a header value can hold only tabs, so a part of Unicode
+// spaces is a cookie to it.
+func (rq *request) headerCounts() (int, int) {
+	fields, cookies := 0, 0
+
+	for _, values := range rq.in.Header {
+		fields += len(values)
+	}
+
+	for _, value := range rq.in.Header.Values("Cookie") {
+		for part := range strings.SplitSeq(value, ";") {
+			if strings.Trim(part, " \t") != "" {
+				cookies++
+			}
+		}
+	}
+
+	return fields, cookies
+}

internal/discriminator/heuristics.go

@@ -0,0 +1,134 @@
+package discriminator
+
+import (
+	"context"
+	"time"
+)
+
+// To add a heuristic, write one file with a type that implements heuristic,
+// and add one line for it to one of the lists below. Its settings go in
+// config.Config, and the comment at the top of its file names them.
+
+// heuristic is one of the checks that decide what discriminator does with a
+// request.
+type heuristic interface {
+	// name is the heuristic's name, which the log line gives.
+	name() string
+	// check looks at the request at now, and returns nil to let it through.
+	// ctx is the request's own context.
+	check(ctx context.Context, rq *request, now time.Time) *verdict
+}
+
+// verdict is a heuristic refusing a request, or banning its client: the action
+// the log line names, one of requestlog's Action constants, and the figures the
+// heuristic acted on, by name, such as the count and the limit.
+type verdict struct {
+	action  string
+	figures map[string]any
+}
+
+// The names of the figures that more than one heuristic gives.
+const (
+	figureWindow   = "window"
+	figureCount    = "count"
+	figureLimit    = "limit"
+	figureScore    = "score"
+	figureNetblock = "netblock"
+)
+
+// The lists of heuristics, each in the order its heuristics run. A client in
+// SWW_ALLOW_NETS skips them all.
+
+// beforeLookup are the heuristics that run before the client's AS number and
+// country are looked up, so that a client they refuse is not looked up.
+func beforeLookup() []heuristic {
+	return []heuristic{
+		denyNets{},
+		activeBan{},
+	}
+}
+
+// afterLookup are the heuristics that run once the lookup has been made.
+func afterLookup() []heuristic {
+	return []heuristic{
+		countries{},
+		blocklists{},
+		crowdSec{},
+		dnsbl{},
+		abuseIPDB{},
+		rateLimits{},
+		trapPaths{},
+		ruleFiles{},
+		headerLimits{},
+		coreRuleSet{},
+	}
+}
+
+// afterAnswer are the heuristics that run on a request once discriminator has
+// answered it, before its decision line is written. They ban the client, and do
+// not change what the request got: their verdict's action is not used.
+func afterAnswer() []heuristic {
+	return []heuristic{
+		refusalLimit{},
+	}
+}
+
+// onReport are the heuristics that run on an allowed request when rproxy's
+// report of it comes, before its report line is written. Like afterAnswer, they
+// only ban.
+func onReport() []heuristic {
+	return []heuristic{
+		byteLimits{},
+	}
+}
+
+// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
+// it is called.
+func (rq *request) runAfterAnswer(ctx context.Context) {
+	rq.runBanningHeuristics(ctx, afterAnswer())
+}
+
+// runHeuristics runs list, beforeLookup or afterLookup, on the request at now,
+// and returns the first verdict, or nil when none gives one. The log line's
+// heuristic and heuristic_figures name the heuristic that gave it, and give its
+// figures. ctx is the request's own context.
+func (rq *request) runHeuristics(
+	ctx context.Context, list []heuristic, now time.Time,
+) *verdict {
+	name, v := rq.firstVerdict(ctx, list, now)
+	if v != nil {
+		rq.line.Heuristic = name
+		rq.line.HeuristicFigures = v.figures
+	}
+
+	return v
+}
+
+// runBanningHeuristics runs list, afterAnswer or onReport, on the request at
+// the time it is called. The log line's after_answer_heuristic and
+// after_answer_heuristic_figures name the heuristic that gave a verdict, and
+// give its figures, so that heuristic and heuristic_figures still name the one
+// that refused the request. ctx is the request's own context.
+func (rq *request) runBanningHeuristics(ctx context.Context, list []heuristic) {
+	name, v := rq.firstVerdict(ctx, list, rq.h.now())
+	if v != nil {
+		rq.line.AfterAnswerHeuristic = name
+		rq.line.AfterAnswerHeuristicFigures = v.figures
+	}
+}
+
+// firstVerdict runs list on the request at now, in order, until a heuristic
+// gives a verdict, and returns that heuristic's name and its verdict, or "" and
+// nil when none gives one. ctx is the request's own context.
+func (rq *request) firstVerdict(
+	ctx context.Context, list []heuristic, now time.Time,
+) (string, *verdict) {
+	for _, each := range list {
+		v := each.check(ctx, rq, now)
+		if v != nil {
+			return each.name(), v
+		}
+	}
+
+	return "", nil
+}

internal/discriminator/observe_test.go

@@ -14,6 +14,18 @@ import (
 // observe is the value of SWW_MODE for observe mode.
 const observe = "observe"
 
+func TestObserveModeNamesTheHeuristicThatWouldHaveRefused(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{
+		mode: observe, denyNets: client,
+	})
+
+	line := s.get(client, http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionDenied)
+	wantHeuristic(t, line, "deny_nets", `{"netblock": "`+client+`/32"}`)
+}
+
 func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/ratelimits.go

@@ -0,0 +1,65 @@
+package discriminator
+
+// The heuristic rate_limits. It reads SWW_RATE_LIMIT_EXEMPT_NETS,
+// SWW_RATE_LIMIT_EXEMPT_PATHS and SWW_BAN_RESPONSE, and the rate limits and
+// biased thresholds through the limiter and limitPercentages.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// rateLimits counts the request for the rate limits, unless
+// countedByRateLimits exempts it, by the client's limit percentages, which it
+// notes for the log line with the client's counts. A request that takes the
+// client over a rate limit, as its limit percentage lowers it, breaks it: it is
+// refused, and bans the client. A request exempt from the rate limits is exempt
+// from the byte limits too. Its figures are the window, the count and the
+// limit.
+type rateLimits struct{}
+
+func (rateLimits) name() string { return "rate_limits" }
+
+func (rateLimits) check(_ context.Context, rq *request, now time.Time) *verdict {
+	rq.counted = rq.countedByRateLimits()
+	if !rq.counted {
+		return nil
+	}
+
+	rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
+	rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
+	rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
+
+	counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
+		rq.limitPercent.percent)
+	rq.line.Counts = counts
+
+	if !over {
+		return nil
+	}
+
+	rq.banForLimit(now, hit, rq.h.config.BanResponse)
+
+	return &verdict{action: requestlog.ActionRateLimited, figures: limitFigures(hit)}
+}
+
+// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
+// request at now, whatever its client's limit percentages: the rate limits
+// count it, and one more request from its client takes it over a whole rate
+// limit. It counts nothing.
+func (rq *request) overAWholeRateLimit(now time.Time) bool {
+	return rq.countedByRateLimits() &&
+		rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now)
+}
+
+// countedByRateLimits reports whether the rate limits count the request: its
+// client is not in SWW_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
+// under SWW_RATE_LIMIT_EXEMPT_PATHS.
+func (rq *request) countedByRateLimits() bool {
+	cfg := rq.h.config
+
+	return !isInside(rq.client, cfg.RateLimitExemptNets) &&
+		!pathExempt(rq.in, cfg.RateLimitExemptPaths)
+}

internal/discriminator/ratelimits_test.go

@@ -72,6 +72,17 @@ func TestRateLimitRefusesBeforeTheApp(t *testing.T) {
 	}
 }
 
+func TestRateLimitRefusalNamesItsHeuristicWindowCountAndLimit(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{rateLimitPerMinute: "1"})
+
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+
+	line := s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
+	wantHeuristic(t, line, "rate_limits", `{"window": "minute", "count": 2, "limit": 1}`)
+}
+
 func TestIPv6GroupPrefixSetsTheClientTheLimitsCount(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/refusallimit.go

@@ -0,0 +1,54 @@
+package discriminator
+
+// The heuristic refusal_limit. It reads SWW_ERROR_BURST_THRESHOLD and
+// SWW_BAN_RESPONSE.
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// refusalLimit counts the request once it has been answered, if discriminator
+// refused it after a rule file match, a trap path or a Core Rule Set match, and
+// in observe mode if enforce mode would have: more than
+// SWW_ERROR_BURST_THRESHOLD such refusals of the client within a minute break
+// a limit, which bans the client. A client in SWW_ALLOW_NETS, which the
+// heuristics skip, is not counted, and nothing is while the threshold is off.
+// Its figures are the window, the count and the limit.
+type refusalLimit struct{}
+
+func (refusalLimit) name() string { return "refusal_limit" }
+
+func (refusalLimit) check(_ context.Context, rq *request, now time.Time) *verdict {
+	cfg := rq.h.config
+	if cfg.ErrorBurstThreshold == 0 {
+		return nil
+	}
+
+	if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked {
+		return nil
+	}
+
+	hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
+		cfg.ErrorBurstThreshold)
+	if !over {
+		return nil
+	}
+
+	// What the client was sent, or in observe mode would have been.
+	status := rq.out.status
+
+	switch rq.line.WouldAction {
+	case requestlog.ActionRuleBlocked, requestlog.ActionWAFBlocked:
+		status = http.StatusForbidden
+	case requestlog.ActionBanned:
+		status = cfg.BanResponse
+	}
+
+	rq.banForLimit(now, hit, status)
+
+	return &verdict{action: requestlog.ActionBanned, figures: limitFigures(hit)}
+}

internal/discriminator/refusallimit_test.go

@@ -106,6 +106,25 @@ func TestErrorBurstBreaksAtOneOverTheThreshold(t *testing.T) {
 	}
 }
 
+func TestRefusalLimitBanNamesItsHeuristicWindowCountAndLimit(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startForErrorBurst(t, "1", nil)
+
+	const ruleFigures = `{"rule_id": "blocked", "rule_action": "block"}`
+
+	line := blockRule.send(s, client)
+	wantHeuristic(t, line, "rule_files", ruleFigures)
+	wantAfterAnswerHeuristic(t, line, "", "null")
+
+	// The second refusal is the request refusal_limit bans for, and the line
+	// names both.
+	line = blockRule.send(s, client)
+	wantHeuristic(t, line, "rule_files", ruleFigures)
+	wantAfterAnswerHeuristic(t, line, "refusal_limit",
+		`{"window": "minute", "count": 2, "limit": 1}`)
+}
+
 func TestErrorBurstBanNotesHistoryAndMetrics(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/reputation.go

@@ -1,91 +1,17 @@
 package discriminator
 
-import (
-	"context"
-	"time"
+// What the reputation heuristics share: blocklists, crowdsec, dnsbl and
+// abuseipdb.
 
+import (
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
-	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
-	"sneak.berlin/go/smallwebwaf/internal/reputation"
 )
 
 // deny is the SWW_BLOCKLIST_ACTION and the SWW_REPUTATION_ACTION that
 // refuses the requests of a client a source lists.
 const deny = "deny"
 
-// blocklistDenied notes the blocklists that list the client, as
-// noteListed does, and reports whether SWW_BLOCKLIST_ACTION, being deny,
-// refuses the request. Being limit, it lowers the client's limits instead
-// (see limitPercentages), and being log, it does nothing more.
-func (rq *request) blocklistDenied() bool {
-	listedBy := rq.h.lists.ListedBy(rq.client)
-	rq.blocklisted = len(listedBy) > 0
-	rq.noteListed(listedBy, "listed by a blocklist")
-
-	return rq.blocklisted && rq.h.config.BlocklistAction == deny
-}
-
-// crowdSecBanned reports whether a decision of the CrowdSec decision list
-// on the client is in force at now. If one is, it notes the list, as
-// noteHit does, and bans the client until that decision ends.
-func (rq *request) crowdSecBanned(now time.Time) bool {
-	decision, listed := rq.h.lists.CrowdSecDecision(rq.client, now)
-	if !listed {
-		return false
-	}
-
-	rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
-		"listed by the CrowdSec decision list")
-	rq.banForCrowdSec(now, decision)
-
-	return true
-}
-
-// dnsblDenied notes the DNSBL zones whose verdict lists the client, as
-// noteListed does, and reports whether SWW_REPUTATION_ACTION, being
-// deny, refuses the request. Being limit, it lowers the client's limits
-// instead (see limitPercentages), and being log, it does nothing more. A
-// zone without a verdict on the client is asked about it in the
-// background, and the request does not wait for the answer. ctx is the
-// request's own context.
-func (rq *request) dnsblDenied(ctx context.Context) bool {
-	listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
-	rq.dnsblListed = len(listedBy) > 0
-	rq.noteListed(listedBy, "listed by a DNSBL zone")
-
-	return rq.dnsblListed && rq.h.config.ReputationAction == deny
-}
-
-// abuseIPDBDenied notes AbuseIPDB, as noteHit does, with the score, when
-// its score of the client is a hit, and reports whether
-// SWW_REPUTATION_ACTION, being deny, refuses the request, as dnsblDenied
-// does for a zone. While SWW_ABUSEIPDB_KEY is unset it does nothing. A
-// client without a score is checked in the background, by the request's
-// address, if its history counts an offence, and the request does not
-// wait for the answer. The score is then used for each address of the
-// client. ctx is the request's own context.
-func (rq *request) abuseIPDBDenied(ctx context.Context) bool {
-	if rq.h.config.AbuseIPDBKey == "" {
-		return false
-	}
-
-	client := rq.h.clientGroup(rq.client)
-	held, _ := rq.h.limiter.Client(client)
-	offender := held.History.Offences != ratelimit.Offences{}
-
-	score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
-	if !hit {
-		return false
-	}
-
-	rq.abuseIPDBHit = true
-	rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
-		"scored by AbuseIPDB at or over SWW_ABUSEIPDB_MIN_SCORE")
-
-	return rq.h.config.ReputationAction == deny
-}
-
 // noteListed notes each of sources, the URLs of the blocklists or the
 // DNSBL zones, their keys masked, that list the client, as noteHit does,
 // with reason.

internal/discriminator/reputation_test.go

@@ -118,6 +118,56 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
 	}
 }
 
+func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		heuristic string
+		env       map[string]string
+		load      func(*testing.T, *discriminator.Server)
+		figures   string
+	}{
+		{
+			"blocklists",
+			map[string]string{blocklistURLs: dropURL, blocklistAction: actionDeny},
+			func(t *testing.T, server *discriminator.Server) {
+				t.Helper()
+				loadLists(t, server, map[string][]string{dropURL: {client}})
+			},
+			`{"lists": ["` + dropURL + `"]}`,
+		},
+		{
+			"dnsbl",
+			map[string]string{
+				dnsblZones: dnsblZone, dnsblResolver: noResolver,
+				reputationAction: actionDeny,
+			},
+			func(_ *testing.T, server *discriminator.Server) {
+				loadVerdicts(server, map[string][]string{client: {dnsblZone}})
+			},
+			`{"zones": ["` + dnsblZone + `"]}`,
+		},
+		{
+			"abuseipdb",
+			map[string]string{abuseIPDBKey: accountKey, reputationAction: actionDeny},
+			func(_ *testing.T, server *discriminator.Server) {
+				loadScores(server, map[string]int64{client: 80})
+			},
+			`{"score": 80, "min_score": 75}`,
+		},
+	} {
+		t.Run(tc.heuristic, func(t *testing.T) {
+			t.Parallel()
+
+			s, server, _ := startWithLookups(t, tc.env)
+			tc.load(t, server)
+
+			line := s.get(client, http.StatusForbidden, requestlog.ActionDenied)
+			wantHeuristic(t, line, tc.heuristic, tc.figures)
+		})
+	}
+}
+
 func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/request.go

@@ -160,16 +160,20 @@ func requestHeaders(r *http.Request, names []string) map[string]string {
 }
 
 // check is the one place where a request can be refused. It returns nil to let
-// the request through. The checks of checkClient decide: a request they refuse
-// is answered with SWW_BAN_RESPONSE, 403 for a block rule or the Core Rule Set,
-// or 431 for the header limits. In observe mode nothing is refused, and the log
-// line names what enforce mode would have done. ctx is the request's own context.
+// the request through. The heuristics of checkClient decide: a request they
+// refuse is answered with SWW_BAN_RESPONSE, 403 for a block rule or the Core
+// Rule Set, or 431 for the header limits. In observe mode nothing is refused,
+// and the log line names what enforce mode would have done. ctx is the
+// request's own context.
 func (rq *request) check(ctx context.Context) *refusal {
-	action := rq.checkClient(ctx)
+	v := rq.checkClient(ctx)
+	if v == nil {
+		return nil
+	}
+
+	action := v.action
 
 	switch {
-	case action == "":
-		return nil
 	case rq.h.config.Observe:
 		rq.line.WouldAction = action
 
@@ -183,35 +187,21 @@ func (rq *request) check(ctx context.Context) *refusal {
 	}
 }
 
-// checkClient runs the checks on the request's client, and returns the
-// action of the first that refuses the request, or "" when none does. A
-// client in SWW_ALLOW_NETS skips them, and is not looked up. For any
-// other client, SWW_DENY_NETS comes first, then a ban on its netblock,
-// so that a client either refuses is not looked up, then the lookup of
-// its AS number and country, then the country lists, then the blocklists,
-// then the CrowdSec decision list, which bans the client it lists, then
-// the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of
-// them refuses is not counted for the rate limits. Then come the
-// rate limits, unless the client is in SWW_RATE_LIMIT_EXEMPT_NETS or the
-// request's path is exempt under SWW_RATE_LIMIT_EXEMPT_PATHS, so that
-// every other request is counted, each of them by the client's limit
-// percentages, then SWW_TRAP_PATHS, then the rule files, then the header
-// limits, and last the Core Rule Set. A request exempt from the rate limits
-// is exempt from the byte limits too. ctx is the request's own context.
-func (rq *request) checkClient(ctx context.Context) string {
-	cfg := rq.h.config
-	if isInside(rq.client, cfg.AllowNets) {
-		return ""
+// checkClient runs the heuristics of beforeLookup on the request, then looks
+// up the client's AS number and country, then runs those of afterLookup. It
+// returns the first verdict, or nil when none gives one. A client in
+// SWW_ALLOW_NETS skips them all, and is not looked up. ctx is the request's
+// own context.
+func (rq *request) checkClient(ctx context.Context) *verdict {
+	if isInside(rq.client, rq.h.config.AllowNets) {
+		return nil
 	}
 
 	now := rq.h.now()
 
-	if isInside(rq.client, cfg.DenyNets) {
-		return requestlog.ActionDenied
-	}
-
-	if rq.banned(now) {
-		return requestlog.ActionBanned
+	v := rq.runHeuristics(ctx, beforeLookup(), now)
+	if v != nil {
+		return v
 	}
 
 	rq.lookUp(ctx, now)
@@ -220,59 +210,7 @@ func (rq *request) checkClient(ctx context.Context) string {
 	// again: a request counted at the time it arrived could be counted
 	// after a later one of its client's, which the rate limits would take
 	// for the clock set back.
-	now = rq.h.now()
-
-	if rq.countryDenied() {
-		return requestlog.ActionCountryDenied
-	}
-
-	if rq.blocklistDenied() {
-		return requestlog.ActionDenied
-	}
-
-	if rq.crowdSecBanned(now) {
-		return requestlog.ActionBanned
-	}
-
-	if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
-		return requestlog.ActionDenied
-	}
-
-	rq.counted = rq.countedByRateLimits()
-	if rq.counted {
-		rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
-		rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
-		rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
-	}
-
-	if rq.counted && rq.limitBroken(now) {
-		return requestlog.ActionRateLimited
-	}
-
-	if rq.trapPath(now) {
-		return requestlog.ActionBanned
-	}
-
-	action := rq.checkRules(now)
-	if action != "" {
-		return action
-	}
-
-	if rq.overHeaderLimits() {
-		return requestlog.ActionTooLarge
-	}
-
-	return rq.checkCoreRuleSet()
-}
-
-// countedByRateLimits reports whether the rate limits count the request: its
-// client is not in SWW_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
-// under SWW_RATE_LIMIT_EXEMPT_PATHS.
-func (rq *request) countedByRateLimits() bool {
-	cfg := rq.h.config
-
-	return !isInside(rq.client, cfg.RateLimitExemptNets) &&
-		!pathExempt(rq.in, cfg.RateLimitExemptPaths)
+	return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
 }
 
 // pathExempt reports whether r is exempt under prefixes,

internal/discriminator/requestlog_test.go

@@ -1,6 +1,7 @@
 package discriminator_test
 
 import (
+	"encoding/json"
 	"io"
 	"math"
 	"net/http"
@@ -272,6 +273,48 @@ func wantTimings(t *testing.T, line logLine, want ...string) {
 	}
 }
 
+// wantHeuristic checks that line names heuristic as the one that refused the
+// request, with figures, a JSON object. A heuristic of "" with figures "null" is
+// a line that names none.
+func wantHeuristic(t *testing.T, line logLine, heuristic, figures string) {
+	t.Helper()
+
+	if line.Heuristic != heuristic ||
+		!reflect.DeepEqual(line.HeuristicFigures, decodeFigures(t, figures)) {
+		t.Errorf("log line has heuristic %q with heuristic_figures %v, want %q with %s",
+			line.Heuristic, line.HeuristicFigures, heuristic, figures)
+	}
+}
+
+// wantAfterAnswerHeuristic checks after_answer_heuristic and
+// after_answer_heuristic_figures as wantHeuristic checks heuristic and
+// heuristic_figures.
+func wantAfterAnswerHeuristic(t *testing.T, line logLine, heuristic, figures string) {
+	t.Helper()
+
+	if line.AfterAnswerHeuristic != heuristic ||
+		!reflect.DeepEqual(line.AfterAnswerHeuristicFigures, decodeFigures(t, figures)) {
+		t.Errorf("log line has after_answer_heuristic %q with "+
+			"after_answer_heuristic_figures %v, want %q with %s",
+			line.AfterAnswerHeuristic, line.AfterAnswerHeuristicFigures, heuristic, figures)
+	}
+}
+
+// decodeFigures decodes figures, a JSON object of a heuristic's figures, as
+// the log line's figures are decoded.
+func decodeFigures(t *testing.T, figures string) map[string]any {
+	t.Helper()
+
+	var decoded map[string]any
+
+	err := json.Unmarshal([]byte(figures), &decoded)
+	if err != nil {
+		t.Fatalf("decode %s: %v", figures, err)
+	}
+
+	return decoded
+}
+
 // microseconds is a timing in whole microseconds.
 func microseconds(milliseconds float64) int64 {
 	return int64(math.Round(milliseconds * 1000))

internal/discriminator/rulefiles.go

@@ -0,0 +1,55 @@
+package discriminator
+
+// The heuristic rule_files. It reads the rules of the rule files, which
+// SWW_RULES_DIR and SWW_RULES_ENABLED name.
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
+)
+
+// ruleFiles checks the request against the rules of the rule files at now,
+// notes the ids of those it matches in the log line, and refuses the request
+// when the last rule it matches refuses it: with ActionRuleBlocked for a block
+// rule, and ActionBanned for a ban rule. A ban rule bans the client's netblock
+// for a clear sign of attack, or in observe mode raises the alert for the ban
+// it would have made. Either rule's match is noted as an offence, for the
+// client's history. Its figures are the rule's id and action.
+type ruleFiles struct{}
+
+func (ruleFiles) name() string { return "rule_files" }
+
+func (ruleFiles) check(_ context.Context, rq *request, now time.Time) *verdict {
+	matched := rq.h.rules.Match(rq.in)
+
+	for _, rule := range matched {
+		rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
+		rq.h.metrics.RuleMatched(rule.ID, rule.Action)
+	}
+
+	if len(matched) == 0 {
+		return nil
+	}
+
+	// Only the last rule matched can refuse the request.
+	last := matched[len(matched)-1]
+	figures := map[string]any{"rule_id": last.ID, "rule_action": last.Action}
+
+	switch last.Action {
+	case rules.ActionBlock:
+		rq.ruleBlocked = true
+
+		return &verdict{action: requestlog.ActionRuleBlocked, figures: figures}
+	case rules.ActionBan:
+		rq.attack = true
+		rq.banForAttack(now, bans.Notes{RuleID: last.ID, Target: last.Target})
+
+		return &verdict{action: requestlog.ActionBanned, figures: figures}
+	default:
+		return nil
+	}
+}

internal/discriminator/rulefiles_test.go

@@ -93,6 +93,21 @@ func TestEachRuleAction(t *testing.T) {
 	s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
 }
 
+func TestRuleRefusalNamesItsHeuristicAndTheRule(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{
+		rulesDir: writeRules(t, "noted path log ^/\n"+testRules),
+	})
+
+	line := s.request(client, "/blocked", http.StatusForbidden,
+		requestlog.ActionRuleBlocked)
+	wantHeuristic(t, line, "rule_files", `{"rule_id": "blocked", "rule_action": "block"}`)
+
+	line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	wantHeuristic(t, line, "rule_files", `{"rule_id": "probe", "rule_action": "ban"}`)
+}
+
 func TestNextClearSignOfAttackAfterABanBansPermanently(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/rules.go

@@ -1,64 +0,0 @@
-package discriminator
-
-import (
-	"slices"
-	"time"
-
-	"sneak.berlin/go/smallwebwaf/internal/bans"
-	"sneak.berlin/go/smallwebwaf/internal/requestlog"
-	"sneak.berlin/go/smallwebwaf/internal/rules"
-)
-
-// trapPath reports whether the request asks for a path in
-// SWW_TRAP_PATHS: its path as a path rule sees it, without the query,
-// as sent or once percent-decoded, is one of them. Such a request is a
-// clear sign of attack, as a ban rule's match is: it bans the client's
-// netblock, or in observe mode raises the alert for the ban it would have
-// made. The ban's notes give the path as sent.
-func (rq *request) trapPath(now time.Time) bool {
-	path := rules.Path(rq.in)
-	if !slices.Contains(rq.h.config.TrapPaths, path) &&
-		!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
-		return false
-	}
-
-	rq.attack = true
-	rq.banForAttack(now, bans.Notes{TrapPath: path})
-
-	return true
-}
-
-// checkRules checks the request against the rules of the rule files at
-// now, notes the ids of those it matches in the log line, and returns the
-// action of the rule that refuses it, ActionRuleBlocked for a block rule
-// and ActionBanned for a ban rule, or "" when none does. A ban rule bans
-// the client's netblock for a clear sign of attack, or in observe mode
-// raises the alert for the ban it would have made. Either rule's match
-// is noted as an offence, for the client's history.
-func (rq *request) checkRules(now time.Time) string {
-	matched := rq.h.rules.Match(rq.in)
-
-	for _, rule := range matched {
-		rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
-		rq.h.metrics.RuleMatched(rule.ID, rule.Action)
-	}
-
-	if len(matched) == 0 {
-		return ""
-	}
-
-	// Only the last rule matched can refuse the request.
-	switch last := matched[len(matched)-1]; last.Action {
-	case rules.ActionBlock:
-		rq.ruleBlocked = true
-
-		return requestlog.ActionRuleBlocked
-	case rules.ActionBan:
-		rq.attack = true
-		rq.banForAttack(now, bans.Notes{RuleID: last.ID, Target: last.Target})
-
-		return requestlog.ActionBanned
-	default:
-		return ""
-	}
-}

internal/discriminator/staticlists_test.go

@@ -104,6 +104,15 @@ func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) {
 	}
 }
 
+func TestDenyNetsRefusalNamesItsHeuristicAndNetblock(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{denyNets: clientsNet})
+
+	line := s.get(client, http.StatusForbidden, requestlog.ActionDenied)
+	wantHeuristic(t, line, "deny_nets", `{"netblock": "`+clientsNet+`"}`)
+}
+
 func TestRequestRefusedByDenyNetsIsNotCounted(t *testing.T) {
 	t.Parallel()
 

internal/discriminator/trappaths.go

@@ -0,0 +1,39 @@
+package discriminator
+
+// The heuristic trap_paths. It reads SWW_TRAP_PATHS.
+
+import (
+	"context"
+	"slices"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
+)
+
+// trapPaths refuses a request for a path in SWW_TRAP_PATHS: its path as a
+// path rule sees it, without the query, as sent or once percent-decoded, is one
+// of them. Such a request is a clear sign of attack, as a ban rule's match is:
+// it bans the client's netblock, or in observe mode raises the alert for the
+// ban it would have made. The ban's notes give the path as sent, and so does
+// its figure.
+type trapPaths struct{}
+
+func (trapPaths) name() string { return "trap_paths" }
+
+func (trapPaths) check(_ context.Context, rq *request, now time.Time) *verdict {
+	path := rules.Path(rq.in)
+	if !slices.Contains(rq.h.config.TrapPaths, path) &&
+		!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
+		return nil
+	}
+
+	rq.attack = true
+	rq.banForAttack(now, bans.Notes{TrapPath: path})
+
+	return &verdict{
+		action:  requestlog.ActionBanned,
+		figures: map[string]any{"path": path},
+	}
+}

internal/discriminator/trappaths_test.go

@@ -83,6 +83,16 @@ func TestTrapPathBansAsABanRuleDoes(t *testing.T) {
 	}
 }
 
+func TestTrapPathRefusalNamesItsHeuristicAndPath(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{trapPaths: trapPathList})
+
+	line := s.request(client, "/wp-login.php?redirect_to=x", http.StatusForbidden,
+		requestlog.ActionBanned)
+	wantHeuristic(t, line, "trap_paths", `{"path": "/wp-login.php"}`)
+}
+
 func TestTrapPathMatchesThePathDecodedOnce(t *testing.T) {
 	t.Parallel()