From 17bbea919ab2e57ae478a4a5bfcf99507dedbd92 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Nov 2026 22:05:37 +0000 Subject: [PATCH 1/3] waf: heuristics pass facts through typed fields of decision (closes #442) The tags, what observe mode would have done, the broken limit and the three reputation flags are fields of decision, each naming what sets it. The log line is written from them and from the lookup answer, and no heuristic, ban note or alert reads it back. A ban's notes take the request's method, Host and User-Agent from decision, since the report's steps have no request to read them from. Model: opus-5-5 --- internal/waf/bans.go | 20 ++++++++------ internal/waf/biased.go | 4 +-- internal/waf/bytelimits.go | 2 +- internal/waf/coreruleset.go | 6 ++--- internal/waf/countries.go | 2 +- internal/waf/lookup.go | 4 +-- internal/waf/refusallimit.go | 2 +- internal/waf/reputation.go | 6 ++--- internal/waf/request.go | 52 +++++++++++++++++++++++++----------- internal/waf/tagdeny.go | 2 +- internal/waf/waf.go | 4 +-- 11 files changed, 64 insertions(+), 40 deletions(-) diff --git a/internal/waf/bans.go b/internal/waf/bans.go index a433da8..a81bd2b 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -45,8 +45,11 @@ func (a activeBan) check( } ban, banned, madePermanent := check(rq.client, now) + + var expires string if banned { - rq.line.BanExpires = banExpires(ban) + expires = banExpires(ban) + rq.line.BanExpires = expires } if madePermanent { @@ -61,7 +64,7 @@ func (a activeBan) check( return &verdict{ action: banAction(ban.Permanent()), figures: map[string]any{ - figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires, + figureNetblock: ban.Netblock.String(), "ban_expires": expires, }, } } @@ -148,6 +151,7 @@ func (b banMaker) banForLimit( rq.line.LimitHit = hit.Window } + rq.brokeLimit = true rq.line.Offence = requestlog.OffenceLimit b.metrics.LimitHit(hit) @@ -259,9 +263,9 @@ func (b banMaker) withClientNotes( status int, netblock netip.Prefix, ) bans.Notes { - notes.ASN = rq.line.ASN - notes.ASName = rq.line.ASName - notes.Country = rq.line.Country + notes.ASN = rq.lookupAnswer.ASN + notes.ASName = rq.lookupAnswer.ASName + notes.Country = rq.lookupAnswer.Country notes.Reputation = rq.reputation notes.Request = rq.noted(now, status) notes.Requests = b.netblockRequests(netblock) @@ -328,11 +332,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 { func (rq *request) noted(now time.Time, status int) bans.Request { return bans.Request{ Time: now, - Method: rq.line.Method, - Host: rq.line.Host, + Method: rq.method, + Host: rq.host, Path: rq.requestURI, Status: status, - UserAgent: rq.line.UserAgent, + UserAgent: rq.userAgent, } } diff --git a/internal/waf/biased.go b/internal/waf/biased.go index 3c74f97..f77255f 100644 --- a/internal/waf/biased.go +++ b/internal/waf/biased.go @@ -68,7 +68,7 @@ type biasedThresholds struct { func (b biasedThresholds) limitPercentages( rq *request, ) (percentage, percentage) { - asn, country := rq.line.ASN, rq.line.Country + asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country unknown := percentage{percent: whole} if country == "" && !rq.skippedLookupWait { @@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages( otherTags := percentage{percent: whole} - for _, tag := range rq.line.Tags { + for _, tag := range rq.tags { if tag != asnTag(asn) && tag != countryTag(country) { otherTags = lowest( otherTags, diff --git a/internal/waf/bytelimits.go b/internal/waf/bytelimits.go index 73fba6e..308a424 100644 --- a/internal/waf/bytelimits.go +++ b/internal/waf/bytelimits.go @@ -33,7 +33,7 @@ func (b byteLimits) check( rq *request, now time.Time, ) *verdict { - if !rq.counted || rq.line.WouldAction != "" { + if !rq.counted || rq.wouldAction != "" { return nil } diff --git a/internal/waf/coreruleset.go b/internal/waf/coreruleset.go index fd7f3c1..fcb0864 100644 --- a/internal/waf/coreruleset.go +++ b/internal/waf/coreruleset.go @@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch( Host: rq.site, Client: rq.client, Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, + ASN: rq.lookupAnswer.ASN, + ASName: rq.lookupAnswer.ASName, + Country: rq.lookupAnswer.Country, Reason: "scored by the Core Rule Set at or over " + "SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD", Detail: detail, diff --git a/internal/waf/countries.go b/internal/waf/countries.go index f841b7b..96a4bba 100644 --- a/internal/waf/countries.go +++ b/internal/waf/countries.go @@ -27,7 +27,7 @@ func (c countries) check( _ time.Time, ) *verdict { allowed := c.allowed - country := rq.line.Country + country := rq.lookupAnswer.Country if len(allowed) == 0 || slices.Contains(allowed, country) { return nil diff --git a/internal/waf/lookup.go b/internal/waf/lookup.go index 2a64b34..e558cee 100644 --- a/internal/waf/lookup.go +++ b/internal/waf/lookup.go @@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) { rq.line.ASName = rq.lookupAnswer.ASName rq.line.Country = rq.lookupAnswer.Country - if tag := asnTag(rq.line.ASN); tag != "" { + if tag := asnTag(rq.lookupAnswer.ASN); tag != "" { rq.addTag(tag) } - if tag := countryTag(rq.line.Country); tag != "" { + if tag := countryTag(rq.lookupAnswer.Country); tag != "" { rq.addTag(tag) } } diff --git a/internal/waf/refusallimit.go b/internal/waf/refusallimit.go index f3227c8..5eb5741 100644 --- a/internal/waf/refusallimit.go +++ b/internal/waf/refusallimit.go @@ -54,7 +54,7 @@ func (r refusalLimit) check( // What the client was sent, or in observe mode would have been. status := rq.out.status - switch rq.line.WouldAction { + switch rq.wouldAction { case requestlog.ActionBlockedByBlockRule, requestlog.ActionBlockedByCoreRuleSet: status = http.StatusForbidden diff --git a/internal/waf/reputation.go b/internal/waf/reputation.go index f898071..e5cda87 100644 --- a/internal/waf/reputation.go +++ b/internal/waf/reputation.go @@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit( Host: rq.site, Client: rq.client, Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, + ASN: rq.lookupAnswer.ASN, + ASName: rq.lookupAnswer.ASName, + Country: rq.lookupAnswer.Country, Reason: reason, Detail: detail, }) diff --git a/internal/waf/request.go b/internal/waf/request.go index d665171..d9a8d95 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -24,15 +24,22 @@ type refusal struct { action string } -// decision is what the checks note of a request: its log line, and what -// counting the request needs once it has ended. waf keeps a copy of it -// until the request's report comes. It holds no credentials, since the log line -// never does, and neither the request's objects nor their bodies. +// decision is what the checks note of a request: its log line, what the +// heuristics tell each other, and what counting the request needs once it has +// ended. Nothing reads the log line back to decide. waf keeps a copy of it +// until the request's report comes. It holds no credentials, since the log +// line never does, and neither the request's objects nor their bodies. type decision struct { line requestlog.Line client netip.Addr - // requestURI is the path with the query, as a ban's notes keep it. - requestURI string + // method, host and userAgent are the request's method, Host and + // User-Agent, and requestURI its path with the query, as a ban's notes + // keep them. startRequest sets them. + method, host, userAgent, requestURI string + // tags are the request's tags, sorted, each once. addTag adds them, for + // tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and + // coreRuleSet. + tags []string // site is, for a question, the host the visitor asked for, which the // alerts name, since one waf serves many sites. site string @@ -41,6 +48,7 @@ type decision struct { // lookedUp is true once the client's AS number and country have been // looked up, whether or not an answer was there, and lookupAnswer is // what the lookup gave then, the zero Answer while GeoJS had given none. + // noteLookup sets them. lookedUp bool lookupAnswer geojs.Answer // skippedLookupWait is true for a request that did not wait for GeoJS and @@ -58,8 +66,19 @@ type decision struct { // blockedByCoreRuleSet for one the Core Rule Set refused, each an offence // its client's history counts. attack, blockedByBlockRule, blockedByCoreRuleSet bool + // wouldAction is, in observe mode, the action enforce mode would have + // refused the request with, one of requestlog's Action constants, and "" + // otherwise. check sets it. + wouldAction string + // brokeLimit is true for a request that broke a limit, an offence its + // client's history counts. banForLimit sets it. + brokeLimit bool + // blocklisted is true once blocklists finds a blocklist that lists the + // client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does, + // and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit. + blocklisted, dnsblListed, abuseIPDBHit bool // reputation is the reputation sources that list the client, for the - // notes of a ban the request makes. + // notes of a ban the request makes. noteHit adds them. reputation []bans.ReputationHit } @@ -72,11 +91,7 @@ type request struct { in *http.Request out *responseWriter - // blocklisted is true once a blocklist is found to list the client, - // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once - // AbuseIPDB's score of it is a hit. - blocklisted, dnsblListed, abuseIPDBHit bool - start time.Time + start time.Time // checked is when the checks were done. checked time.Time // refused is what the request was refused with, nil if it was not. @@ -106,6 +121,9 @@ func (h *handler) startRequest( rq := &request{ decision: decision{ client: client, + method: r.Method, + host: r.Host, + userAgent: r.UserAgent(), requestURI: r.URL.RequestURI(), line: requestlog.Line{ Time: requestlog.FormatTime(start), @@ -151,11 +169,12 @@ func (h *handler) startRequest( } // addTag adds tag to the request's tags, which stay sorted and hold each tag -// once. +// once, and to its log line. func (rq *request) addTag(tag string) { - at, found := slices.BinarySearch(rq.line.Tags, tag) + at, found := slices.BinarySearch(rq.tags, tag) if !found { - rq.line.Tags = slices.Insert(rq.line.Tags, at, tag) + rq.tags = slices.Insert(rq.tags, at, tag) + rq.line.Tags = rq.tags } } @@ -196,6 +215,7 @@ func (rq *request) check(ctx context.Context) *refusal { switch { case rq.h.config.Observe: + rq.wouldAction = action rq.line.WouldAction = action return nil @@ -371,7 +391,7 @@ func (rq *request) addToHistory() { Status: rq.out.status, RequestBytes: rq.requestBytes, ResponseBytes: rq.out.bytes, - BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, + BrokeLimit: rq.brokeLimit, Attack: rq.attack, BlockedByBlockRule: rq.blockedByBlockRule, BlockedByCoreRuleSet: rq.blockedByCoreRuleSet, diff --git a/internal/waf/tagdeny.go b/internal/waf/tagdeny.go index a28322e..f49f0ce 100644 --- a/internal/waf/tagdeny.go +++ b/internal/waf/tagdeny.go @@ -23,7 +23,7 @@ func (t tagDeny) check( rq *request, _ time.Time, ) *verdict { - for _, tag := range rq.line.Tags { + for _, tag := range rq.tags { if slices.Contains(t.tags, tag) { return &verdict{ action: requestlog.ActionClientDenied, diff --git a/internal/waf/waf.go b/internal/waf/waf.go index eea7054..bb09c5f 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -202,8 +202,8 @@ func (rq *request) allow() { answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow} if rq.h.config.AddLookupHeaders { - answer.ASN = rq.line.ASN - answer.Country = rq.line.Country + answer.ASN = rq.lookupAnswer.ASN + answer.Country = rq.lookupAnswer.Country } wafprotocol.SetAnswerHeaders(rq.out.Header(), answer) -- 2.54.0 From eb1c75d264eab0ff925987dda38f0bdb0f22659a Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Nov 2026 22:14:03 +0000 Subject: [PATCH 2/3] waf: a ban's notes take the host from site, not a second field decision held the visitor's Host twice: as site, set by answerQuestion for the alerts, and as host, added for a ban's notes. Bans are made only from questions, where the two are always equal. noted now reads site, and host is gone. Model: opus-5-5 --- internal/waf/bans.go | 2 +- internal/waf/request.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/waf/bans.go b/internal/waf/bans.go index a81bd2b..b327cf4 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -333,7 +333,7 @@ func (rq *request) noted(now time.Time, status int) bans.Request { return bans.Request{ Time: now, Method: rq.method, - Host: rq.host, + Host: rq.site, Path: rq.requestURI, Status: status, UserAgent: rq.userAgent, diff --git a/internal/waf/request.go b/internal/waf/request.go index d9a8d95..56e6a03 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -32,16 +32,17 @@ type refusal struct { type decision struct { line requestlog.Line client netip.Addr - // method, host and userAgent are the request's method, Host and - // User-Agent, and requestURI its path with the query, as a ban's notes - // keep them. startRequest sets them. - method, host, userAgent, requestURI string + // method and userAgent are the request's method and User-Agent, and + // requestURI its path with the query, as a ban's notes keep them. + // startRequest sets them. + method, userAgent, requestURI string // tags are the request's tags, sorted, each once. addTag adds them, for // tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and // coreRuleSet. tags []string // site is, for a question, the host the visitor asked for, which the - // alerts name, since one waf serves many sites. + // alerts and a ban's notes name, since one waf serves many sites. + // answerQuestion sets it. site string // allowed is true for a question answered with DispositionAllow. allowed bool @@ -122,7 +123,6 @@ func (h *handler) startRequest( decision: decision{ client: client, method: r.Method, - host: r.Host, userAgent: r.UserAgent(), requestURI: r.URL.RequestURI(), line: requestlog.Line{ -- 2.54.0 From d3761ac7d00b62371e8a950036477961dad14d62 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Nov 2026 22:14:34 +0000 Subject: [PATCH 3/3] waf: name the setter in the comment of every fact a heuristic leaves The issue's definition of done asks that each fact a heuristic leaves for a later one is a field of decision whose comment names what sets it. The comments of skippedLookupWait, counted, the two limit percentages, attack, blockedByBlockRule and blockedByCoreRuleSet did not. Model: opus-5-5 --- internal/waf/request.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/waf/request.go b/internal/waf/request.go index 56e6a03..fa5c908 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -54,18 +54,20 @@ type decision struct { lookupAnswer geojs.Answer // skippedLookupWait is true for a request that did not wait for GeoJS and // found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it, - // so that not waiting never lowers its client's limits. + // so that not waiting never lowers its client's limits. noteLookup sets + // it. skippedLookupWait bool // counted is true for a request the rate limits counted, whose bytes // the byte limits count once it has ended. rateLimitPercent and // bytesLimitPercent are then its client's limit percentages for the rate - // limits and for the byte limits. + // limits and for the byte limits. rateLimits sets them. counted bool rateLimitPercent, bytesLimitPercent percentage // attack is true for a request that matched a ban rule or asked for a // trap path, blockedByBlockRule for one a block rule refused, and // blockedByCoreRuleSet for one the Core Rule Set refused, each an offence - // its client's history counts. + // its client's history counts. trapPaths and ruleFiles set attack, + // ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet. attack, blockedByBlockRule, blockedByCoreRuleSet bool // wouldAction is, in observe mode, the action enforce mode would have // refused the request with, one of requestlog's Action constants, and "" -- 2.54.0