From 17bbea919ab2e57ae478a4a5bfcf99507dedbd92 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Nov 2026 22:05:37 +0000 Subject: [PATCH] waf: heuristics pass facts through typed fields of decision (closes #442) The tags, what observe mode would have done, the broken limit and the three reputation flags are fields of decision, each naming what sets it. The log line is written from them and from the lookup answer, and no heuristic, ban note or alert reads it back. A ban's notes take the request's method, Host and User-Agent from decision, since the report's steps have no request to read them from. Model: opus-5-5 --- internal/waf/bans.go | 20 ++++++++------ internal/waf/biased.go | 4 +-- internal/waf/bytelimits.go | 2 +- internal/waf/coreruleset.go | 6 ++--- internal/waf/countries.go | 2 +- internal/waf/lookup.go | 4 +-- internal/waf/refusallimit.go | 2 +- internal/waf/reputation.go | 6 ++--- internal/waf/request.go | 52 +++++++++++++++++++++++++----------- internal/waf/tagdeny.go | 2 +- internal/waf/waf.go | 4 +-- 11 files changed, 64 insertions(+), 40 deletions(-) diff --git a/internal/waf/bans.go b/internal/waf/bans.go index a433da8..a81bd2b 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -45,8 +45,11 @@ func (a activeBan) check( } ban, banned, madePermanent := check(rq.client, now) + + var expires string if banned { - rq.line.BanExpires = banExpires(ban) + expires = banExpires(ban) + rq.line.BanExpires = expires } if madePermanent { @@ -61,7 +64,7 @@ func (a activeBan) check( return &verdict{ action: banAction(ban.Permanent()), figures: map[string]any{ - figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires, + figureNetblock: ban.Netblock.String(), "ban_expires": expires, }, } } @@ -148,6 +151,7 @@ func (b banMaker) banForLimit( rq.line.LimitHit = hit.Window } + rq.brokeLimit = true rq.line.Offence = requestlog.OffenceLimit b.metrics.LimitHit(hit) @@ -259,9 +263,9 @@ func (b banMaker) withClientNotes( status int, netblock netip.Prefix, ) bans.Notes { - notes.ASN = rq.line.ASN - notes.ASName = rq.line.ASName - notes.Country = rq.line.Country + notes.ASN = rq.lookupAnswer.ASN + notes.ASName = rq.lookupAnswer.ASName + notes.Country = rq.lookupAnswer.Country notes.Reputation = rq.reputation notes.Request = rq.noted(now, status) notes.Requests = b.netblockRequests(netblock) @@ -328,11 +332,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 { func (rq *request) noted(now time.Time, status int) bans.Request { return bans.Request{ Time: now, - Method: rq.line.Method, - Host: rq.line.Host, + Method: rq.method, + Host: rq.host, Path: rq.requestURI, Status: status, - UserAgent: rq.line.UserAgent, + UserAgent: rq.userAgent, } } diff --git a/internal/waf/biased.go b/internal/waf/biased.go index 3c74f97..f77255f 100644 --- a/internal/waf/biased.go +++ b/internal/waf/biased.go @@ -68,7 +68,7 @@ type biasedThresholds struct { func (b biasedThresholds) limitPercentages( rq *request, ) (percentage, percentage) { - asn, country := rq.line.ASN, rq.line.Country + asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country unknown := percentage{percent: whole} if country == "" && !rq.skippedLookupWait { @@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages( otherTags := percentage{percent: whole} - for _, tag := range rq.line.Tags { + for _, tag := range rq.tags { if tag != asnTag(asn) && tag != countryTag(country) { otherTags = lowest( otherTags, diff --git a/internal/waf/bytelimits.go b/internal/waf/bytelimits.go index 73fba6e..308a424 100644 --- a/internal/waf/bytelimits.go +++ b/internal/waf/bytelimits.go @@ -33,7 +33,7 @@ func (b byteLimits) check( rq *request, now time.Time, ) *verdict { - if !rq.counted || rq.line.WouldAction != "" { + if !rq.counted || rq.wouldAction != "" { return nil } diff --git a/internal/waf/coreruleset.go b/internal/waf/coreruleset.go index fd7f3c1..fcb0864 100644 --- a/internal/waf/coreruleset.go +++ b/internal/waf/coreruleset.go @@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch( Host: rq.site, Client: rq.client, Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, + ASN: rq.lookupAnswer.ASN, + ASName: rq.lookupAnswer.ASName, + Country: rq.lookupAnswer.Country, Reason: "scored by the Core Rule Set at or over " + "SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD", Detail: detail, diff --git a/internal/waf/countries.go b/internal/waf/countries.go index f841b7b..96a4bba 100644 --- a/internal/waf/countries.go +++ b/internal/waf/countries.go @@ -27,7 +27,7 @@ func (c countries) check( _ time.Time, ) *verdict { allowed := c.allowed - country := rq.line.Country + country := rq.lookupAnswer.Country if len(allowed) == 0 || slices.Contains(allowed, country) { return nil diff --git a/internal/waf/lookup.go b/internal/waf/lookup.go index 2a64b34..e558cee 100644 --- a/internal/waf/lookup.go +++ b/internal/waf/lookup.go @@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) { rq.line.ASName = rq.lookupAnswer.ASName rq.line.Country = rq.lookupAnswer.Country - if tag := asnTag(rq.line.ASN); tag != "" { + if tag := asnTag(rq.lookupAnswer.ASN); tag != "" { rq.addTag(tag) } - if tag := countryTag(rq.line.Country); tag != "" { + if tag := countryTag(rq.lookupAnswer.Country); tag != "" { rq.addTag(tag) } } diff --git a/internal/waf/refusallimit.go b/internal/waf/refusallimit.go index f3227c8..5eb5741 100644 --- a/internal/waf/refusallimit.go +++ b/internal/waf/refusallimit.go @@ -54,7 +54,7 @@ func (r refusalLimit) check( // What the client was sent, or in observe mode would have been. status := rq.out.status - switch rq.line.WouldAction { + switch rq.wouldAction { case requestlog.ActionBlockedByBlockRule, requestlog.ActionBlockedByCoreRuleSet: status = http.StatusForbidden diff --git a/internal/waf/reputation.go b/internal/waf/reputation.go index f898071..e5cda87 100644 --- a/internal/waf/reputation.go +++ b/internal/waf/reputation.go @@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit( Host: rq.site, Client: rq.client, Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, + ASN: rq.lookupAnswer.ASN, + ASName: rq.lookupAnswer.ASName, + Country: rq.lookupAnswer.Country, Reason: reason, Detail: detail, }) diff --git a/internal/waf/request.go b/internal/waf/request.go index d665171..d9a8d95 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -24,15 +24,22 @@ type refusal struct { action string } -// decision is what the checks note of a request: its log line, and what -// counting the request needs once it has ended. waf keeps a copy of it -// until the request's report comes. It holds no credentials, since the log line -// never does, and neither the request's objects nor their bodies. +// decision is what the checks note of a request: its log line, what the +// heuristics tell each other, and what counting the request needs once it has +// ended. Nothing reads the log line back to decide. waf keeps a copy of it +// until the request's report comes. It holds no credentials, since the log +// line never does, and neither the request's objects nor their bodies. type decision struct { line requestlog.Line client netip.Addr - // requestURI is the path with the query, as a ban's notes keep it. - requestURI string + // method, host and userAgent are the request's method, Host and + // User-Agent, and requestURI its path with the query, as a ban's notes + // keep them. startRequest sets them. + method, host, userAgent, requestURI string + // tags are the request's tags, sorted, each once. addTag adds them, for + // tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and + // coreRuleSet. + tags []string // site is, for a question, the host the visitor asked for, which the // alerts name, since one waf serves many sites. site string @@ -41,6 +48,7 @@ type decision struct { // lookedUp is true once the client's AS number and country have been // looked up, whether or not an answer was there, and lookupAnswer is // what the lookup gave then, the zero Answer while GeoJS had given none. + // noteLookup sets them. lookedUp bool lookupAnswer geojs.Answer // skippedLookupWait is true for a request that did not wait for GeoJS and @@ -58,8 +66,19 @@ type decision struct { // blockedByCoreRuleSet for one the Core Rule Set refused, each an offence // its client's history counts. attack, blockedByBlockRule, blockedByCoreRuleSet bool + // wouldAction is, in observe mode, the action enforce mode would have + // refused the request with, one of requestlog's Action constants, and "" + // otherwise. check sets it. + wouldAction string + // brokeLimit is true for a request that broke a limit, an offence its + // client's history counts. banForLimit sets it. + brokeLimit bool + // blocklisted is true once blocklists finds a blocklist that lists the + // client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does, + // and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit. + blocklisted, dnsblListed, abuseIPDBHit bool // reputation is the reputation sources that list the client, for the - // notes of a ban the request makes. + // notes of a ban the request makes. noteHit adds them. reputation []bans.ReputationHit } @@ -72,11 +91,7 @@ type request struct { in *http.Request out *responseWriter - // blocklisted is true once a blocklist is found to list the client, - // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once - // AbuseIPDB's score of it is a hit. - blocklisted, dnsblListed, abuseIPDBHit bool - start time.Time + start time.Time // checked is when the checks were done. checked time.Time // refused is what the request was refused with, nil if it was not. @@ -106,6 +121,9 @@ func (h *handler) startRequest( rq := &request{ decision: decision{ client: client, + method: r.Method, + host: r.Host, + userAgent: r.UserAgent(), requestURI: r.URL.RequestURI(), line: requestlog.Line{ Time: requestlog.FormatTime(start), @@ -151,11 +169,12 @@ func (h *handler) startRequest( } // addTag adds tag to the request's tags, which stay sorted and hold each tag -// once. +// once, and to its log line. func (rq *request) addTag(tag string) { - at, found := slices.BinarySearch(rq.line.Tags, tag) + at, found := slices.BinarySearch(rq.tags, tag) if !found { - rq.line.Tags = slices.Insert(rq.line.Tags, at, tag) + rq.tags = slices.Insert(rq.tags, at, tag) + rq.line.Tags = rq.tags } } @@ -196,6 +215,7 @@ func (rq *request) check(ctx context.Context) *refusal { switch { case rq.h.config.Observe: + rq.wouldAction = action rq.line.WouldAction = action return nil @@ -371,7 +391,7 @@ func (rq *request) addToHistory() { Status: rq.out.status, RequestBytes: rq.requestBytes, ResponseBytes: rq.out.bytes, - BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, + BrokeLimit: rq.brokeLimit, Attack: rq.attack, BlockedByBlockRule: rq.blockedByBlockRule, BlockedByCoreRuleSet: rq.blockedByCoreRuleSet, diff --git a/internal/waf/tagdeny.go b/internal/waf/tagdeny.go index a28322e..f49f0ce 100644 --- a/internal/waf/tagdeny.go +++ b/internal/waf/tagdeny.go @@ -23,7 +23,7 @@ func (t tagDeny) check( rq *request, _ time.Time, ) *verdict { - for _, tag := range rq.line.Tags { + for _, tag := range rq.tags { if slices.Contains(t.tags, tag) { return &verdict{ action: requestlog.ActionClientDenied, diff --git a/internal/waf/waf.go b/internal/waf/waf.go index eea7054..bb09c5f 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -202,8 +202,8 @@ func (rq *request) allow() { answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow} if rq.h.config.AddLookupHeaders { - answer.ASN = rq.line.ASN - answer.Country = rq.line.Country + answer.ASN = rq.lookupAnswer.ASN + answer.Country = rq.lookupAnswer.Country } wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)