From 17400f68b82b90c4aa8b223614e18176757767c7 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 19 Nov 2026 23:43:00 +0100 Subject: [PATCH] waf: heuristics pass facts through typed fields of decision (closes #442) The tags, what observe mode would have done, the broken limit and the three reputation flags are fields of decision, each naming what sets it. The log line is written from them and from the lookup answer, and no heuristic, ban note or alert reads it back. A ban's notes take the request's method, Host and User-Agent from decision, since the report's steps have no request to read them from. Model: opus-5-5 --- internal/waf/bans.go | 20 +++++++----- internal/waf/biased.go | 4 +-- internal/waf/bytelimits.go | 2 +- internal/waf/coreruleset.go | 6 ++-- internal/waf/countries.go | 2 +- internal/waf/lookup.go | 4 +-- internal/waf/refusallimit.go | 2 +- internal/waf/reputation.go | 6 ++-- internal/waf/request.go | 62 ++++++++++++++++++++++++------------ internal/waf/tagdeny.go | 2 +- internal/waf/waf.go | 4 +-- 11 files changed, 70 insertions(+), 44 deletions(-) diff --git a/internal/waf/bans.go b/internal/waf/bans.go index a433da8..b327cf4 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -45,8 +45,11 @@ func (a activeBan) check( } ban, banned, madePermanent := check(rq.client, now) + + var expires string if banned { - rq.line.BanExpires = banExpires(ban) + expires = banExpires(ban) + rq.line.BanExpires = expires } if madePermanent { @@ -61,7 +64,7 @@ func (a activeBan) check( return &verdict{ action: banAction(ban.Permanent()), figures: map[string]any{ - figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires, + figureNetblock: ban.Netblock.String(), "ban_expires": expires, }, } } @@ -148,6 +151,7 @@ func (b banMaker) banForLimit( rq.line.LimitHit = hit.Window } + rq.brokeLimit = true rq.line.Offence = requestlog.OffenceLimit b.metrics.LimitHit(hit) @@ -259,9 +263,9 @@ func (b banMaker) withClientNotes( status int, netblock netip.Prefix, ) bans.Notes { - notes.ASN = rq.line.ASN - notes.ASName = rq.line.ASName - notes.Country = rq.line.Country + notes.ASN = rq.lookupAnswer.ASN + notes.ASName = rq.lookupAnswer.ASName + notes.Country = rq.lookupAnswer.Country notes.Reputation = rq.reputation notes.Request = rq.noted(now, status) notes.Requests = b.netblockRequests(netblock) @@ -328,11 +332,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 { func (rq *request) noted(now time.Time, status int) bans.Request { return bans.Request{ Time: now, - Method: rq.line.Method, - Host: rq.line.Host, + Method: rq.method, + Host: rq.site, Path: rq.requestURI, Status: status, - UserAgent: rq.line.UserAgent, + UserAgent: rq.userAgent, } } diff --git a/internal/waf/biased.go b/internal/waf/biased.go index 3c74f97..f77255f 100644 --- a/internal/waf/biased.go +++ b/internal/waf/biased.go @@ -68,7 +68,7 @@ type biasedThresholds struct { func (b biasedThresholds) limitPercentages( rq *request, ) (percentage, percentage) { - asn, country := rq.line.ASN, rq.line.Country + asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country unknown := percentage{percent: whole} if country == "" && !rq.skippedLookupWait { @@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages( otherTags := percentage{percent: whole} - for _, tag := range rq.line.Tags { + for _, tag := range rq.tags { if tag != asnTag(asn) && tag != countryTag(country) { otherTags = lowest( otherTags, diff --git a/internal/waf/bytelimits.go b/internal/waf/bytelimits.go index 73fba6e..308a424 100644 --- a/internal/waf/bytelimits.go +++ b/internal/waf/bytelimits.go @@ -33,7 +33,7 @@ func (b byteLimits) check( rq *request, now time.Time, ) *verdict { - if !rq.counted || rq.line.WouldAction != "" { + if !rq.counted || rq.wouldAction != "" { return nil } diff --git a/internal/waf/coreruleset.go b/internal/waf/coreruleset.go index fd7f3c1..fcb0864 100644 --- a/internal/waf/coreruleset.go +++ b/internal/waf/coreruleset.go @@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch( Host: rq.site, Client: rq.client, Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, + ASN: rq.lookupAnswer.ASN, + ASName: rq.lookupAnswer.ASName, + Country: rq.lookupAnswer.Country, Reason: "scored by the Core Rule Set at or over " + "SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD", Detail: detail, diff --git a/internal/waf/countries.go b/internal/waf/countries.go index f841b7b..96a4bba 100644 --- a/internal/waf/countries.go +++ b/internal/waf/countries.go @@ -27,7 +27,7 @@ func (c countries) check( _ time.Time, ) *verdict { allowed := c.allowed - country := rq.line.Country + country := rq.lookupAnswer.Country if len(allowed) == 0 || slices.Contains(allowed, country) { return nil diff --git a/internal/waf/lookup.go b/internal/waf/lookup.go index 2a64b34..e558cee 100644 --- a/internal/waf/lookup.go +++ b/internal/waf/lookup.go @@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) { rq.line.ASName = rq.lookupAnswer.ASName rq.line.Country = rq.lookupAnswer.Country - if tag := asnTag(rq.line.ASN); tag != "" { + if tag := asnTag(rq.lookupAnswer.ASN); tag != "" { rq.addTag(tag) } - if tag := countryTag(rq.line.Country); tag != "" { + if tag := countryTag(rq.lookupAnswer.Country); tag != "" { rq.addTag(tag) } } diff --git a/internal/waf/refusallimit.go b/internal/waf/refusallimit.go index f3227c8..5eb5741 100644 --- a/internal/waf/refusallimit.go +++ b/internal/waf/refusallimit.go @@ -54,7 +54,7 @@ func (r refusalLimit) check( // What the client was sent, or in observe mode would have been. status := rq.out.status - switch rq.line.WouldAction { + switch rq.wouldAction { case requestlog.ActionBlockedByBlockRule, requestlog.ActionBlockedByCoreRuleSet: status = http.StatusForbidden diff --git a/internal/waf/reputation.go b/internal/waf/reputation.go index f898071..e5cda87 100644 --- a/internal/waf/reputation.go +++ b/internal/waf/reputation.go @@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit( Host: rq.site, Client: rq.client, Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, + ASN: rq.lookupAnswer.ASN, + ASName: rq.lookupAnswer.ASName, + Country: rq.lookupAnswer.Country, Reason: reason, Detail: detail, }) diff --git a/internal/waf/request.go b/internal/waf/request.go index d665171..fa5c908 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -24,42 +24,64 @@ type refusal struct { action string } -// decision is what the checks note of a request: its log line, and what -// counting the request needs once it has ended. waf keeps a copy of it -// until the request's report comes. It holds no credentials, since the log line -// never does, and neither the request's objects nor their bodies. +// decision is what the checks note of a request: its log line, what the +// heuristics tell each other, and what counting the request needs once it has +// ended. Nothing reads the log line back to decide. waf keeps a copy of it +// until the request's report comes. It holds no credentials, since the log +// line never does, and neither the request's objects nor their bodies. type decision struct { line requestlog.Line client netip.Addr - // requestURI is the path with the query, as a ban's notes keep it. - requestURI string + // method and userAgent are the request's method and User-Agent, and + // requestURI its path with the query, as a ban's notes keep them. + // startRequest sets them. + method, userAgent, requestURI string + // tags are the request's tags, sorted, each once. addTag adds them, for + // tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and + // coreRuleSet. + tags []string // site is, for a question, the host the visitor asked for, which the - // alerts name, since one waf serves many sites. + // alerts and a ban's notes name, since one waf serves many sites. + // answerQuestion sets it. site string // allowed is true for a question answered with DispositionAllow. allowed bool // lookedUp is true once the client's AS number and country have been // looked up, whether or not an answer was there, and lookupAnswer is // what the lookup gave then, the zero Answer while GeoJS had given none. + // noteLookup sets them. lookedUp bool lookupAnswer geojs.Answer // skippedLookupWait is true for a request that did not wait for GeoJS and // found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it, - // so that not waiting never lowers its client's limits. + // so that not waiting never lowers its client's limits. noteLookup sets + // it. skippedLookupWait bool // counted is true for a request the rate limits counted, whose bytes // the byte limits count once it has ended. rateLimitPercent and // bytesLimitPercent are then its client's limit percentages for the rate - // limits and for the byte limits. + // limits and for the byte limits. rateLimits sets them. counted bool rateLimitPercent, bytesLimitPercent percentage // attack is true for a request that matched a ban rule or asked for a // trap path, blockedByBlockRule for one a block rule refused, and // blockedByCoreRuleSet for one the Core Rule Set refused, each an offence - // its client's history counts. + // its client's history counts. trapPaths and ruleFiles set attack, + // ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet. attack, blockedByBlockRule, blockedByCoreRuleSet bool + // wouldAction is, in observe mode, the action enforce mode would have + // refused the request with, one of requestlog's Action constants, and "" + // otherwise. check sets it. + wouldAction string + // brokeLimit is true for a request that broke a limit, an offence its + // client's history counts. banForLimit sets it. + brokeLimit bool + // blocklisted is true once blocklists finds a blocklist that lists the + // client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does, + // and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit. + blocklisted, dnsblListed, abuseIPDBHit bool // reputation is the reputation sources that list the client, for the - // notes of a ban the request makes. + // notes of a ban the request makes. noteHit adds them. reputation []bans.ReputationHit } @@ -72,11 +94,7 @@ type request struct { in *http.Request out *responseWriter - // blocklisted is true once a blocklist is found to list the client, - // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once - // AbuseIPDB's score of it is a hit. - blocklisted, dnsblListed, abuseIPDBHit bool - start time.Time + start time.Time // checked is when the checks were done. checked time.Time // refused is what the request was refused with, nil if it was not. @@ -106,6 +124,8 @@ func (h *handler) startRequest( rq := &request{ decision: decision{ client: client, + method: r.Method, + userAgent: r.UserAgent(), requestURI: r.URL.RequestURI(), line: requestlog.Line{ Time: requestlog.FormatTime(start), @@ -151,11 +171,12 @@ func (h *handler) startRequest( } // addTag adds tag to the request's tags, which stay sorted and hold each tag -// once. +// once, and to its log line. func (rq *request) addTag(tag string) { - at, found := slices.BinarySearch(rq.line.Tags, tag) + at, found := slices.BinarySearch(rq.tags, tag) if !found { - rq.line.Tags = slices.Insert(rq.line.Tags, at, tag) + rq.tags = slices.Insert(rq.tags, at, tag) + rq.line.Tags = rq.tags } } @@ -196,6 +217,7 @@ func (rq *request) check(ctx context.Context) *refusal { switch { case rq.h.config.Observe: + rq.wouldAction = action rq.line.WouldAction = action return nil @@ -371,7 +393,7 @@ func (rq *request) addToHistory() { Status: rq.out.status, RequestBytes: rq.requestBytes, ResponseBytes: rq.out.bytes, - BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, + BrokeLimit: rq.brokeLimit, Attack: rq.attack, BlockedByBlockRule: rq.blockedByBlockRule, BlockedByCoreRuleSet: rq.blockedByCoreRuleSet, diff --git a/internal/waf/tagdeny.go b/internal/waf/tagdeny.go index a28322e..f49f0ce 100644 --- a/internal/waf/tagdeny.go +++ b/internal/waf/tagdeny.go @@ -23,7 +23,7 @@ func (t tagDeny) check( rq *request, _ time.Time, ) *verdict { - for _, tag := range rq.line.Tags { + for _, tag := range rq.tags { if slices.Contains(t.tags, tag) { return &verdict{ action: requestlog.ActionClientDenied, diff --git a/internal/waf/waf.go b/internal/waf/waf.go index 3385452..e6de793 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -206,8 +206,8 @@ func (rq *request) allow() { answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow} if rq.h.config.AddLookupHeaders { - answer.ASN = rq.line.ASN - answer.Country = rq.line.Country + answer.ASN = rq.lookupAnswer.ASN + answer.Country = rq.lookupAnswer.Country } wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)