waf: heuristics pass facts through typed fields of decision (closes #442) #477

Merged
clawbot merged 3 commits from issue-442-decision-typed-facts into next 2026-11-19 23:43:01 +01:00
11 changed files with 70 additions and 44 deletions

No files matched your search

+12 -8
View File
@@ -45,8 +45,11 @@ func (a activeBan) check(
} }
ban, banned, madePermanent := check(rq.client, now) ban, banned, madePermanent := check(rq.client, now)
var expires string
if banned { if banned {
rq.line.BanExpires = banExpires(ban) expires = banExpires(ban)
rq.line.BanExpires = expires
} }
if madePermanent { if madePermanent {
@@ -61,7 +64,7 @@ func (a activeBan) check(
return &verdict{ return &verdict{
action: banAction(ban.Permanent()), action: banAction(ban.Permanent()),
figures: map[string]any{ figures: map[string]any{
figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires, figureNetblock: ban.Netblock.String(), "ban_expires": expires,
}, },
} }
} }
@@ -148,6 +151,7 @@ func (b banMaker) banForLimit(
rq.line.LimitHit = hit.Window rq.line.LimitHit = hit.Window
} }
rq.brokeLimit = true
rq.line.Offence = requestlog.OffenceLimit rq.line.Offence = requestlog.OffenceLimit
b.metrics.LimitHit(hit) b.metrics.LimitHit(hit)
@@ -259,9 +263,9 @@ func (b banMaker) withClientNotes(
status int, status int,
netblock netip.Prefix, netblock netip.Prefix,
) bans.Notes { ) bans.Notes {
notes.ASN = rq.line.ASN notes.ASN = rq.lookupAnswer.ASN
notes.ASName = rq.line.ASName notes.ASName = rq.lookupAnswer.ASName
notes.Country = rq.line.Country notes.Country = rq.lookupAnswer.Country
notes.Reputation = rq.reputation notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status) notes.Request = rq.noted(now, status)
notes.Requests = b.netblockRequests(netblock) notes.Requests = b.netblockRequests(netblock)
@@ -328,11 +332,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
func (rq *request) noted(now time.Time, status int) bans.Request { func (rq *request) noted(now time.Time, status int) bans.Request {
return bans.Request{ return bans.Request{
Time: now, Time: now,
Method: rq.line.Method, Method: rq.method,
Host: rq.line.Host, Host: rq.site,
Path: rq.requestURI, Path: rq.requestURI,
Status: status, Status: status,
UserAgent: rq.line.UserAgent, UserAgent: rq.userAgent,
} }
} }
+2 -2
View File
@@ -68,7 +68,7 @@ type biasedThresholds struct {
func (b biasedThresholds) limitPercentages( func (b biasedThresholds) limitPercentages(
rq *request, rq *request,
) (percentage, percentage) { ) (percentage, percentage) {
asn, country := rq.line.ASN, rq.line.Country asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country
unknown := percentage{percent: whole} unknown := percentage{percent: whole}
if country == "" && !rq.skippedLookupWait { if country == "" && !rq.skippedLookupWait {
@@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages(
otherTags := percentage{percent: whole} otherTags := percentage{percent: whole}
for _, tag := range rq.line.Tags { for _, tag := range rq.tags {
if tag != asnTag(asn) && tag != countryTag(country) { if tag != asnTag(asn) && tag != countryTag(country) {
otherTags = lowest( otherTags = lowest(
otherTags, otherTags,
+1 -1
View File
@@ -33,7 +33,7 @@ func (b byteLimits) check(
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
if !rq.counted || rq.line.WouldAction != "" { if !rq.counted || rq.wouldAction != "" {
return nil return nil
} }
+3 -3
View File
@@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch(
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN, ASN: rq.lookupAnswer.ASN,
ASName: rq.line.ASName, ASName: rq.lookupAnswer.ASName,
Country: rq.line.Country, Country: rq.lookupAnswer.Country,
Reason: "scored by the Core Rule Set at or over " + Reason: "scored by the Core Rule Set at or over " +
"SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD", "SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD",
Detail: detail, Detail: detail,
+1 -1
View File
@@ -27,7 +27,7 @@ func (c countries) check(
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
allowed := c.allowed allowed := c.allowed
country := rq.line.Country country := rq.lookupAnswer.Country
if len(allowed) == 0 || slices.Contains(allowed, country) { if len(allowed) == 0 || slices.Contains(allowed, country) {
return nil return nil
+2 -2
View File
@@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) {
rq.line.ASName = rq.lookupAnswer.ASName rq.line.ASName = rq.lookupAnswer.ASName
rq.line.Country = rq.lookupAnswer.Country rq.line.Country = rq.lookupAnswer.Country
if tag := asnTag(rq.line.ASN); tag != "" { if tag := asnTag(rq.lookupAnswer.ASN); tag != "" {
rq.addTag(tag) rq.addTag(tag)
} }
if tag := countryTag(rq.line.Country); tag != "" { if tag := countryTag(rq.lookupAnswer.Country); tag != "" {
rq.addTag(tag) rq.addTag(tag)
} }
} }
+1 -1
View File
@@ -54,7 +54,7 @@ func (r refusalLimit) check(
// What the client was sent, or in observe mode would have been. // What the client was sent, or in observe mode would have been.
status := rq.out.status status := rq.out.status
switch rq.line.WouldAction { switch rq.wouldAction {
case requestlog.ActionBlockedByBlockRule, case requestlog.ActionBlockedByBlockRule,
requestlog.ActionBlockedByCoreRuleSet: requestlog.ActionBlockedByCoreRuleSet:
status = http.StatusForbidden status = http.StatusForbidden
+3 -3
View File
@@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit(
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN, ASN: rq.lookupAnswer.ASN,
ASName: rq.line.ASName, ASName: rq.lookupAnswer.ASName,
Country: rq.line.Country, Country: rq.lookupAnswer.Country,
Reason: reason, Reason: reason,
Detail: detail, Detail: detail,
}) })
+42 -20
View File
@@ -24,42 +24,64 @@ type refusal struct {
action string action string
} }
// decision is what the checks note of a request: its log line, and what // decision is what the checks note of a request: its log line, what the
// counting the request needs once it has ended. waf keeps a copy of it // heuristics tell each other, and what counting the request needs once it has
// until the request's report comes. It holds no credentials, since the log line // ended. Nothing reads the log line back to decide. waf keeps a copy of it
// never does, and neither the request's objects nor their bodies. // until the request's report comes. It holds no credentials, since the log
// line never does, and neither the request's objects nor their bodies.
type decision struct { type decision struct {
line requestlog.Line line requestlog.Line
client netip.Addr client netip.Addr
// requestURI is the path with the query, as a ban's notes keep it. // method and userAgent are the request's method and User-Agent, and
requestURI string // requestURI its path with the query, as a ban's notes keep them.
// startRequest sets them.
method, userAgent, requestURI string
// tags are the request's tags, sorted, each once. addTag adds them, for
// tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and
// coreRuleSet.
tags []string
// site is, for a question, the host the visitor asked for, which the // site is, for a question, the host the visitor asked for, which the
// alerts name, since one waf serves many sites. // alerts and a ban's notes name, since one waf serves many sites.
// answerQuestion sets it.
site string site string
// allowed is true for a question answered with DispositionAllow. // allowed is true for a question answered with DispositionAllow.
allowed bool allowed bool
// lookedUp is true once the client's AS number and country have been // lookedUp is true once the client's AS number and country have been
// looked up, whether or not an answer was there, and lookupAnswer is // looked up, whether or not an answer was there, and lookupAnswer is
// what the lookup gave then, the zero Answer while GeoJS had given none. // what the lookup gave then, the zero Answer while GeoJS had given none.
// noteLookup sets them.
lookedUp bool lookedUp bool
lookupAnswer geojs.Answer lookupAnswer geojs.Answer
// skippedLookupWait is true for a request that did not wait for GeoJS and // skippedLookupWait is true for a request that did not wait for GeoJS and
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it, // found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
// so that not waiting never lowers its client's limits. // so that not waiting never lowers its client's limits. noteLookup sets
// it.
skippedLookupWait bool skippedLookupWait bool
// counted is true for a request the rate limits counted, whose bytes // counted is true for a request the rate limits counted, whose bytes
// the byte limits count once it has ended. rateLimitPercent and // the byte limits count once it has ended. rateLimitPercent and
// bytesLimitPercent are then its client's limit percentages for the rate // bytesLimitPercent are then its client's limit percentages for the rate
// limits and for the byte limits. // limits and for the byte limits. rateLimits sets them.
counted bool counted bool
rateLimitPercent, bytesLimitPercent percentage rateLimitPercent, bytesLimitPercent percentage
// attack is true for a request that matched a ban rule or asked for a // attack is true for a request that matched a ban rule or asked for a
// trap path, blockedByBlockRule for one a block rule refused, and // trap path, blockedByBlockRule for one a block rule refused, and
// blockedByCoreRuleSet for one the Core Rule Set refused, each an offence // blockedByCoreRuleSet for one the Core Rule Set refused, each an offence
// its client's history counts. // its client's history counts. trapPaths and ruleFiles set attack,
// ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet.
attack, blockedByBlockRule, blockedByCoreRuleSet bool attack, blockedByBlockRule, blockedByCoreRuleSet bool
// wouldAction is, in observe mode, the action enforce mode would have
// refused the request with, one of requestlog's Action constants, and ""
// otherwise. check sets it.
wouldAction string
// brokeLimit is true for a request that broke a limit, an offence its
// client's history counts. banForLimit sets it.
brokeLimit bool
// blocklisted is true once blocklists finds a blocklist that lists the
// client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does,
// and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit.
blocklisted, dnsblListed, abuseIPDBHit bool
// reputation is the reputation sources that list the client, for the // reputation is the reputation sources that list the client, for the
// notes of a ban the request makes. // notes of a ban the request makes. noteHit adds them.
reputation []bans.ReputationHit reputation []bans.ReputationHit
} }
@@ -72,11 +94,7 @@ type request struct {
in *http.Request in *http.Request
out *responseWriter out *responseWriter
// blocklisted is true once a blocklist is found to list the client, start time.Time
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
// AbuseIPDB's score of it is a hit.
blocklisted, dnsblListed, abuseIPDBHit bool
start time.Time
// checked is when the checks were done. // checked is when the checks were done.
checked time.Time checked time.Time
// refused is what the request was refused with, nil if it was not. // refused is what the request was refused with, nil if it was not.
@@ -106,6 +124,8 @@ func (h *handler) startRequest(
rq := &request{ rq := &request{
decision: decision{ decision: decision{
client: client, client: client,
method: r.Method,
userAgent: r.UserAgent(),
requestURI: r.URL.RequestURI(), requestURI: r.URL.RequestURI(),
line: requestlog.Line{ line: requestlog.Line{
Time: requestlog.FormatTime(start), Time: requestlog.FormatTime(start),
@@ -151,11 +171,12 @@ func (h *handler) startRequest(
} }
// addTag adds tag to the request's tags, which stay sorted and hold each tag // addTag adds tag to the request's tags, which stay sorted and hold each tag
// once. // once, and to its log line.
func (rq *request) addTag(tag string) { func (rq *request) addTag(tag string) {
at, found := slices.BinarySearch(rq.line.Tags, tag) at, found := slices.BinarySearch(rq.tags, tag)
if !found { if !found {
rq.line.Tags = slices.Insert(rq.line.Tags, at, tag) rq.tags = slices.Insert(rq.tags, at, tag)
rq.line.Tags = rq.tags
} }
} }
@@ -196,6 +217,7 @@ func (rq *request) check(ctx context.Context) *refusal {
switch { switch {
case rq.h.config.Observe: case rq.h.config.Observe:
rq.wouldAction = action
rq.line.WouldAction = action rq.line.WouldAction = action
return nil return nil
@@ -371,7 +393,7 @@ func (rq *request) addToHistory() {
Status: rq.out.status, Status: rq.out.status,
RequestBytes: rq.requestBytes, RequestBytes: rq.requestBytes,
ResponseBytes: rq.out.bytes, ResponseBytes: rq.out.bytes,
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, BrokeLimit: rq.brokeLimit,
Attack: rq.attack, Attack: rq.attack,
BlockedByBlockRule: rq.blockedByBlockRule, BlockedByBlockRule: rq.blockedByBlockRule,
BlockedByCoreRuleSet: rq.blockedByCoreRuleSet, BlockedByCoreRuleSet: rq.blockedByCoreRuleSet,
+1 -1
View File
@@ -23,7 +23,7 @@ func (t tagDeny) check(
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
for _, tag := range rq.line.Tags { for _, tag := range rq.tags {
if slices.Contains(t.tags, tag) { if slices.Contains(t.tags, tag) {
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
+2 -2
View File
@@ -202,8 +202,8 @@ func (rq *request) allow() {
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow} answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
if rq.h.config.AddLookupHeaders { if rq.h.config.AddLookupHeaders {
answer.ASN = rq.line.ASN answer.ASN = rq.lookupAnswer.ASN
answer.Country = rq.line.Country answer.Country = rq.lookupAnswer.Country
} }
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer) wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)