From 622109cea68c0eab0756b42ebcaa1c35b66146da Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 16:46:56 +0000 Subject: [PATCH] waf: each heuristic holds its own settings and parts (closes #441) Every heuristic is a struct built once in newHandler with the settings and collaborators it reads; no check reads the handler. The six lists are handler fields built there. Code several heuristics share takes what it needs as arguments or lives on a struct they hold: banMaker for bans and their alerts, reputationHitNoter for reputation hits, biasedThresholds for limit percentages. The handler keeps blocklists, rateLimits, trapPaths, ruleFiles and coreRuleSet as fields too, since refusedWhateverTheAnswer and findBesideLookup use them. Model: opus-5-5 --- internal/waf/abuseipdb.go | 29 ++-- internal/waf/abuseipdbblacklist.go | 4 +- internal/waf/admin.go | 10 +- internal/waf/bans.go | 151 +++++++++++-------- internal/waf/besidelookup.go | 24 +-- internal/waf/biased.go | 51 +++++-- internal/waf/blocklists.go | 20 ++- internal/waf/bytelimits.go | 20 ++- internal/waf/client.go | 11 +- internal/waf/coreruleset.go | 48 +++--- internal/waf/countries.go | 8 +- internal/waf/denynets.go | 13 +- internal/waf/denynets_internal_test.go | 47 ++++++ internal/waf/dnsbl.go | 20 ++- internal/waf/heuristics.go | 77 +--------- internal/waf/lookup.go | 8 +- internal/waf/proxy.go | 199 +++++++++++++++++++++---- internal/waf/ratelimits.go | 56 ++++--- internal/waf/refusallimit.go | 24 +-- internal/waf/reputation.go | 29 +++- internal/waf/request.go | 21 ++- internal/waf/routewatchtags.go | 15 +- internal/waf/rulefiles.go | 14 +- internal/waf/tagdeny.go | 12 +- internal/waf/taglists.go | 14 +- internal/waf/trappaths.go | 23 ++- internal/waf/waf.go | 16 +- 27 files changed, 632 insertions(+), 332 deletions(-) create mode 100644 internal/waf/denynets_internal_test.go diff --git a/internal/waf/abuseipdb.go b/internal/waf/abuseipdb.go index fe1d75a..d3c4964 100644 --- a/internal/waf/abuseipdb.go +++ b/internal/waf/abuseipdb.go @@ -22,31 +22,42 @@ import ( // an offence, and the request does not wait for the answer. The score is then // used for each address of the client. Its figures are the score and // SWW_WAF_ABUSEIPDB_MIN_SCORE. -type abuseIPDB struct{} +type abuseIPDB struct { + abuseIPDB *reputation.AbuseIPDB + // limiter is the table of clients the rate limits count, whose histories + // say which clients are offenders. + limiter *ratelimit.Limiter + keySet bool // SWW_WAF_ABUSEIPDB_KEY is set + minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE + action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + hitNoter reputationHitNoter +} func (abuseIPDB) name() string { return "abuseipdb" } -func (abuseIPDB) check( +func (a abuseIPDB) check( ctx context.Context, rq *request, _ time.Time, ) *verdict { - if rq.h.config.AbuseIPDBKey == "" { + if !a.keySet { return nil } - client := clientGroup(rq.h.config, rq.client) - held, _ := rq.h.limiter.Client(client) + client := clientGroup(a.ipv6GroupPrefix, rq.client) + held, _ := a.limiter.Client(client) offender := held.History.Offences != ratelimit.Offences{} - score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender) + score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender) if !hit { return nil } rq.abuseIPDBHit = true rq.addTag(config.AbuseIPDBTag) - rq.noteHit( + a.hitNoter.noteHit( + rq, bans.ReputationHit{ Source: reputation.AbuseIPDBSource, Score: &score, @@ -54,14 +65,14 @@ func (abuseIPDB) check( "scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE", ) - if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny { + if a.action != config.ReputationActionDeny { return nil } return &verdict{ action: requestlog.ActionClientDenied, figures: map[string]any{ - figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore, + figureScore: score, "min_score": a.minScore, }, } } diff --git a/internal/waf/abuseipdbblacklist.go b/internal/waf/abuseipdbblacklist.go index c4dfb84..63e49a2 100644 --- a/internal/waf/abuseipdbblacklist.go +++ b/internal/waf/abuseipdbblacklist.go @@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist( scores := make(map[netip.Prefix]int64, len(listed)) for _, client := range listed { - netblock := banNetblock(cfg, client.Addr) + netblock := banNetblock( + cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr, + ) scores[netblock] = max(scores[netblock], client.Score) } diff --git a/internal/waf/admin.go b/internal/waf/admin.go index 99ddec2..e98dbef 100644 --- a/internal/waf/admin.go +++ b/internal/waf/admin.go @@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) { return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone) } - return banNetblock(h.config, addr), nil + return banNetblock( + h.config.BanScopeV4Prefix, + h.config.IPv6GroupPrefix, + addr, + ), nil } // expiry returns when a ban made at now for duration ends: duration @@ -295,7 +299,9 @@ func (rq *request) showClient() { Bans: state.BanEntries(rq.h.ledger.Covering(addr)), } - client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr)) + client, seen := rq.h.limiter.Client( + clientGroup(rq.h.config.IPv6GroupPrefix, addr), + ) if seen { answer.Client = &client } diff --git a/internal/waf/bans.go b/internal/waf/bans.go index f6ca9a8..0c20e6e 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -1,7 +1,7 @@ package waf -// The heuristic active_ban, and the code that makes bans, which several -// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's. +// The heuristic active_ban, and banMaker, which makes the bans of several +// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's. import ( "context" @@ -11,6 +11,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -25,18 +26,22 @@ func (rq *request) banResponse(action string) *refusal { // in, in force at now. It notes for the log line when that ban ends. A request // that makes the ban permanent, or in observe mode would have, raises the alert // for it. Its figures are the ban's netblock and when it ends. -type activeBan struct{} +type activeBan struct { + banMaker banMaker +} func (activeBan) name() string { return "active_ban" } -func (activeBan) check( +func (a activeBan) check( _ context.Context, rq *request, now time.Time, ) *verdict { - check := rq.h.ledger.Check - if rq.h.config.Observe { - check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is + ledger := a.banMaker.ledger + + check := ledger.Check + if a.banMaker.observe { + check = ledger.Find // the ban refuses nothing, and stays as it is } ban, banned, madePermanent := check(rq.client, now) @@ -46,7 +51,7 @@ func (activeBan) check( if madePermanent { ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is - rq.alertBan(ban) + a.banMaker.alertBan(rq, ban) } if !banned { @@ -81,11 +86,12 @@ func limitFigures(hit ratelimit.Hit) map[string]any { // countedBytes returns the request's bytes, once it has ended, as the // byte limits and the anomaly thresholds count them: the response's body -// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says. -func (rq *request) countedBytes() int64 { +// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED, +// says. +func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 { response, request := rq.out.bytes, rq.requestBytes - switch rq.h.config.BytesCounted { + switch bytesCounted { case config.BytesCountedResponse: return response case config.BytesCountedRequest: @@ -96,7 +102,29 @@ func (rq *request) countedBytes() int64 { return response + request } -// banForLimit bans the client's netblock at now for a broken limit, the +// banMaker makes the bans of the heuristics that ban a client, in the ban +// ledger, and raises the alert for each. In observe mode it makes no ban, and +// raises the alert for the ban it would have made. +type banMaker struct { + ledger *bans.Ledger + // limiter is the table of clients the rate limits count. A ban for a limit + // sets the client's counters back to zero, and a ban's notes give how many + // requests its netblock has sent. + limiter *ratelimit.Limiter + metrics *metrics.Metrics + alerts *alerts.Queue + // observe is true in observe mode (SWW_WAF_MODE). + observe bool + // banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a + // clear sign of attack give. + banResponse int + // banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix + // SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers. + banScopeV4Prefix int + ipv6GroupPrefix int +} + +// banForLimit bans the netblock of rq's client at now for a broken limit, the // one hit names, notes the offence for the log line and counts the hit in // the metrics. status is what the client was sent, or is sent: // SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for @@ -106,7 +134,8 @@ func (rq *request) countedBytes() int64 { // the client's counters back to zero. In observe mode it makes no ban and // sets nothing back, and raises the alert for the ban it would have made, // if that alert would be sent. -func (rq *request) banForLimit( +func (b banMaker) banForLimit( + rq *request, now time.Time, hit ratelimit.Hit, status int, @@ -121,11 +150,11 @@ func (rq *request) banForLimit( } rq.line.Offence = requestlog.OffenceLimit - rq.h.metrics.LimitHit(hit) - netblock := banNetblock(rq.h.config, rq.client) - if rq.h.config.Observe && - !rq.wouldAlertBan(netblock, now, bans.CauseLimit) { + b.metrics.LimitHit(hit) + + netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client) + if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseLimit) { return } @@ -139,7 +168,7 @@ func (rq *request) banForLimit( Count: hit.Count, Reputation: rq.reputation, Request: rq.noted(now, status), - Requests: rq.netblockRequests(netblock), + Requests: b.netblockRequests(netblock), } switch hit.Kind { @@ -149,35 +178,36 @@ func (rq *request) banForLimit( notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged() } - if rq.h.config.Observe { - ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes) + if b.observe { + ban, wouldBan := b.ledger.WouldBanForLimit(netblock, now, notes) if wouldBan { - rq.alertBan(ban) + b.alertBan(rq, ban) } return } - ban, made := rq.h.ledger.BanForLimit(netblock, now, notes) - rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client)) + ban, made := b.ledger.BanForLimit(netblock, now, notes) + b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client)) rq.line.BanExpires = banExpires(ban) if made { - rq.alertBan(ban) + b.alertBan(rq, ban) } } -// banForAttack bans the client's netblock at now for a clear sign of +// banForAttack bans the netblock of rq's client at now for a clear sign of // attack, which notes name: the ban rule that matched, or the trap path // asked for. It fills in the rest of the notes. In observe mode it makes // no ban, and raises the alert for the ban it would have made, if that // alert would be sent. It returns the action for that ban. -func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { - netblock := banNetblock(rq.h.config, rq.client) - if rq.h.config.Observe && - !rq.wouldAlertBan(netblock, now, bans.CauseAttack) { +func (b banMaker) banForAttack( + rq *request, now time.Time, notes bans.Notes, +) string { + netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client) + if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseAttack) { return banAction( - rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack), + b.ledger.WouldBePermanent(netblock, now, bans.CauseAttack), ) } @@ -185,23 +215,23 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { notes.ASName = rq.line.ASName notes.Country = rq.line.Country notes.Reputation = rq.reputation - notes.Request = rq.noted(now, rq.h.config.BanResponse) - notes.Requests = rq.netblockRequests(netblock) + notes.Request = rq.noted(now, b.banResponse) + notes.Requests = b.netblockRequests(netblock) - if rq.h.config.Observe { - ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes) + if b.observe { + ban, wouldBan := b.ledger.WouldBanForAttack(netblock, now, notes) if wouldBan { - rq.alertBan(ban) + b.alertBan(rq, ban) } return banAction(ban.Permanent()) } - ban, made := rq.h.ledger.BanForAttack(netblock, now, notes) + ban, made := b.ledger.BanForAttack(netblock, now, notes) rq.line.BanExpires = banExpires(ban) if made { - rq.alertBan(ban) + b.alertBan(rq, ban) } return banAction(ban.Permanent()) @@ -212,23 +242,22 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { // have made is worked out only then, at most once per // SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the // netblock's requests, which can mean going through every client. -func (rq *request) wouldAlertBan( +func (b banMaker) wouldAlertBan( netblock netip.Prefix, now time.Time, cause string, ) bool { event := alerts.EventBan - if rq.h.ledger.WouldBePermanent(netblock, now, cause) { + if b.ledger.WouldBePermanent(netblock, now, cause) { event = alerts.EventPermanentBan } - return rq.h.alerts.WouldSend(event, netblock) + return b.alerts.WouldSend(event, netblock) } -// alertBan raises the alert for ban, which the request made, or made -// permanent: permanent_ban for a permanent ban, ban for another. Its -// detail gives the ban's cause, when it ends, and its notes, and in -// observe mode, where ban is the ban that would have been made, or made -// permanent, mode, observe. -func (rq *request) alertBan(ban bans.Ban) { +// alertBan raises the alert for ban, which rq made, or made permanent: +// permanent_ban for a permanent ban, ban for another. Its detail gives the +// ban's cause, when it ends, and its notes, and in observe mode, where ban is +// the ban that would have been made, or made permanent, mode, observe. +func (b banMaker) alertBan(rq *request, ban bans.Ban) { event := alerts.EventBan if ban.Permanent() { event = alerts.EventPermanentBan @@ -237,11 +266,11 @@ func (rq *request) alertBan(ban bans.Ban) { detail := map[string]any{ "cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes, } - if rq.h.config.Observe { + if b.observe { detail["mode"] = "observe" } - rq.h.alerts.Raise(alerts.Alert{ + b.alerts.Raise(alerts.Alert{ Event: event, Host: rq.site, Client: rq.client, @@ -254,6 +283,13 @@ func (rq *request) alertBan(ban bans.Ban) { }) } +// netblockRequests is how many requests netblock has sent since it was +// first seen, the request being banned for included: the histories count it +// only once it has ended. +func (b banMaker) netblockRequests(netblock netip.Prefix) int64 { + return b.limiter.Requests(netblock) + 1 +} + // noted is the request, at now, with status, what the client was sent, or // in observe mode would have been, as the notes of the ban it makes keep // it. @@ -268,23 +304,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request { } } -// netblockRequests is how many requests netblock has sent since it was -// first seen, this one included: the histories count it only once it has -// ended. -func (rq *request) netblockRequests(netblock netip.Prefix) int64 { - return rq.h.limiter.Requests(netblock) + 1 -} - // banNetblock is the netblock a ban on client covers: its IPv4 address, -// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup -// counts it in. -func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix { +// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group +// clientGroup counts it in, of length ipv6GroupPrefix. +func banNetblock( + banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr, +) netip.Prefix { addr := client.Unmap() if addr.Is4() { - return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked() + return netip.PrefixFrom(addr, banScopeV4Prefix).Masked() } - return clientGroup(cfg, addr) + return clientGroup(ipv6GroupPrefix, addr) } // banExpires is when ban ends, as the log line gives it: a time, or diff --git a/internal/waf/besidelookup.go b/internal/waf/besidelookup.go index d1edf44..ffdc026 100644 --- a/internal/waf/besidelookup.go +++ b/internal/waf/besidelookup.go @@ -54,7 +54,7 @@ func (rq *request) runBesideLookup( ) *verdict { var running atomic.Pointer[string] - running.Store(new(trapPaths{}.name())) + running.Store(new(rq.h.trapPaths.name())) found := make(chan *besideLookupFindings, 1) @@ -92,7 +92,7 @@ func (rq *request) runBesideLookup( return v } - return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now()) + return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now()) } // findBesideLookup does the work of the heuristics of besideLookup on the @@ -105,39 +105,39 @@ func (rq *request) findBesideLookup( ) *besideLookupFindings { findings := &besideLookupFindings{} - findings.trapPath = rq.trapPath() + findings.trapPath = rq.h.trapPaths.trapPath(rq) if findings.trapPath != "" { findings.refuses = true return findings } - running.Store(new(ruleFiles{}.name())) + running.Store(new(rq.h.ruleFiles.name())) - findings.rules = rq.h.rules.Match(rq.in) + findings.rules = rq.h.ruleFiles.rules.Match(rq.in) if refusedByRule(findings.rules) { findings.refuses = true return findings } - running.Store(new(coreRuleSet{}.name())) + coreRuleSet := rq.h.coreRuleSet + running.Store(new(coreRuleSet.name())) - cfg := rq.h.config - if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff || - pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) { + if coreRuleSet.mode == config.CoreRuleSetModeOff || + pathExempt(rq.in, coreRuleSet.exemptPaths) { return findings } start := time.Now() // A question's body has been read into memory, so that reading it here // cannot fail. - findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect( + findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect( rq.in, rq.client, rq.in.Body, ) findings.inspectionTime = time.Since(start) findings.inspected = true - findings.refuses = rq.refusedByCoreRuleSet(findings.inspection) + findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection) return findings } @@ -151,7 +151,7 @@ func (rq *request) applyFindings( ) *verdict { rq.findings = findings - return rq.runHeuristics(ctx, besideLookup(), rq.h.now()) + return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now()) } // timedOut refuses the request for the heuristics of besideLookup not having diff --git a/internal/waf/biased.go b/internal/waf/biased.go index 7f5739d..fef9e87 100644 --- a/internal/waf/biased.go +++ b/internal/waf/biased.go @@ -2,6 +2,7 @@ package waf import ( "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // whole is the percentage of each limit a client gets when no biased @@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool { cfg.ASNLimitPercentURL != "" } +// biasedThresholds are the settings and the lists by which limitPercentages +// lowers a client's limits. +type biasedThresholds struct { + // lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names. + lists *reputation.Lists + tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT + asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT + countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT + unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT + // blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION + // and its percentage. + blocklistAction config.ReputationAction + blocklistLimitPercent int64 + // dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are + // SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage. + dnsblAbuseIPDBAction config.ReputationAction + dnsblAbuseIPDBLimitPercent int64 +} + // limitPercentages returns the client's limit percentages, for the rate // limits and for the byte limits, by its tags, its AS number and country // as looked up, each "" when unknown, and the blocklists, DNSBL zones and @@ -45,37 +65,38 @@ func biasedThresholdsSet(cfg *config.WAF) bool { // the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for // an AS number or a country they list. -func (rq *request) limitPercentages() (percentage, percentage) { - cfg := rq.h.config +func (b biasedThresholds) limitPercentages( + rq *request, +) (percentage, percentage) { asn, country := rq.line.ASN, rq.line.Country unknown := percentage{percent: whole} if country == "" && !rq.skippedLookupWait { unknown = percentage{ - cfg.UnknownCountryLimitPercent, + b.unknownCountryLimitPercent, "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", } } fetched := percentage{percent: whole} - if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed { + if percent, listed := b.lists.ASNLimitPercent(asn); listed { fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"} } blocklisted := percentage{percent: whole} if rq.blocklisted && - cfg.BlocklistAction == config.ReputationActionLimit { + b.blocklistAction == config.ReputationActionLimit { blocklisted = percentage{ - cfg.BlocklistLimitPercent, + b.blocklistLimitPercent, "SWW_WAF_BLOCKLIST_ACTION", } } dnsblAbuseIPDBListed := percentage{percent: whole} if (rq.dnsblListed || rq.abuseIPDBHit) && - cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit { + b.dnsblAbuseIPDBAction == config.ReputationActionLimit { dnsblAbuseIPDBListed = percentage{ - cfg.DNSBLAbuseIPDBLimitPercent, + b.dnsblAbuseIPDBLimitPercent, "SWW_WAF_DNSBL_ABUSEIPDB_ACTION", } } @@ -83,11 +104,11 @@ func (rq *request) limitPercentages() (percentage, percentage) { const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT" asnRequests := lowest( - given(cfg.TagLimitPercent, asnTag(asn), tagSetting), + given(b.tagLimitPercent, asnTag(asn), tagSetting), fetched, ) countryRequests := given( - cfg.TagLimitPercent, + b.tagLimitPercent, countryTag(country), tagSetting, ) @@ -98,23 +119,23 @@ func (rq *request) limitPercentages() (percentage, percentage) { if tag != asnTag(asn) && tag != countryTag(country) { otherTags = lowest( otherTags, - given(cfg.TagLimitPercent, tag, tagSetting), + given(b.tagLimitPercent, tag, tagSetting), ) } } asnBytes, countryBytes := asnRequests, countryRequests - if _, listed := cfg.ASNBytesLimitPercent[asn]; listed { + if _, listed := b.asnBytesLimitPercent[asn]; listed { asnBytes = given( - cfg.ASNBytesLimitPercent, + b.asnBytesLimitPercent, asn, "SWW_WAF_ASN_BYTES_LIMIT_PERCENT", ) } - if _, listed := cfg.CountryBytesLimitPercent[country]; listed { + if _, listed := b.countryBytesLimitPercent[country]; listed { countryBytes = given( - cfg.CountryBytesLimitPercent, + b.countryBytesLimitPercent, country, "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", ) diff --git a/internal/waf/blocklists.go b/internal/waf/blocklists.go index 7879634..0e35966 100644 --- a/internal/waf/blocklists.go +++ b/internal/waf/blocklists.go @@ -8,6 +8,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -15,20 +16,24 @@ import ( // and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it // lowers the client's limits instead (see limitPercentages), and being log, it // does nothing more. Its figure is the blocklists that list the client. -type blocklists struct{} +type blocklists struct { + lists *reputation.Lists + action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION + hitNoter reputationHitNoter +} func (blocklists) name() string { return "blocklists" } -func (blocklists) check( +func (b blocklists) check( _ context.Context, rq *request, _ time.Time, ) *verdict { - listedBy := rq.h.lists.ListedBy(rq.client) + listedBy := b.lists.ListedBy(rq.client) rq.blocklisted = len(listedBy) > 0 - rq.noteListed(listedBy, "listed by a blocklist") + b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist") - if !rq.refusedByBlocklists(listedBy) { + if !b.refusedByBlocklists(listedBy) { return nil } @@ -41,7 +46,6 @@ func (blocklists) check( // refusedByBlocklists reports whether the heuristic blocklists refuses a // client, given listedBy, the blocklists that list it: it does when one lists // it and SWW_WAF_BLOCKLIST_ACTION is deny. -func (rq *request) refusedByBlocklists(listedBy []string) bool { - return len(listedBy) > 0 && rq.h.config.BlocklistAction == - config.ReputationActionDeny +func (b blocklists) refusedByBlocklists(listedBy []string) bool { + return len(listedBy) > 0 && b.action == config.ReputationActionDeny } diff --git a/internal/waf/bytelimits.go b/internal/waf/bytelimits.go index c1f3c32..73fba6e 100644 --- a/internal/waf/bytelimits.go +++ b/internal/waf/bytelimits.go @@ -6,6 +6,9 @@ package waf import ( "context" "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) // byteLimits counts the request's bytes, as countedBytes gives them, for the @@ -16,11 +19,16 @@ import ( // the client over a byte limit, as its limit percentage for the byte limits // lowers it, break it, which bans the client; the response was passed on whole. // Its figures are the window, the count and the limit. -type byteLimits struct{} +type byteLimits struct { + limiter *ratelimit.Limiter + bytesCounted config.BytesCounted // SWW_WAF_BYTES_COUNTED + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + banMaker banMaker +} func (byteLimits) name() string { return "byte_limits" } -func (byteLimits) check( +func (b byteLimits) check( _ context.Context, rq *request, now time.Time, @@ -29,10 +37,10 @@ func (byteLimits) check( return nil } - counts, hit, over := rq.h.limiter.CountBytes( - clientGroup(rq.h.config, rq.client), + counts, hit, over := b.limiter.CountBytes( + clientGroup(b.ipv6GroupPrefix, rq.client), now, - rq.countedBytes(), + rq.countedBytes(b.bytesCounted), rq.bytesLimitPercent.percent, ) rq.line.Counts.MinuteBytes = counts.MinuteBytes @@ -43,7 +51,7 @@ func (byteLimits) check( return nil } - rq.banForLimit(now, hit, rq.out.status) + b.banMaker.banForLimit(rq, now, hit, rq.out.status) return &verdict{figures: limitFigures(hit)} } diff --git a/internal/waf/client.go b/internal/waf/client.go index 8381f0a..549858a 100644 --- a/internal/waf/client.go +++ b/internal/waf/client.go @@ -5,8 +5,6 @@ import ( "net/http" "net/netip" "slices" - - "sneak.berlin/go/smallwebwaf/internal/config" ) // peerAddress is the address of the request's TCP peer. @@ -45,12 +43,13 @@ func scheme(r *http.Request, isQuestion bool) string { // clientGroup is the client a request is counted toward: its IPv4 // address, or its IPv6 group, the netblock its IPv6 address is in of the -// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser -// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. -func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix { +// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since +// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as +// IPv4. +func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix { addr = addr.Unmap() if addr.Is6() { - return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked() + return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked() } return netip.PrefixFrom(addr, addr.BitLen()) diff --git a/internal/waf/coreruleset.go b/internal/waf/coreruleset.go index 5c401cf..fd7f3c1 100644 --- a/internal/waf/coreruleset.go +++ b/internal/waf/coreruleset.go @@ -10,6 +10,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/coreruleset" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -22,11 +23,22 @@ import ( // exploitsignature, raises the core_rule_set_match alert, and in block mode // refuses the request with 403, which is an offence its client's history // counts. Its figures are the score and the threshold. -type coreRuleSet struct{} +type coreRuleSet struct { + // inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is + // off. + inspector CoreRuleSetInspector + metrics *metrics.Metrics + alerts *alerts.Queue + mode config.CoreRuleSetMode // SWW_WAF_CORE_RULE_SET_MODE + exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS + scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD + observe bool // SWW_WAF_MODE is observe + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX +} func (coreRuleSet) name() string { return "core_rule_set" } -func (coreRuleSet) check( +func (c coreRuleSet) check( _ context.Context, rq *request, _ time.Time, @@ -36,7 +48,6 @@ func (coreRuleSet) check( return nil } - cfg := rq.h.config result := findings.inspection rq.requestBytes = int64(len(findings.read)) @@ -47,18 +58,18 @@ func (coreRuleSet) check( rq.line.CoreRuleSetScore = &result.Score for _, id := range result.RuleIDs { - rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id) + c.metrics.CoreRuleSetMatched(string(c.mode), id) } - threshold := cfg.CoreRuleSetScoreThreshold + threshold := c.scoreThreshold if threshold == 0 || result.Score < threshold { return nil } rq.addTag(config.ExploitSignatureTag) - rq.alertCoreRuleSetMatch(result) + c.alertCoreRuleSetMatch(rq, result) - if !rq.refusedByCoreRuleSet(result) { + if !c.refusedByCoreRuleSet(result) { return nil } @@ -76,20 +87,21 @@ func (coreRuleSet) check( // refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core // Rule Set found result of: whether the score is at or over // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode. -func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool { - cfg := rq.h.config - threshold := cfg.CoreRuleSetScoreThreshold +func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool { + threshold := c.scoreThreshold - return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock && + return c.mode == config.CoreRuleSetModeBlock && threshold != 0 && result.Score >= threshold } -// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request, -// which the Core Rule Set scored at result, at or over +// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the +// Core Rule Set scored at result, at or over // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the // score, the method and the path with the query, and, for a request that is // not refused for it, the mode: detect, or observe in observe mode. -func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) { +func (c coreRuleSet) alertCoreRuleSetMatch( + rq *request, result coreruleset.Result, +) { detail := map[string]any{ "rule_ids": result.RuleIDs, "score": result.Score, @@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) { } switch { - case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect: + case c.mode == config.CoreRuleSetModeDetect: detail["mode"] = string(config.CoreRuleSetModeDetect) - case rq.h.config.Observe: + case c.observe: detail["mode"] = "observe" } - rq.h.alerts.Raise(alerts.Alert{ + c.alerts.Raise(alerts.Alert{ Event: alerts.EventCoreRuleSetMatch, Host: rq.site, Client: rq.client, - Netblock: clientGroup(rq.h.config, rq.client), + Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, diff --git a/internal/waf/countries.go b/internal/waf/countries.go index 7bae8d5..8515726 100644 --- a/internal/waf/countries.go +++ b/internal/waf/countries.go @@ -14,16 +14,18 @@ import ( // does not hold the client's country as it was looked up. A client without a // country, or whose country cannot be found, is refused. Its figures are the // country and the setting that refused it. -type countries struct{} +type countries struct { + allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES +} func (countries) name() string { return "countries" } -func (countries) check( +func (c countries) check( _ context.Context, rq *request, _ time.Time, ) *verdict { - allowed := rq.h.config.ExclusivelyAllowedCountries + allowed := c.allowed country := rq.line.Country if len(allowed) == 0 || slices.Contains(allowed, country) { diff --git a/internal/waf/denynets.go b/internal/waf/denynets.go index 97bcb04..a33a843 100644 --- a/internal/waf/denynets.go +++ b/internal/waf/denynets.go @@ -4,6 +4,7 @@ package waf import ( "context" + "net/netip" "time" "sneak.berlin/go/smallwebwaf/internal/requestlog" @@ -11,12 +12,18 @@ import ( // denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of // the setting the client is in. -type denyNets struct{} +type denyNets struct { + netblocks []netip.Prefix // SWW_WAF_DENY_NETS +} func (denyNets) name() string { return "deny_nets" } -func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict { - for _, netblock := range rq.h.config.DenyNets { +func (d denyNets) check( + _ context.Context, + rq *request, + _ time.Time, +) *verdict { + for _, netblock := range d.netblocks { if netblock.Contains(rq.client) { return &verdict{ action: requestlog.ActionClientDenied, diff --git a/internal/waf/denynets_internal_test.go b/internal/waf/denynets_internal_test.go new file mode 100644 index 0000000..51f5737 --- /dev/null +++ b/internal/waf/denynets_internal_test.go @@ -0,0 +1,47 @@ +package waf + +import ( + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) { + t.Parallel() + + heuristic := denyNets{ + netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}, + } + + // The request has no handler: the heuristic reads only what it holds. + inside := &request{ + decision: decision{client: netip.MustParseAddr("198.51.100.7")}, + } + + v := heuristic.check(t.Context(), inside, time.Time{}) + if v == nil { + t.Fatal("a client inside the netblock was let through") + } + + if v.action != requestlog.ActionClientDenied { + t.Errorf( + "action = %q, want %q", + v.action, + requestlog.ActionClientDenied, + ) + } + + if got := v.figures[figureNetblock]; got != "198.51.100.0/24" { + t.Errorf("netblock figure = %v, want 198.51.100.0/24", got) + } + + outside := &request{ + decision: decision{client: netip.MustParseAddr("203.0.113.7")}, + } + + if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil { + t.Errorf("a client outside the netblock was refused: %+v", v) + } +} diff --git a/internal/waf/dnsbl.go b/internal/waf/dnsbl.go index 51572f7..79f6b4b 100644 --- a/internal/waf/dnsbl.go +++ b/internal/waf/dnsbl.go @@ -9,6 +9,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -19,21 +20,28 @@ import ( // being log, it does nothing more. A zone without a verdict on the client is // asked about it in the background, and the request does not wait for the // answer. Its figure is the zones that list the client, their keys masked. -type dnsbl struct{} +type dnsbl struct { + dnsbl *reputation.DNSBL + action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION + hitNoter reputationHitNoter +} func (dnsbl) name() string { return "dnsbl" } -func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict { - listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client) +func (d dnsbl) check( + ctx context.Context, + rq *request, + _ time.Time, +) *verdict { + listedBy := d.dnsbl.ListedBy(ctx, rq.client) rq.dnsblListed = len(listedBy) > 0 - rq.noteListed(listedBy, "listed by a DNSBL zone") + d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone") for _, zone := range listedBy { rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone)) } - if !rq.dnsblListed || - rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny { + if !rq.dnsblListed || d.action != config.ReputationActionDeny { return nil } diff --git a/internal/waf/heuristics.go b/internal/waf/heuristics.go index 9fb7e59..1161b1b 100644 --- a/internal/waf/heuristics.go +++ b/internal/waf/heuristics.go @@ -5,9 +5,10 @@ import ( "time" ) -// To add a heuristic, write one file with a type that implements heuristic, -// and add one line for it to one of the lists below. Its settings go in -// config.WAF, and the comment at the top of its file names them. +// To add a heuristic, write one file with a struct that implements heuristic +// and holds the settings and parts it reads. Add one line for it to one of the +// lists that newHandler builds, which hands it those settings and parts. Its +// settings go in config.WAF, and the comment at the top of its file names them. // heuristic is one of the checks that decide what waf does with a // request. @@ -36,78 +37,10 @@ const ( figureNetblock = "netblock" ) -// The lists of heuristics, each in the order its heuristics run. A client in -// SWW_WAF_ALLOW_NETS skips them all. - -// beforeLookup are the heuristics that run before the client's AS number and -// country are looked up, so that a client they refuse is not looked up. -// header_limits comes first: it only measures and counts, and a request it -// refuses causes no other work. -func beforeLookup() []heuristic { - return []heuristic{ - headerLimits{}, - tagLists{}, - routeWatchTags{}, - denyNets{}, - activeBan{}, - } -} - -// afterLookup are the heuristics that run once the lookup has been made. -func afterLookup() []heuristic { - return []heuristic{ - countries{}, - blocklists{}, - dnsbl{}, - abuseIPDB{}, - tagDeny{}, - rateLimits{}, - } -} - -// besideLookup are the heuristics that only read the request. Their work runs -// beside the lookup and afterLookup, in findBesideLookup, which changes nothing -// of the request. Their check then applies what that work found, and runs only -// once waf takes it. -func besideLookup() []heuristic { - return []heuristic{ - trapPaths{}, - ruleFiles{}, - coreRuleSet{}, - } -} - -// afterBesideLookup are the heuristics that run once afterLookup and -// besideLookup have both let the request through. tag_deny runs a second time, -// on the tag rule_files and core_rule_set can add. -func afterBesideLookup() []heuristic { - return []heuristic{ - tagDeny{}, - } -} - -// afterAnswer are the heuristics that run on a request once waf has -// answered it, before its decision line is written. They ban the client, and do -// not change what the request got: their verdict's action is not used. -func afterAnswer() []heuristic { - return []heuristic{ - refusalLimit{}, - } -} - -// onReport are the heuristics that run on an allowed request when rproxy's -// report of it comes, before its report line is written. Like afterAnswer, they -// only ban. -func onReport() []heuristic { - return []heuristic{ - byteLimits{}, - } -} - // runAfterAnswer runs the heuristics of afterAnswer on the request, at the time // it is called. func (rq *request) runAfterAnswer(ctx context.Context) { - rq.runBanningHeuristics(ctx, afterAnswer()) + rq.runBanningHeuristics(ctx, rq.h.afterAnswer) } // runHeuristics runs list, one of those checkClient runs, on the request at now, diff --git a/internal/waf/lookup.go b/internal/waf/lookup.go index 9451299..b415b65 100644 --- a/internal/waf/lookup.go +++ b/internal/waf/lookup.go @@ -56,7 +56,7 @@ func (rq *request) startLookUp( return result } - client := clientGroup(rq.h.config, rq.client) + client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client) if rq.h.config.LookupSource == config.LookupSourceFile { result <- lookupResult{ @@ -143,8 +143,10 @@ func AddLookup( ) { limiter.AddLookup(answer.Client, answer.Answered, answer.ASN, answer.ASName, answer.Country) - ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()), - answer.ASN, answer.ASName, answer.Country) + netblock := banNetblock( + cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(), + ) + ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country) } // GeoJSFailed logs a failure of GeoJS to processLog, raises a diff --git a/internal/waf/proxy.go b/internal/waf/proxy.go index a65430a..ac50af5 100644 --- a/internal/waf/proxy.go +++ b/internal/waf/proxy.go @@ -133,15 +133,19 @@ type Server struct { } // newHandler returns the handler with what every request shares, the parts -// in params among it. +// in params among it. It builds the heuristics, each with the settings and +// parts it reads, and the lists they run in. +// +//nolint:funlen // a few lines for each heuristic, a list that grows with them func newHandler(params Params) *handler { awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) if err != nil { panic(err) // New fails only for a size below one } - return &handler{ - config: params.Config, + cfg := params.Config + h := &handler{ + config: cfg, requestLog: params.RequestLog, processLog: params.ProcessLog, now: params.Now, @@ -152,47 +156,142 @@ func newHandler(params Params) *handler { geojs: params.GeoJS, anomalies: params.Anomalies, lookupFile: params.LookupFile, - lists: params.Lists, - dnsbl: params.DNSBL, - abuseIPDB: params.AbuseIPDB, - routeWatch: params.RouteWatch, - rules: params.Rules, - coreRuleSet: params.CoreRuleSet, - alerts: params.Alerts, // The country list, the headers, the tag settings that name an asn= // or country= tag and the biased thresholds act on the answer before // the request goes on. - waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 || - params.Config.AddLookupHeaders || - params.Config.TagsNeedLookup() || - biasedThresholdsSet(params.Config), + waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 || + cfg.AddLookupHeaders || + cfg.TagsNeedLookup() || + biasedThresholdsSet(cfg), besideLookupLimit: cmp.Or( params.BesideLookupTimeLimit, besideLookupTimeLimit, ), } + + sharedBanMaker := banMaker{ + ledger: params.Ledger, + limiter: params.Limiter, + metrics: params.Metrics, + alerts: params.Alerts, + observe: cfg.Observe, + banResponse: cfg.BanResponse, + banScopeV4Prefix: cfg.BanScopeV4Prefix, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + } + hitNoter := reputationHitNoter{ + metrics: params.Metrics, + alerts: params.Alerts, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + } + + h.blocklists = blocklists{ + lists: params.Lists, + action: cfg.BlocklistAction, + hitNoter: hitNoter, + } + h.rateLimits = rateLimits{ + limiter: params.Limiter, + exemptNets: cfg.RateLimitExemptNets, + exemptPaths: cfg.RateLimitExemptPaths, + banResponse: cfg.BanResponse, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + biasedThresholds: biasedThresholds{ + lists: params.Lists, + tagLimitPercent: cfg.TagLimitPercent, + asnBytesLimitPercent: cfg.ASNBytesLimitPercent, + countryBytesLimitPercent: cfg.CountryBytesLimitPercent, + unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent, + blocklistAction: cfg.BlocklistAction, + blocklistLimitPercent: cfg.BlocklistLimitPercent, + dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction, + dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent, + }, + banMaker: sharedBanMaker, + } + h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker} + h.ruleFiles = ruleFiles{ + rules: params.Rules, + metrics: params.Metrics, + banMaker: sharedBanMaker, + } + h.coreRuleSet = coreRuleSet{ + inspector: params.CoreRuleSet, + metrics: params.Metrics, + alerts: params.Alerts, + mode: cfg.CoreRuleSetMode, + exemptPaths: cfg.CoreRuleSetExemptPaths, + scoreThreshold: cfg.CoreRuleSetScoreThreshold, + observe: cfg.Observe, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + } + + h.beforeLookup = []heuristic{ + headerLimits{}, + tagLists{lists: params.Lists}, + routeWatchTags{ + routeWatch: params.RouteWatch, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + }, + denyNets{netblocks: cfg.DenyNets}, + activeBan{banMaker: sharedBanMaker}, + } + h.afterLookup = []heuristic{ + countries{allowed: cfg.ExclusivelyAllowedCountries}, + h.blocklists, + dnsbl{ + dnsbl: params.DNSBL, + action: cfg.DNSBLAbuseIPDBAction, + hitNoter: hitNoter, + }, + abuseIPDB{ + abuseIPDB: params.AbuseIPDB, + limiter: params.Limiter, + keySet: cfg.AbuseIPDBKey != "", + minScore: cfg.AbuseIPDBMinScore, + action: cfg.DNSBLAbuseIPDBAction, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + hitNoter: hitNoter, + }, + tagDeny{tags: cfg.TagDeny}, + h.rateLimits, + } + h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet} + h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}} + h.afterAnswer = []heuristic{ + refusalLimit{ + limiter: params.Limiter, + perMinute: cfg.RefusalsLimitPerMinute, + banResponse: cfg.BanResponse, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + banMaker: sharedBanMaker, + }, + } + h.onReport = []heuristic{ + byteLimits{ + limiter: params.Limiter, + bytesCounted: cfg.BytesCounted, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + banMaker: sharedBanMaker, + }, + } + + return h } // handler is waf's handler. It holds what every request shares; what // belongs to one request is in a request. type handler struct { - config *config.WAF - requestLog io.Writer - processLog *slog.Logger - now func() time.Time - metrics *metrics.Metrics - limiter *ratelimit.Limiter - ledger *bans.Ledger - geojs *geojs.GeoJS - anomalies *anomaly.Counters - lookupFile *lookup.File - lists *reputation.Lists - dnsbl *reputation.DNSBL - abuseIPDB *reputation.AbuseIPDB - routeWatch *routewatch.Client - rules *rules.Files - coreRuleSet CoreRuleSetInspector - alerts *alerts.Queue + config *config.WAF + requestLog io.Writer + processLog *slog.Logger + now func() time.Time + metrics *metrics.Metrics + limiter *ratelimit.Limiter + ledger *bans.Ledger + geojs *geojs.GeoJS + anomalies *anomaly.Counters + lookupFile *lookup.File // waitForGeoJS is true when a setting needs each request's answer from // GeoJS before the request goes on. waitForGeoJS bool @@ -202,4 +301,42 @@ type handler struct { // awaitingReport holds the decision on each answered question by its id // until its report comes, and nil for a question still being handled. awaitingReport *lru.Cache[string, *decision] + + // The lists of heuristics, each in the order its heuristics run. A client + // in SWW_WAF_ALLOW_NETS skips them all. + + // beforeLookup are the heuristics that run before the client's AS number + // and country are looked up, so that a client they refuse is not looked + // up. header_limits comes first: it only measures and counts, and a + // request it refuses causes no other work. + beforeLookup []heuristic + // afterLookup are the heuristics that run once the lookup has been made. + afterLookup []heuristic + // besideLookup are the heuristics that only read the request. Their work + // runs beside the lookup and afterLookup, in findBesideLookup, which + // changes nothing of the request. Their check then applies what that work + // found, and runs only once waf takes it. + besideLookup []heuristic + // afterBesideLookup are the heuristics that run once afterLookup and + // besideLookup have both let the request through. tag_deny runs a second + // time, on the tag rule_files and core_rule_set can add. + afterBesideLookup []heuristic + // afterAnswer are the heuristics that run on a request once waf has + // answered it, before its decision line is written. They ban the client, + // and do not change what the request got: their verdict's action is not + // used. + afterAnswer []heuristic + // onReport are the heuristics that run on an allowed request when + // rproxy's report of it comes, before its report line is written. Like + // afterAnswer, they only ban. + onReport []heuristic + + // The heuristics of the lists that other steps of a request use too: + // refusedWhateverTheAnswer asks blocklists and rateLimits, and + // findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet. + blocklists blocklists + rateLimits rateLimits + trapPaths trapPaths + ruleFiles ruleFiles + coreRuleSet coreRuleSet } diff --git a/internal/waf/ratelimits.go b/internal/waf/ratelimits.go index 28ba812..7d92f5b 100644 --- a/internal/waf/ratelimits.go +++ b/internal/waf/ratelimits.go @@ -6,8 +6,10 @@ package waf import ( "context" + "net/netip" "time" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -18,28 +20,37 @@ import ( // refused, and bans the client. A request exempt from the rate limits is exempt // from the byte limits too. Its figures are the window, the count and the // limit. -type rateLimits struct{} +type rateLimits struct { + limiter *ratelimit.Limiter + exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS + exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS + banResponse int // SWW_WAF_BAN_RESPONSE + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + biasedThresholds biasedThresholds + banMaker banMaker +} func (rateLimits) name() string { return "rate_limits" } -func (rateLimits) check( +func (r rateLimits) check( _ context.Context, rq *request, now time.Time, ) *verdict { - rq.counted = rq.countedByRateLimits() + rq.counted = r.countedByRateLimits(rq) if !rq.counted { return nil } - rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages() + rq.rateLimitPercent, rq.bytesLimitPercent = + r.biasedThresholds.limitPercentages(rq) rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting = rq.rateLimitPercent.logged() rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting = rq.bytesLimitPercent.logged() - counts, hit, over := rq.h.limiter.Count( - clientGroup(rq.h.config, rq.client), + counts, hit, over := r.limiter.Count( + clientGroup(r.ipv6GroupPrefix, rq.client), now, rq.rateLimitPercent.percent, ) @@ -49,28 +60,29 @@ func (rateLimits) check( return nil } - rq.banForLimit(now, hit, rq.h.config.BanResponse) + r.banMaker.banForLimit(rq, now, hit, r.banResponse) return &verdict{ action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit), } } -// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the -// request at now, whatever its client's limit percentages: the rate limits -// count it, and one more request from its client takes it over a whole rate -// limit. It counts nothing. -func (rq *request) overAWholeRateLimit(now time.Time) bool { - return rq.countedByRateLimits() && - rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now) +// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq +// at now, whatever its client's limit percentages: the rate limits count it, +// and one more request from its client takes it over a whole rate limit. It +// counts nothing. +func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool { + return r.countedByRateLimits(rq) && + r.limiter.OverARateLimit( + clientGroup(r.ipv6GroupPrefix, rq.client), + now, + ) } -// countedByRateLimits reports whether the rate limits count the request: its -// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt -// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS. -func (rq *request) countedByRateLimits() bool { - cfg := rq.h.config - - return !isInside(rq.client, cfg.RateLimitExemptNets) && - !pathExempt(rq.in, cfg.RateLimitExemptPaths) +// countedByRateLimits reports whether the rate limits count rq: its client is +// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under +// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS. +func (r rateLimits) countedByRateLimits(rq *request) bool { + return !isInside(rq.client, r.exemptNets) && + !pathExempt(rq.in, r.exemptPaths) } diff --git a/internal/waf/refusallimit.go b/internal/waf/refusallimit.go index 1c8a4ee..f3227c8 100644 --- a/internal/waf/refusallimit.go +++ b/internal/waf/refusallimit.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -18,17 +19,22 @@ import ( // a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the // heuristics skip, is not counted, and nothing is while the limit is off. // Its figures are the window, the count and the limit. -type refusalLimit struct{} +type refusalLimit struct { + limiter *ratelimit.Limiter + perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE + banResponse int // SWW_WAF_BAN_RESPONSE + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + banMaker banMaker +} func (refusalLimit) name() string { return "refusal_limit" } -func (refusalLimit) check( +func (r refusalLimit) check( _ context.Context, rq *request, now time.Time, ) *verdict { - cfg := rq.h.config - if cfg.RefusalsLimitPerMinute == 0 { + if r.perMinute == 0 { return nil } @@ -36,10 +42,10 @@ func (refusalLimit) check( return nil } - hit, over := rq.h.limiter.CountRefusal( - clientGroup(rq.h.config, rq.client), + hit, over := r.limiter.CountRefusal( + clientGroup(r.ipv6GroupPrefix, rq.client), now, - cfg.RefusalsLimitPerMinute, + r.perMinute, ) if !over { return nil @@ -54,10 +60,10 @@ func (refusalLimit) check( status = http.StatusForbidden case requestlog.ActionTemporarilyBanned, requestlog.ActionPermanentlyBanned: - status = cfg.BanResponse + status = r.banResponse } - rq.banForLimit(now, hit, status) + r.banMaker.banForLimit(rq, now, hit, status) return &verdict{figures: limitFigures(hit)} } diff --git a/internal/waf/reputation.go b/internal/waf/reputation.go index ddee4b7..f898071 100644 --- a/internal/waf/reputation.go +++ b/internal/waf/reputation.go @@ -5,22 +5,35 @@ package waf import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/metrics" ) +// reputationHitNoter notes the reputation sources that list a request's +// client, for the reputation heuristics. +type reputationHitNoter struct { + metrics *metrics.Metrics + alerts *alerts.Queue + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX +} + // noteListed notes each of sources, the URLs of the blocklists or the -// DNSBL zones, their keys masked, that list the client, as noteHit does, +// DNSBL zones, their keys masked, that list rq's client, as noteHit does, // with reason. -func (rq *request) noteListed(sources []string, reason string) { +func (n reputationHitNoter) noteListed( + rq *request, sources []string, reason string, +) { for _, source := range sources { - rq.noteHit(bans.ReputationHit{Source: source}, reason) + n.noteHit(rq, bans.ReputationHit{Source: source}, reason) } } -// noteHit adds hit's source, which lists the client, to the log line's +// noteHit adds hit's source, which lists rq's client, to the log line's // reputation, and hit to the notes of a ban the request makes, counts the // source in the metrics, and raises a reputation_hit alert with reason, // whose detail gives hit's source and score. -func (rq *request) noteHit(hit bans.ReputationHit, reason string) { +func (n reputationHitNoter) noteHit( + rq *request, hit bans.ReputationHit, reason string, +) { detail := map[string]any{"source": hit.Source} if hit.Score != nil { detail["score"] = *hit.Score @@ -28,12 +41,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) { rq.line.Reputation = append(rq.line.Reputation, hit.Source) rq.reputation = append(rq.reputation, hit) - rq.h.metrics.ReputationHit(hit.Source) - rq.h.alerts.Raise(alerts.Alert{ + n.metrics.ReputationHit(hit.Source) + n.alerts.Raise(alerts.Alert{ Event: alerts.EventReputationHit, Host: rq.site, Client: rq.client, - Netblock: clientGroup(rq.h.config, rq.client), + Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, diff --git a/internal/waf/request.go b/internal/waf/request.go index bc900aa..6f4a203 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -122,8 +122,11 @@ func (h *handler) startRequest( RequestID: requestID(r, isQuestion), PeerIP: peer.String(), ForwardedFor: strings.Join(forwardedFor, ", "), - ClientGroup: clientGroup(h.config, client).String(), - ContentType: r.Header.Get("Content-Type"), + ClientGroup: clientGroup( + h.config.IPv6GroupPrefix, + client, + ).String(), + ContentType: r.Header.Get("Content-Type"), RequestHeaders: requestHeaders( r, h.config.LogRequestHeaders, @@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict { now := rq.h.now() - v := rq.runHeuristics(ctx, beforeLookup(), now) + v := rq.runHeuristics(ctx, rq.h.beforeLookup, now) if v != nil { return v } @@ -265,7 +268,7 @@ func (rq *request) runAfterLookup( // again: a request counted at the time it arrived could be counted // after a later one of its client's, which the rate limits would take // for the clock set back. - return rq.runHeuristics(ctx, afterLookup(), rq.h.now()) + return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now()) } // pathExempt reports whether r is exempt under prefixes, @@ -365,7 +368,7 @@ func (rq *request) addToHistory() { } rq.h.limiter.AddToHistory( - clientGroup(rq.h.config, rq.client), + clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), rq.h.now(), request, ) @@ -394,11 +397,11 @@ func (rq *request) countAnomalies() { rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ Host: rq.site, Client: rq.client, - ClientGroup: clientGroup(rq.h.config, rq.client), + ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), ASN: answer.ASN, ASName: answer.ASName, Country: answer.Country, - Bytes: rq.countedBytes(), + Bytes: rq.countedBytes(rq.h.config.BytesCounted), }) } @@ -425,7 +428,9 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) { return rq.lookupAnswer, true } - return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client)) + return rq.h.geojs.Kept( + clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), + ) } // responseWriter is waf's answer to a request. It notes the status and diff --git a/internal/waf/routewatchtags.go b/internal/waf/routewatchtags.go index d1a528c..f7383a1 100644 --- a/internal/waf/routewatchtags.go +++ b/internal/waf/routewatchtags.go @@ -8,6 +8,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/routewatch" ) // routeWatchTags adds to the request the tags asn= and prefix= @@ -17,22 +18,26 @@ import ( // SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and // asked for by the request's address, and a route without an origin AS gives only // the prefix= tag. -type routeWatchTags struct{} +type routeWatchTags struct { + // routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset. + routeWatch *routewatch.Client + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX +} func (routeWatchTags) name() string { return "routewatch_tags" } -func (routeWatchTags) check( +func (r routeWatchTags) check( ctx context.Context, rq *request, _ time.Time, ) *verdict { - if rq.h.routeWatch == nil { + if r.routeWatch == nil { return nil } - route, missing := rq.h.routeWatch.Route( + route, missing := r.routeWatch.Route( ctx, - clientGroup(rq.h.config, rq.client), + clientGroup(r.ipv6GroupPrefix, rq.client), rq.client, ) if missing != "" { diff --git a/internal/waf/rulefiles.go b/internal/waf/rulefiles.go index 22b211f..9240544 100644 --- a/internal/waf/rulefiles.go +++ b/internal/waf/rulefiles.go @@ -9,6 +9,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -22,11 +23,15 @@ import ( // mode raises the alert for the ban // it would have made. Either rule's match is noted as an offence, for the // client's history. Its figures are the rule's id and action. -type ruleFiles struct{} +type ruleFiles struct { + rules *rules.Files + metrics *metrics.Metrics + banMaker banMaker +} func (ruleFiles) name() string { return "rule_files" } -func (ruleFiles) check( +func (r ruleFiles) check( _ context.Context, rq *request, now time.Time, @@ -35,7 +40,7 @@ func (ruleFiles) check( for _, rule := range matched { rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID) - rq.h.metrics.RuleMatched(rule.ID, rule.Action) + r.metrics.RuleMatched(rule.ID, rule.Action) } if len(matched) == 0 { @@ -60,7 +65,8 @@ func (ruleFiles) check( rq.attack = true return &verdict{ - action: rq.banForAttack( + action: r.banMaker.banForAttack( + rq, now, bans.Notes{RuleID: last.ID, Target: last.Target}, ), diff --git a/internal/waf/tagdeny.go b/internal/waf/tagdeny.go index 91ad03a..a28322e 100644 --- a/internal/waf/tagdeny.go +++ b/internal/waf/tagdeny.go @@ -12,13 +12,19 @@ import ( // tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is // the first such tag, in sorted order. -type tagDeny struct{} +type tagDeny struct { + tags []string // SWW_WAF_TAG_DENY +} func (tagDeny) name() string { return "tag_deny" } -func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict { +func (t tagDeny) check( + _ context.Context, + rq *request, + _ time.Time, +) *verdict { for _, tag := range rq.line.Tags { - if slices.Contains(rq.h.config.TagDeny, tag) { + if slices.Contains(t.tags, tag) { return &verdict{ action: requestlog.ActionClientDenied, figures: map[string]any{"tag": tag}, diff --git a/internal/waf/taglists.go b/internal/waf/taglists.go index b4345fc..468c927 100644 --- a/internal/waf/taglists.go +++ b/internal/waf/taglists.go @@ -6,17 +6,25 @@ package waf import ( "context" "time" + + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that // holds the client, the tag being the list's name. It refuses nothing, and runs // first so that every refusal's log line carries these tags. -type tagLists struct{} +type tagLists struct { + lists *reputation.Lists +} func (tagLists) name() string { return "tag_lists" } -func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict { - for _, name := range rq.h.lists.TagsOf(rq.client) { +func (t tagLists) check( + _ context.Context, + rq *request, + _ time.Time, +) *verdict { + for _, name := range t.lists.TagsOf(rq.client) { rq.addTag(name) } diff --git a/internal/waf/trappaths.go b/internal/waf/trappaths.go index 0941500..39ad33b 100644 --- a/internal/waf/trappaths.go +++ b/internal/waf/trappaths.go @@ -16,11 +16,14 @@ import ( // it bans the client's netblock, or in observe mode raises the alert for the // ban it would have made. The ban's notes give the path as sent, and so does // its figure. -type trapPaths struct{} +type trapPaths struct { + paths []string // SWW_WAF_TRAP_PATHS + banMaker banMaker +} func (trapPaths) name() string { return "trap_paths" } -func (trapPaths) check( +func (t trapPaths) check( _ context.Context, rq *request, now time.Time, @@ -33,18 +36,22 @@ func (trapPaths) check( rq.attack = true return &verdict{ - action: rq.banForAttack(now, bans.Notes{TrapPath: path}), + action: t.banMaker.banForAttack( + rq, + now, + bans.Notes{TrapPath: path}, + ), figures: map[string]any{"path": path}, } } -// trapPath returns the request's path as sent, when its path as a path rule -// sees it, without the query, as sent or once percent-decoded, is in +// trapPath returns rq's path as sent, when its path as a path rule sees it, +// without the query, as sent or once percent-decoded, is in // SWW_WAF_TRAP_PATHS, and "" otherwise. -func (rq *request) trapPath() string { +func (t trapPaths) trapPath(rq *request) string { path := rules.Path(rq.in) - if !slices.Contains(rq.h.config.TrapPaths, path) && - !slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) { + if !slices.Contains(t.paths, path) && + !slices.Contains(t.paths, rules.DecodeOnce(path)) { return "" } diff --git a/internal/waf/waf.go b/internal/waf/waf.go index ec8d9ea..f0ca609 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -123,10 +123,10 @@ func New(params Params) *Server { GeoJS: h.geojs, Anomalies: h.anomalies, LookupFile: h.lookupFile, - Lists: h.lists, - DNSBL: h.dnsbl, - AbuseIPDB: h.abuseIPDB, - RouteWatch: h.routeWatch, + Lists: params.Lists, + DNSBL: params.DNSBL, + AbuseIPDB: params.AbuseIPDB, + RouteWatch: params.RouteWatch, Metrics: h.metrics, } } @@ -154,8 +154,10 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool { return false } - return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) || - rq.overAWholeRateLimit(now) + listedBy := rq.h.blocklists.lists.ListedBy(rq.client) + + return rq.h.blocklists.refusedByBlocklists(listedBy) || + rq.h.rateLimits.overAWholeRateLimit(rq, now) } // answerQuestion runs the checks on the visitor's request a question @@ -491,7 +493,7 @@ func (h *handler) finishWith( if rq.allowed { rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS - rq.runBanningHeuristics(ctx, onReport()) + rq.runBanningHeuristics(ctx, rq.h.onReport) rq.countAnomalies() rq.addToHistory() -- 2.54.0