diff --git a/internal/waf/abuseipdb.go b/internal/waf/abuseipdb.go index 0802ee2..ac43aa3 100644 --- a/internal/waf/abuseipdb.go +++ b/internal/waf/abuseipdb.go @@ -22,31 +22,42 @@ import ( // an offence, and the request does not wait for the answer. The score is then // used for each address of the client. Its figures are the score and // SWW_WAF_ABUSEIPDB_MIN_SCORE. -type abuseIPDB struct{} +type abuseIPDB struct { + abuseIPDB *reputation.AbuseIPDB + // limiter is the table of clients the rate limits count, whose histories + // say which clients are offenders. + limiter *ratelimit.Limiter + keySet bool // SWW_WAF_ABUSEIPDB_KEY is set + minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE + action string // SWW_WAF_DNSBL_ABUSEIPDB_ACTION + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + hitNoter reputationHitNoter +} func (abuseIPDB) name() string { return "abuseipdb" } -func (abuseIPDB) check( +func (a abuseIPDB) check( ctx context.Context, rq *request, _ time.Time, ) *verdict { - if rq.h.config.AbuseIPDBKey == "" { + if !a.keySet { return nil } - client := clientGroup(rq.h.config, rq.client) - held, _ := rq.h.limiter.Client(client) + client := clientGroup(a.ipv6GroupPrefix, rq.client) + held, _ := a.limiter.Client(client) offender := held.History.Offences != ratelimit.Offences{} - score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender) + score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender) if !hit { return nil } rq.abuseIPDBHit = true rq.addTag(config.AbuseIPDBTag) - rq.noteHit( + a.hitNoter.noteHit( + rq, bans.ReputationHit{ Source: reputation.AbuseIPDBSource, Score: &score, @@ -54,14 +65,14 @@ func (abuseIPDB) check( "scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE", ) - if rq.h.config.DNSBLAbuseIPDBAction != deny { + if a.action != deny { return nil } return &verdict{ action: requestlog.ActionClientDenied, figures: map[string]any{ - figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore, + figureScore: score, "min_score": a.minScore, }, } } diff --git a/internal/waf/abuseipdbblacklist.go b/internal/waf/abuseipdbblacklist.go index c4dfb84..63e49a2 100644 --- a/internal/waf/abuseipdbblacklist.go +++ b/internal/waf/abuseipdbblacklist.go @@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist( scores := make(map[netip.Prefix]int64, len(listed)) for _, client := range listed { - netblock := banNetblock(cfg, client.Addr) + netblock := banNetblock( + cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr, + ) scores[netblock] = max(scores[netblock], client.Score) } diff --git a/internal/waf/admin.go b/internal/waf/admin.go index 99ddec2..e98dbef 100644 --- a/internal/waf/admin.go +++ b/internal/waf/admin.go @@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) { return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone) } - return banNetblock(h.config, addr), nil + return banNetblock( + h.config.BanScopeV4Prefix, + h.config.IPv6GroupPrefix, + addr, + ), nil } // expiry returns when a ban made at now for duration ends: duration @@ -295,7 +299,9 @@ func (rq *request) showClient() { Bans: state.BanEntries(rq.h.ledger.Covering(addr)), } - client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr)) + client, seen := rq.h.limiter.Client( + clientGroup(rq.h.config.IPv6GroupPrefix, addr), + ) if seen { answer.Client = &client } diff --git a/internal/waf/bans.go b/internal/waf/bans.go index bf19d96..15a6e76 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -1,7 +1,7 @@ package waf -// The heuristic active_ban, and the code that makes bans, which several -// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's. +// The heuristic active_ban, and banMaker, which makes the bans of several +// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's. import ( "context" @@ -10,7 +10,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" - "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -25,18 +25,22 @@ func (rq *request) banResponse(action string) *refusal { // in, in force at now. It notes for the log line when that ban ends. A request // that makes the ban permanent, or in observe mode would have, raises the alert // for it. Its figures are the ban's netblock and when it ends. -type activeBan struct{} +type activeBan struct { + banMaker banMaker +} func (activeBan) name() string { return "active_ban" } -func (activeBan) check( +func (a activeBan) check( _ context.Context, rq *request, now time.Time, ) *verdict { - check := rq.h.ledger.Check - if rq.h.config.Observe { - check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is + ledger := a.banMaker.ledger + + check := ledger.Check + if a.banMaker.observe { + check = ledger.Find // the ban refuses nothing, and stays as it is } ban, banned, madePermanent := check(rq.client, now) @@ -46,7 +50,7 @@ func (activeBan) check( if madePermanent { ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is - rq.alertBan(ban) + a.banMaker.alertBan(rq, ban) } if !banned { @@ -81,11 +85,12 @@ func limitFigures(hit ratelimit.Hit) map[string]any { // countedBytes returns the request's bytes, once it has ended, as the // byte limits and the anomaly thresholds count them: the response's body -// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says. -func (rq *request) countedBytes() int64 { +// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED, +// says. +func (rq *request) countedBytes(bytesCounted string) int64 { response, request := rq.out.bytes, rq.requestBytes - switch rq.h.config.BytesCounted { + switch bytesCounted { case "response": return response case "request": @@ -95,7 +100,29 @@ func (rq *request) countedBytes() int64 { } } -// banForLimit bans the client's netblock at now for a broken limit, the +// banMaker makes the bans of the heuristics that ban a client, in the ban +// ledger, and raises the alert for each. In observe mode it makes no ban, and +// raises the alert for the ban it would have made. +type banMaker struct { + ledger *bans.Ledger + // limiter is the table of clients the rate limits count. A ban for a limit + // sets the client's counters back to zero, and a ban's notes give how many + // requests its netblock has sent. + limiter *ratelimit.Limiter + metrics *metrics.Metrics + alerts *alerts.Queue + // observe is true in observe mode (SWW_WAF_MODE). + observe bool + // banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a + // clear sign of attack give. + banResponse int + // banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix + // SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers. + banScopeV4Prefix int + ipv6GroupPrefix int +} + +// banForLimit bans the netblock of rq's client at now for a broken limit, the // one hit names, notes the offence for the log line and counts the hit in // the metrics. status is what the client was sent, or is sent: // SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for @@ -105,7 +132,8 @@ func (rq *request) countedBytes() int64 { // the client's counters back to zero. In observe mode it makes no ban and // sets nothing back, and raises the alert for the ban it would have made, // if that alert would be sent. -func (rq *request) banForLimit( +func (b banMaker) banForLimit( + rq *request, now time.Time, hit ratelimit.Hit, status int, @@ -120,11 +148,11 @@ func (rq *request) banForLimit( } rq.line.Offence = requestlog.OffenceLimit - rq.h.metrics.LimitHit(hit) - netblock := banNetblock(rq.h.config, rq.client) - if rq.h.config.Observe && - !rq.wouldAlertBan(netblock, now, bans.CauseLimit) { + b.metrics.LimitHit(hit) + + netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client) + if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseLimit) { return } @@ -138,7 +166,7 @@ func (rq *request) banForLimit( Count: hit.Count, Reputation: rq.reputation, Request: rq.noted(now, status), - Requests: rq.netblockRequests(netblock), + Requests: b.netblockRequests(netblock), } switch hit.Kind { @@ -148,35 +176,36 @@ func (rq *request) banForLimit( notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged() } - if rq.h.config.Observe { - ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes) + if b.observe { + ban, wouldBan := b.ledger.WouldBanForLimit(netblock, now, notes) if wouldBan { - rq.alertBan(ban) + b.alertBan(rq, ban) } return } - ban, made := rq.h.ledger.BanForLimit(netblock, now, notes) - rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client)) + ban, made := b.ledger.BanForLimit(netblock, now, notes) + b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client)) rq.line.BanExpires = banExpires(ban) if made { - rq.alertBan(ban) + b.alertBan(rq, ban) } } -// banForAttack bans the client's netblock at now for a clear sign of +// banForAttack bans the netblock of rq's client at now for a clear sign of // attack, which notes name: the ban rule that matched, or the trap path // asked for. It fills in the rest of the notes. In observe mode it makes // no ban, and raises the alert for the ban it would have made, if that // alert would be sent. It returns the action for that ban. -func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { - netblock := banNetblock(rq.h.config, rq.client) - if rq.h.config.Observe && - !rq.wouldAlertBan(netblock, now, bans.CauseAttack) { +func (b banMaker) banForAttack( + rq *request, now time.Time, notes bans.Notes, +) string { + netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client) + if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseAttack) { return banAction( - rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack), + b.ledger.WouldBePermanent(netblock, now, bans.CauseAttack), ) } @@ -184,23 +213,23 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { notes.ASName = rq.line.ASName notes.Country = rq.line.Country notes.Reputation = rq.reputation - notes.Request = rq.noted(now, rq.h.config.BanResponse) - notes.Requests = rq.netblockRequests(netblock) + notes.Request = rq.noted(now, b.banResponse) + notes.Requests = b.netblockRequests(netblock) - if rq.h.config.Observe { - ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes) + if b.observe { + ban, wouldBan := b.ledger.WouldBanForAttack(netblock, now, notes) if wouldBan { - rq.alertBan(ban) + b.alertBan(rq, ban) } return banAction(ban.Permanent()) } - ban, made := rq.h.ledger.BanForAttack(netblock, now, notes) + ban, made := b.ledger.BanForAttack(netblock, now, notes) rq.line.BanExpires = banExpires(ban) if made { - rq.alertBan(ban) + b.alertBan(rq, ban) } return banAction(ban.Permanent()) @@ -211,23 +240,22 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { // have made is worked out only then, at most once per // SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the // netblock's requests, which can mean going through every client. -func (rq *request) wouldAlertBan( +func (b banMaker) wouldAlertBan( netblock netip.Prefix, now time.Time, cause string, ) bool { event := alerts.EventBan - if rq.h.ledger.WouldBePermanent(netblock, now, cause) { + if b.ledger.WouldBePermanent(netblock, now, cause) { event = alerts.EventPermanentBan } - return rq.h.alerts.WouldSend(event, netblock) + return b.alerts.WouldSend(event, netblock) } -// alertBan raises the alert for ban, which the request made, or made -// permanent: permanent_ban for a permanent ban, ban for another. Its -// detail gives the ban's cause, when it ends, and its notes, and in -// observe mode, where ban is the ban that would have been made, or made -// permanent, mode, observe. -func (rq *request) alertBan(ban bans.Ban) { +// alertBan raises the alert for ban, which rq made, or made permanent: +// permanent_ban for a permanent ban, ban for another. Its detail gives the +// ban's cause, when it ends, and its notes, and in observe mode, where ban is +// the ban that would have been made, or made permanent, mode, observe. +func (b banMaker) alertBan(rq *request, ban bans.Ban) { event := alerts.EventBan if ban.Permanent() { event = alerts.EventPermanentBan @@ -236,11 +264,11 @@ func (rq *request) alertBan(ban bans.Ban) { detail := map[string]any{ "cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes, } - if rq.h.config.Observe { + if b.observe { detail["mode"] = "observe" } - rq.h.alerts.Raise(alerts.Alert{ + b.alerts.Raise(alerts.Alert{ Event: event, Host: rq.site, Client: rq.client, @@ -253,6 +281,13 @@ func (rq *request) alertBan(ban bans.Ban) { }) } +// netblockRequests is how many requests netblock has sent since it was +// first seen, the request being banned for included: the histories count it +// only once it has ended. +func (b banMaker) netblockRequests(netblock netip.Prefix) int64 { + return b.limiter.Requests(netblock) + 1 +} + // noted is the request, at now, with status, what the client was sent, or // in observe mode would have been, as the notes of the ban it makes keep // it. @@ -267,23 +302,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request { } } -// netblockRequests is how many requests netblock has sent since it was -// first seen, this one included: the histories count it only once it has -// ended. -func (rq *request) netblockRequests(netblock netip.Prefix) int64 { - return rq.h.limiter.Requests(netblock) + 1 -} - // banNetblock is the netblock a ban on client covers: its IPv4 address, -// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup -// counts it in. -func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix { +// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group +// clientGroup counts it in, of length ipv6GroupPrefix. +func banNetblock( + banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr, +) netip.Prefix { addr := client.Unmap() if addr.Is4() { - return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked() + return netip.PrefixFrom(addr, banScopeV4Prefix).Masked() } - return clientGroup(cfg, addr) + return clientGroup(ipv6GroupPrefix, addr) } // banExpires is when ban ends, as the log line gives it: a time, or diff --git a/internal/waf/besidelookup.go b/internal/waf/besidelookup.go index d1edf44..ffdc026 100644 --- a/internal/waf/besidelookup.go +++ b/internal/waf/besidelookup.go @@ -54,7 +54,7 @@ func (rq *request) runBesideLookup( ) *verdict { var running atomic.Pointer[string] - running.Store(new(trapPaths{}.name())) + running.Store(new(rq.h.trapPaths.name())) found := make(chan *besideLookupFindings, 1) @@ -92,7 +92,7 @@ func (rq *request) runBesideLookup( return v } - return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now()) + return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now()) } // findBesideLookup does the work of the heuristics of besideLookup on the @@ -105,39 +105,39 @@ func (rq *request) findBesideLookup( ) *besideLookupFindings { findings := &besideLookupFindings{} - findings.trapPath = rq.trapPath() + findings.trapPath = rq.h.trapPaths.trapPath(rq) if findings.trapPath != "" { findings.refuses = true return findings } - running.Store(new(ruleFiles{}.name())) + running.Store(new(rq.h.ruleFiles.name())) - findings.rules = rq.h.rules.Match(rq.in) + findings.rules = rq.h.ruleFiles.rules.Match(rq.in) if refusedByRule(findings.rules) { findings.refuses = true return findings } - running.Store(new(coreRuleSet{}.name())) + coreRuleSet := rq.h.coreRuleSet + running.Store(new(coreRuleSet.name())) - cfg := rq.h.config - if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff || - pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) { + if coreRuleSet.mode == config.CoreRuleSetModeOff || + pathExempt(rq.in, coreRuleSet.exemptPaths) { return findings } start := time.Now() // A question's body has been read into memory, so that reading it here // cannot fail. - findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect( + findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect( rq.in, rq.client, rq.in.Body, ) findings.inspectionTime = time.Since(start) findings.inspected = true - findings.refuses = rq.refusedByCoreRuleSet(findings.inspection) + findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection) return findings } @@ -151,7 +151,7 @@ func (rq *request) applyFindings( ) *verdict { rq.findings = findings - return rq.runHeuristics(ctx, besideLookup(), rq.h.now()) + return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now()) } // timedOut refuses the request for the heuristics of besideLookup not having diff --git a/internal/waf/biased.go b/internal/waf/biased.go index 3e729df..468bf20 100644 --- a/internal/waf/biased.go +++ b/internal/waf/biased.go @@ -2,6 +2,7 @@ package waf import ( "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // whole is the percentage of each limit a client gets when no biased @@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool { cfg.ASNLimitPercentURL != "" } +// biasedThresholds are the settings and the lists by which limitPercentages +// lowers a client's limits. +type biasedThresholds struct { + // lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names. + lists *reputation.Lists + tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT + asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT + countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT + unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT + // blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION + // and its percentage. + blocklistAction string + blocklistLimitPercent int64 + // dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are + // SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage. + dnsblAbuseIPDBAction string + dnsblAbuseIPDBLimitPercent int64 +} + // limitPercentages returns the client's limit percentages, for the rate // limits and for the byte limits, by its tags, its AS number and country // as looked up, each "" when unknown, and the blocklists, DNSBL zones and @@ -45,36 +65,37 @@ func biasedThresholdsSet(cfg *config.WAF) bool { // the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for // an AS number or a country they list. -func (rq *request) limitPercentages() (percentage, percentage) { - cfg := rq.h.config +func (b biasedThresholds) limitPercentages( + rq *request, +) (percentage, percentage) { asn, country := rq.line.ASN, rq.line.Country unknown := percentage{percent: whole} if country == "" && !rq.skippedLookupWait { unknown = percentage{ - cfg.UnknownCountryLimitPercent, + b.unknownCountryLimitPercent, "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", } } fetched := percentage{percent: whole} - if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed { + if percent, listed := b.lists.ASNLimitPercent(asn); listed { fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"} } blocklisted := percentage{percent: whole} - if rq.blocklisted && cfg.BlocklistAction == "limit" { + if rq.blocklisted && b.blocklistAction == "limit" { blocklisted = percentage{ - cfg.BlocklistLimitPercent, + b.blocklistLimitPercent, "SWW_WAF_BLOCKLIST_ACTION", } } dnsblAbuseIPDBListed := percentage{percent: whole} if (rq.dnsblListed || rq.abuseIPDBHit) && - cfg.DNSBLAbuseIPDBAction == "limit" { + b.dnsblAbuseIPDBAction == "limit" { dnsblAbuseIPDBListed = percentage{ - cfg.DNSBLAbuseIPDBLimitPercent, + b.dnsblAbuseIPDBLimitPercent, "SWW_WAF_DNSBL_ABUSEIPDB_ACTION", } } @@ -82,11 +103,11 @@ func (rq *request) limitPercentages() (percentage, percentage) { const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT" asnRequests := lowest( - given(cfg.TagLimitPercent, asnTag(asn), tagSetting), + given(b.tagLimitPercent, asnTag(asn), tagSetting), fetched, ) countryRequests := given( - cfg.TagLimitPercent, + b.tagLimitPercent, countryTag(country), tagSetting, ) @@ -97,23 +118,23 @@ func (rq *request) limitPercentages() (percentage, percentage) { if tag != asnTag(asn) && tag != countryTag(country) { otherTags = lowest( otherTags, - given(cfg.TagLimitPercent, tag, tagSetting), + given(b.tagLimitPercent, tag, tagSetting), ) } } asnBytes, countryBytes := asnRequests, countryRequests - if _, listed := cfg.ASNBytesLimitPercent[asn]; listed { + if _, listed := b.asnBytesLimitPercent[asn]; listed { asnBytes = given( - cfg.ASNBytesLimitPercent, + b.asnBytesLimitPercent, asn, "SWW_WAF_ASN_BYTES_LIMIT_PERCENT", ) } - if _, listed := cfg.CountryBytesLimitPercent[country]; listed { + if _, listed := b.countryBytesLimitPercent[country]; listed { countryBytes = given( - cfg.CountryBytesLimitPercent, + b.countryBytesLimitPercent, country, "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", ) diff --git a/internal/waf/blocklists.go b/internal/waf/blocklists.go index 9598cd9..db34013 100644 --- a/internal/waf/blocklists.go +++ b/internal/waf/blocklists.go @@ -7,6 +7,7 @@ import ( "context" "time" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -14,20 +15,24 @@ import ( // and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it // lowers the client's limits instead (see limitPercentages), and being log, it // does nothing more. Its figure is the blocklists that list the client. -type blocklists struct{} +type blocklists struct { + lists *reputation.Lists + action string // SWW_WAF_BLOCKLIST_ACTION + hitNoter reputationHitNoter +} func (blocklists) name() string { return "blocklists" } -func (blocklists) check( +func (b blocklists) check( _ context.Context, rq *request, _ time.Time, ) *verdict { - listedBy := rq.h.lists.ListedBy(rq.client) + listedBy := b.lists.ListedBy(rq.client) rq.blocklisted = len(listedBy) > 0 - rq.noteListed(listedBy, "listed by a blocklist") + b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist") - if !rq.refusedByBlocklists(listedBy) { + if !b.refusedByBlocklists(listedBy) { return nil } @@ -40,6 +45,6 @@ func (blocklists) check( // refusedByBlocklists reports whether the heuristic blocklists refuses a // client, given listedBy, the blocklists that list it: it does when one lists // it and SWW_WAF_BLOCKLIST_ACTION is deny. -func (rq *request) refusedByBlocklists(listedBy []string) bool { - return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny +func (b blocklists) refusedByBlocklists(listedBy []string) bool { + return len(listedBy) > 0 && b.action == deny } diff --git a/internal/waf/bytelimits.go b/internal/waf/bytelimits.go index c1f3c32..ff28c8e 100644 --- a/internal/waf/bytelimits.go +++ b/internal/waf/bytelimits.go @@ -6,6 +6,8 @@ package waf import ( "context" "time" + + "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) // byteLimits counts the request's bytes, as countedBytes gives them, for the @@ -16,11 +18,16 @@ import ( // the client over a byte limit, as its limit percentage for the byte limits // lowers it, break it, which bans the client; the response was passed on whole. // Its figures are the window, the count and the limit. -type byteLimits struct{} +type byteLimits struct { + limiter *ratelimit.Limiter + bytesCounted string // SWW_WAF_BYTES_COUNTED + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + banMaker banMaker +} func (byteLimits) name() string { return "byte_limits" } -func (byteLimits) check( +func (b byteLimits) check( _ context.Context, rq *request, now time.Time, @@ -29,10 +36,10 @@ func (byteLimits) check( return nil } - counts, hit, over := rq.h.limiter.CountBytes( - clientGroup(rq.h.config, rq.client), + counts, hit, over := b.limiter.CountBytes( + clientGroup(b.ipv6GroupPrefix, rq.client), now, - rq.countedBytes(), + rq.countedBytes(b.bytesCounted), rq.bytesLimitPercent.percent, ) rq.line.Counts.MinuteBytes = counts.MinuteBytes @@ -43,7 +50,7 @@ func (byteLimits) check( return nil } - rq.banForLimit(now, hit, rq.out.status) + b.banMaker.banForLimit(rq, now, hit, rq.out.status) return &verdict{figures: limitFigures(hit)} } diff --git a/internal/waf/client.go b/internal/waf/client.go index 8381f0a..549858a 100644 --- a/internal/waf/client.go +++ b/internal/waf/client.go @@ -5,8 +5,6 @@ import ( "net/http" "net/netip" "slices" - - "sneak.berlin/go/smallwebwaf/internal/config" ) // peerAddress is the address of the request's TCP peer. @@ -45,12 +43,13 @@ func scheme(r *http.Request, isQuestion bool) string { // clientGroup is the client a request is counted toward: its IPv4 // address, or its IPv6 group, the netblock its IPv6 address is in of the -// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser -// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. -func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix { +// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since +// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as +// IPv4. +func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix { addr = addr.Unmap() if addr.Is6() { - return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked() + return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked() } return netip.PrefixFrom(addr, addr.BitLen()) diff --git a/internal/waf/coreruleset.go b/internal/waf/coreruleset.go index 049bc6c..5697a89 100644 --- a/internal/waf/coreruleset.go +++ b/internal/waf/coreruleset.go @@ -10,6 +10,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/coreruleset" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -22,11 +23,22 @@ import ( // exploitsignature, raises the core_rule_set_match alert, and in block mode // refuses the request with 403, which is an offence its client's history // counts. Its figures are the score and the threshold. -type coreRuleSet struct{} +type coreRuleSet struct { + // inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is + // off. + inspector CoreRuleSetInspector + metrics *metrics.Metrics + alerts *alerts.Queue + mode string // SWW_WAF_CORE_RULE_SET_MODE + exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS + scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD + observe bool // SWW_WAF_MODE is observe + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX +} func (coreRuleSet) name() string { return "core_rule_set" } -func (coreRuleSet) check( +func (c coreRuleSet) check( _ context.Context, rq *request, _ time.Time, @@ -36,7 +48,6 @@ func (coreRuleSet) check( return nil } - cfg := rq.h.config result := findings.inspection rq.requestBytes = int64(len(findings.read)) @@ -47,18 +58,18 @@ func (coreRuleSet) check( rq.line.CoreRuleSetScore = &result.Score for _, id := range result.RuleIDs { - rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id) + c.metrics.CoreRuleSetMatched(c.mode, id) } - threshold := cfg.CoreRuleSetScoreThreshold + threshold := c.scoreThreshold if threshold == 0 || result.Score < threshold { return nil } rq.addTag(config.ExploitSignatureTag) - rq.alertCoreRuleSetMatch(result) + c.alertCoreRuleSetMatch(rq, result) - if !rq.refusedByCoreRuleSet(result) { + if !c.refusedByCoreRuleSet(result) { return nil } @@ -76,20 +87,21 @@ func (coreRuleSet) check( // refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core // Rule Set found result of: whether the score is at or over // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode. -func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool { - cfg := rq.h.config - threshold := cfg.CoreRuleSetScoreThreshold +func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool { + threshold := c.scoreThreshold - return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock && + return c.mode == config.CoreRuleSetModeBlock && threshold != 0 && result.Score >= threshold } -// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request, -// which the Core Rule Set scored at result, at or over +// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the +// Core Rule Set scored at result, at or over // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the // score, the method and the path with the query, and, for a request that is // not refused for it, the mode: detect, or observe in observe mode. -func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) { +func (c coreRuleSet) alertCoreRuleSetMatch( + rq *request, result coreruleset.Result, +) { detail := map[string]any{ "rule_ids": result.RuleIDs, "score": result.Score, @@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) { } switch { - case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect: + case c.mode == config.CoreRuleSetModeDetect: detail["mode"] = config.CoreRuleSetModeDetect - case rq.h.config.Observe: + case c.observe: detail["mode"] = "observe" } - rq.h.alerts.Raise(alerts.Alert{ + c.alerts.Raise(alerts.Alert{ Event: alerts.EventCoreRuleSetMatch, Host: rq.site, Client: rq.client, - Netblock: clientGroup(rq.h.config, rq.client), + Netblock: clientGroup(c.ipv6GroupPrefix, rq.client), ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, diff --git a/internal/waf/countries.go b/internal/waf/countries.go index 7bae8d5..8515726 100644 --- a/internal/waf/countries.go +++ b/internal/waf/countries.go @@ -14,16 +14,18 @@ import ( // does not hold the client's country as it was looked up. A client without a // country, or whose country cannot be found, is refused. Its figures are the // country and the setting that refused it. -type countries struct{} +type countries struct { + allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES +} func (countries) name() string { return "countries" } -func (countries) check( +func (c countries) check( _ context.Context, rq *request, _ time.Time, ) *verdict { - allowed := rq.h.config.ExclusivelyAllowedCountries + allowed := c.allowed country := rq.line.Country if len(allowed) == 0 || slices.Contains(allowed, country) { diff --git a/internal/waf/denynets.go b/internal/waf/denynets.go index 97bcb04..a33a843 100644 --- a/internal/waf/denynets.go +++ b/internal/waf/denynets.go @@ -4,6 +4,7 @@ package waf import ( "context" + "net/netip" "time" "sneak.berlin/go/smallwebwaf/internal/requestlog" @@ -11,12 +12,18 @@ import ( // denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of // the setting the client is in. -type denyNets struct{} +type denyNets struct { + netblocks []netip.Prefix // SWW_WAF_DENY_NETS +} func (denyNets) name() string { return "deny_nets" } -func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict { - for _, netblock := range rq.h.config.DenyNets { +func (d denyNets) check( + _ context.Context, + rq *request, + _ time.Time, +) *verdict { + for _, netblock := range d.netblocks { if netblock.Contains(rq.client) { return &verdict{ action: requestlog.ActionClientDenied, diff --git a/internal/waf/denynets_internal_test.go b/internal/waf/denynets_internal_test.go new file mode 100644 index 0000000..51f5737 --- /dev/null +++ b/internal/waf/denynets_internal_test.go @@ -0,0 +1,47 @@ +package waf + +import ( + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) { + t.Parallel() + + heuristic := denyNets{ + netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}, + } + + // The request has no handler: the heuristic reads only what it holds. + inside := &request{ + decision: decision{client: netip.MustParseAddr("198.51.100.7")}, + } + + v := heuristic.check(t.Context(), inside, time.Time{}) + if v == nil { + t.Fatal("a client inside the netblock was let through") + } + + if v.action != requestlog.ActionClientDenied { + t.Errorf( + "action = %q, want %q", + v.action, + requestlog.ActionClientDenied, + ) + } + + if got := v.figures[figureNetblock]; got != "198.51.100.0/24" { + t.Errorf("netblock figure = %v, want 198.51.100.0/24", got) + } + + outside := &request{ + decision: decision{client: netip.MustParseAddr("203.0.113.7")}, + } + + if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil { + t.Errorf("a client outside the netblock was refused: %+v", v) + } +} diff --git a/internal/waf/dnsbl.go b/internal/waf/dnsbl.go index b0bb4b4..dc97694 100644 --- a/internal/waf/dnsbl.go +++ b/internal/waf/dnsbl.go @@ -9,6 +9,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -19,20 +20,28 @@ import ( // being log, it does nothing more. A zone without a verdict on the client is // asked about it in the background, and the request does not wait for the // answer. Its figure is the zones that list the client, their keys masked. -type dnsbl struct{} +type dnsbl struct { + dnsbl *reputation.DNSBL + action string // SWW_WAF_DNSBL_ABUSEIPDB_ACTION + hitNoter reputationHitNoter +} func (dnsbl) name() string { return "dnsbl" } -func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict { - listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client) +func (d dnsbl) check( + ctx context.Context, + rq *request, + _ time.Time, +) *verdict { + listedBy := d.dnsbl.ListedBy(ctx, rq.client) rq.dnsblListed = len(listedBy) > 0 - rq.noteListed(listedBy, "listed by a DNSBL zone") + d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone") for _, zone := range listedBy { rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone)) } - if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny { + if !rq.dnsblListed || d.action != deny { return nil } diff --git a/internal/waf/heuristics.go b/internal/waf/heuristics.go index 9fb7e59..1161b1b 100644 --- a/internal/waf/heuristics.go +++ b/internal/waf/heuristics.go @@ -5,9 +5,10 @@ import ( "time" ) -// To add a heuristic, write one file with a type that implements heuristic, -// and add one line for it to one of the lists below. Its settings go in -// config.WAF, and the comment at the top of its file names them. +// To add a heuristic, write one file with a struct that implements heuristic +// and holds the settings and parts it reads. Add one line for it to one of the +// lists that newHandler builds, which hands it those settings and parts. Its +// settings go in config.WAF, and the comment at the top of its file names them. // heuristic is one of the checks that decide what waf does with a // request. @@ -36,78 +37,10 @@ const ( figureNetblock = "netblock" ) -// The lists of heuristics, each in the order its heuristics run. A client in -// SWW_WAF_ALLOW_NETS skips them all. - -// beforeLookup are the heuristics that run before the client's AS number and -// country are looked up, so that a client they refuse is not looked up. -// header_limits comes first: it only measures and counts, and a request it -// refuses causes no other work. -func beforeLookup() []heuristic { - return []heuristic{ - headerLimits{}, - tagLists{}, - routeWatchTags{}, - denyNets{}, - activeBan{}, - } -} - -// afterLookup are the heuristics that run once the lookup has been made. -func afterLookup() []heuristic { - return []heuristic{ - countries{}, - blocklists{}, - dnsbl{}, - abuseIPDB{}, - tagDeny{}, - rateLimits{}, - } -} - -// besideLookup are the heuristics that only read the request. Their work runs -// beside the lookup and afterLookup, in findBesideLookup, which changes nothing -// of the request. Their check then applies what that work found, and runs only -// once waf takes it. -func besideLookup() []heuristic { - return []heuristic{ - trapPaths{}, - ruleFiles{}, - coreRuleSet{}, - } -} - -// afterBesideLookup are the heuristics that run once afterLookup and -// besideLookup have both let the request through. tag_deny runs a second time, -// on the tag rule_files and core_rule_set can add. -func afterBesideLookup() []heuristic { - return []heuristic{ - tagDeny{}, - } -} - -// afterAnswer are the heuristics that run on a request once waf has -// answered it, before its decision line is written. They ban the client, and do -// not change what the request got: their verdict's action is not used. -func afterAnswer() []heuristic { - return []heuristic{ - refusalLimit{}, - } -} - -// onReport are the heuristics that run on an allowed request when rproxy's -// report of it comes, before its report line is written. Like afterAnswer, they -// only ban. -func onReport() []heuristic { - return []heuristic{ - byteLimits{}, - } -} - // runAfterAnswer runs the heuristics of afterAnswer on the request, at the time // it is called. func (rq *request) runAfterAnswer(ctx context.Context) { - rq.runBanningHeuristics(ctx, afterAnswer()) + rq.runBanningHeuristics(ctx, rq.h.afterAnswer) } // runHeuristics runs list, one of those checkClient runs, on the request at now, diff --git a/internal/waf/lookup.go b/internal/waf/lookup.go index a6fc5e4..626bf4f 100644 --- a/internal/waf/lookup.go +++ b/internal/waf/lookup.go @@ -55,7 +55,7 @@ func (rq *request) startLookUp( return result } - client := clientGroup(rq.h.config, rq.client) + client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client) if rq.h.config.LookupSource == "file" { result <- lookupResult{ @@ -142,8 +142,10 @@ func AddLookup( ) { limiter.AddLookup(answer.Client, answer.Answered, answer.ASN, answer.ASName, answer.Country) - ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()), - answer.ASN, answer.ASName, answer.Country) + netblock := banNetblock( + cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(), + ) + ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country) } // GeoJSFailed logs a failure of GeoJS to processLog, raises a diff --git a/internal/waf/proxy.go b/internal/waf/proxy.go index 966fe8b..0a7954e 100644 --- a/internal/waf/proxy.go +++ b/internal/waf/proxy.go @@ -134,15 +134,19 @@ type Server struct { } // newHandler returns the handler with what every request shares, the parts -// in params among it. +// in params among it. It builds the heuristics, each with the settings and +// parts it reads, and the lists they run in. +// +//nolint:funlen // a few lines for each heuristic, a list that grows with them func newHandler(params Params) *handler { awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) if err != nil { panic(err) // New fails only for a size below one } - return &handler{ - config: params.Config, + cfg := params.Config + h := &handler{ + config: cfg, requestLog: params.RequestLog, processLog: params.ProcessLog, now: params.Now, @@ -153,25 +157,127 @@ func newHandler(params Params) *handler { geojs: params.GeoJS, anomalies: params.Anomalies, lookupFile: params.LookupFile, - lists: params.Lists, - dnsbl: params.DNSBL, - abuseIPDB: params.AbuseIPDB, - routeWatch: params.RouteWatch, - rules: params.Rules, - coreRuleSet: params.CoreRuleSet, - alerts: params.Alerts, // The country list, the headers, the tag settings that name an asn= // or country= tag and the biased thresholds act on the answer before // the request goes on. - waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 || - params.Config.AddLookupHeaders || - params.Config.TagsNeedLookup() || - biasedThresholdsSet(params.Config), + waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 || + cfg.AddLookupHeaders || + cfg.TagsNeedLookup() || + biasedThresholdsSet(cfg), besideLookupLimit: cmp.Or( params.BesideLookupTimeLimit, besideLookupTimeLimit, ), } + + sharedBanMaker := banMaker{ + ledger: params.Ledger, + limiter: params.Limiter, + metrics: params.Metrics, + alerts: params.Alerts, + observe: cfg.Observe, + banResponse: cfg.BanResponse, + banScopeV4Prefix: cfg.BanScopeV4Prefix, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + } + hitNoter := reputationHitNoter{ + metrics: params.Metrics, + alerts: params.Alerts, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + } + + h.blocklists = blocklists{ + lists: params.Lists, + action: cfg.BlocklistAction, + hitNoter: hitNoter, + } + h.rateLimits = rateLimits{ + limiter: params.Limiter, + exemptNets: cfg.RateLimitExemptNets, + exemptPaths: cfg.RateLimitExemptPaths, + banResponse: cfg.BanResponse, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + biasedThresholds: biasedThresholds{ + lists: params.Lists, + tagLimitPercent: cfg.TagLimitPercent, + asnBytesLimitPercent: cfg.ASNBytesLimitPercent, + countryBytesLimitPercent: cfg.CountryBytesLimitPercent, + unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent, + blocklistAction: cfg.BlocklistAction, + blocklistLimitPercent: cfg.BlocklistLimitPercent, + dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction, + dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent, + }, + banMaker: sharedBanMaker, + } + h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker} + h.ruleFiles = ruleFiles{ + rules: params.Rules, + metrics: params.Metrics, + banMaker: sharedBanMaker, + } + h.coreRuleSet = coreRuleSet{ + inspector: params.CoreRuleSet, + metrics: params.Metrics, + alerts: params.Alerts, + mode: cfg.CoreRuleSetMode, + exemptPaths: cfg.CoreRuleSetExemptPaths, + scoreThreshold: cfg.CoreRuleSetScoreThreshold, + observe: cfg.Observe, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + } + + h.beforeLookup = []heuristic{ + headerLimits{}, + tagLists{lists: params.Lists}, + routeWatchTags{ + routeWatch: params.RouteWatch, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + }, + denyNets{netblocks: cfg.DenyNets}, + activeBan{banMaker: sharedBanMaker}, + } + h.afterLookup = []heuristic{ + countries{allowed: cfg.ExclusivelyAllowedCountries}, + h.blocklists, + dnsbl{ + dnsbl: params.DNSBL, + action: cfg.DNSBLAbuseIPDBAction, + hitNoter: hitNoter, + }, + abuseIPDB{ + abuseIPDB: params.AbuseIPDB, + limiter: params.Limiter, + keySet: cfg.AbuseIPDBKey != "", + minScore: cfg.AbuseIPDBMinScore, + action: cfg.DNSBLAbuseIPDBAction, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + hitNoter: hitNoter, + }, + tagDeny{tags: cfg.TagDeny}, + h.rateLimits, + } + h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet} + h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}} + h.afterAnswer = []heuristic{ + refusalLimit{ + limiter: params.Limiter, + perMinute: cfg.RefusalsLimitPerMinute, + banResponse: cfg.BanResponse, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + banMaker: sharedBanMaker, + }, + } + h.onReport = []heuristic{ + byteLimits{ + limiter: params.Limiter, + bytesCounted: cfg.BytesCounted, + ipv6GroupPrefix: cfg.IPv6GroupPrefix, + banMaker: sharedBanMaker, + }, + } + + return h } // NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL, @@ -211,23 +317,16 @@ func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) { // handler is waf's handler. It holds what every request shares; what // belongs to one request is in a request. type handler struct { - config *config.WAF - requestLog io.Writer - processLog *slog.Logger - now func() time.Time - metrics *metrics.Metrics - limiter *ratelimit.Limiter - ledger *bans.Ledger - geojs *geojs.GeoJS - anomalies *anomaly.Counters - lookupFile *lookup.File - lists *reputation.Lists - dnsbl *reputation.DNSBL - abuseIPDB *reputation.AbuseIPDB - routeWatch *routewatch.Client - rules *rules.Files - coreRuleSet CoreRuleSetInspector - alerts *alerts.Queue + config *config.WAF + requestLog io.Writer + processLog *slog.Logger + now func() time.Time + metrics *metrics.Metrics + limiter *ratelimit.Limiter + ledger *bans.Ledger + geojs *geojs.GeoJS + anomalies *anomaly.Counters + lookupFile *lookup.File // waitForGeoJS is true when a setting needs each request's answer from // GeoJS before the request goes on. waitForGeoJS bool @@ -237,4 +336,42 @@ type handler struct { // awaitingReport holds the decision on each answered question by its id // until its report comes, and nil for a question still being handled. awaitingReport *lru.Cache[string, *decision] + + // The lists of heuristics, each in the order its heuristics run. A client + // in SWW_WAF_ALLOW_NETS skips them all. + + // beforeLookup are the heuristics that run before the client's AS number + // and country are looked up, so that a client they refuse is not looked + // up. header_limits comes first: it only measures and counts, and a + // request it refuses causes no other work. + beforeLookup []heuristic + // afterLookup are the heuristics that run once the lookup has been made. + afterLookup []heuristic + // besideLookup are the heuristics that only read the request. Their work + // runs beside the lookup and afterLookup, in findBesideLookup, which + // changes nothing of the request. Their check then applies what that work + // found, and runs only once waf takes it. + besideLookup []heuristic + // afterBesideLookup are the heuristics that run once afterLookup and + // besideLookup have both let the request through. tag_deny runs a second + // time, on the tag rule_files and core_rule_set can add. + afterBesideLookup []heuristic + // afterAnswer are the heuristics that run on a request once waf has + // answered it, before its decision line is written. They ban the client, + // and do not change what the request got: their verdict's action is not + // used. + afterAnswer []heuristic + // onReport are the heuristics that run on an allowed request when + // rproxy's report of it comes, before its report line is written. Like + // afterAnswer, they only ban. + onReport []heuristic + + // The heuristics of the lists that other steps of a request use too: + // refusedWhateverTheAnswer asks blocklists and rateLimits, and + // findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet. + blocklists blocklists + rateLimits rateLimits + trapPaths trapPaths + ruleFiles ruleFiles + coreRuleSet coreRuleSet } diff --git a/internal/waf/ratelimits.go b/internal/waf/ratelimits.go index 28ba812..7d92f5b 100644 --- a/internal/waf/ratelimits.go +++ b/internal/waf/ratelimits.go @@ -6,8 +6,10 @@ package waf import ( "context" + "net/netip" "time" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -18,28 +20,37 @@ import ( // refused, and bans the client. A request exempt from the rate limits is exempt // from the byte limits too. Its figures are the window, the count and the // limit. -type rateLimits struct{} +type rateLimits struct { + limiter *ratelimit.Limiter + exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS + exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS + banResponse int // SWW_WAF_BAN_RESPONSE + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + biasedThresholds biasedThresholds + banMaker banMaker +} func (rateLimits) name() string { return "rate_limits" } -func (rateLimits) check( +func (r rateLimits) check( _ context.Context, rq *request, now time.Time, ) *verdict { - rq.counted = rq.countedByRateLimits() + rq.counted = r.countedByRateLimits(rq) if !rq.counted { return nil } - rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages() + rq.rateLimitPercent, rq.bytesLimitPercent = + r.biasedThresholds.limitPercentages(rq) rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting = rq.rateLimitPercent.logged() rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting = rq.bytesLimitPercent.logged() - counts, hit, over := rq.h.limiter.Count( - clientGroup(rq.h.config, rq.client), + counts, hit, over := r.limiter.Count( + clientGroup(r.ipv6GroupPrefix, rq.client), now, rq.rateLimitPercent.percent, ) @@ -49,28 +60,29 @@ func (rateLimits) check( return nil } - rq.banForLimit(now, hit, rq.h.config.BanResponse) + r.banMaker.banForLimit(rq, now, hit, r.banResponse) return &verdict{ action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit), } } -// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the -// request at now, whatever its client's limit percentages: the rate limits -// count it, and one more request from its client takes it over a whole rate -// limit. It counts nothing. -func (rq *request) overAWholeRateLimit(now time.Time) bool { - return rq.countedByRateLimits() && - rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now) +// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq +// at now, whatever its client's limit percentages: the rate limits count it, +// and one more request from its client takes it over a whole rate limit. It +// counts nothing. +func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool { + return r.countedByRateLimits(rq) && + r.limiter.OverARateLimit( + clientGroup(r.ipv6GroupPrefix, rq.client), + now, + ) } -// countedByRateLimits reports whether the rate limits count the request: its -// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt -// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS. -func (rq *request) countedByRateLimits() bool { - cfg := rq.h.config - - return !isInside(rq.client, cfg.RateLimitExemptNets) && - !pathExempt(rq.in, cfg.RateLimitExemptPaths) +// countedByRateLimits reports whether the rate limits count rq: its client is +// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under +// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS. +func (r rateLimits) countedByRateLimits(rq *request) bool { + return !isInside(rq.client, r.exemptNets) && + !pathExempt(rq.in, r.exemptPaths) } diff --git a/internal/waf/refusallimit.go b/internal/waf/refusallimit.go index 1c8a4ee..f3227c8 100644 --- a/internal/waf/refusallimit.go +++ b/internal/waf/refusallimit.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -18,17 +19,22 @@ import ( // a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the // heuristics skip, is not counted, and nothing is while the limit is off. // Its figures are the window, the count and the limit. -type refusalLimit struct{} +type refusalLimit struct { + limiter *ratelimit.Limiter + perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE + banResponse int // SWW_WAF_BAN_RESPONSE + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX + banMaker banMaker +} func (refusalLimit) name() string { return "refusal_limit" } -func (refusalLimit) check( +func (r refusalLimit) check( _ context.Context, rq *request, now time.Time, ) *verdict { - cfg := rq.h.config - if cfg.RefusalsLimitPerMinute == 0 { + if r.perMinute == 0 { return nil } @@ -36,10 +42,10 @@ func (refusalLimit) check( return nil } - hit, over := rq.h.limiter.CountRefusal( - clientGroup(rq.h.config, rq.client), + hit, over := r.limiter.CountRefusal( + clientGroup(r.ipv6GroupPrefix, rq.client), now, - cfg.RefusalsLimitPerMinute, + r.perMinute, ) if !over { return nil @@ -54,10 +60,10 @@ func (refusalLimit) check( status = http.StatusForbidden case requestlog.ActionTemporarilyBanned, requestlog.ActionPermanentlyBanned: - status = cfg.BanResponse + status = r.banResponse } - rq.banForLimit(now, hit, status) + r.banMaker.banForLimit(rq, now, hit, status) return &verdict{figures: limitFigures(hit)} } diff --git a/internal/waf/reputation.go b/internal/waf/reputation.go index b760666..f4b56e5 100644 --- a/internal/waf/reputation.go +++ b/internal/waf/reputation.go @@ -5,26 +5,39 @@ package waf import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/metrics" ) // deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that // refuses the requests of a client a source lists. const deny = "deny" +// reputationHitNoter notes the reputation sources that list a request's +// client, for the reputation heuristics. +type reputationHitNoter struct { + metrics *metrics.Metrics + alerts *alerts.Queue + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX +} + // noteListed notes each of sources, the URLs of the blocklists or the -// DNSBL zones, their keys masked, that list the client, as noteHit does, +// DNSBL zones, their keys masked, that list rq's client, as noteHit does, // with reason. -func (rq *request) noteListed(sources []string, reason string) { +func (n reputationHitNoter) noteListed( + rq *request, sources []string, reason string, +) { for _, source := range sources { - rq.noteHit(bans.ReputationHit{Source: source}, reason) + n.noteHit(rq, bans.ReputationHit{Source: source}, reason) } } -// noteHit adds hit's source, which lists the client, to the log line's +// noteHit adds hit's source, which lists rq's client, to the log line's // reputation, and hit to the notes of a ban the request makes, counts the // source in the metrics, and raises a reputation_hit alert with reason, // whose detail gives hit's source and score. -func (rq *request) noteHit(hit bans.ReputationHit, reason string) { +func (n reputationHitNoter) noteHit( + rq *request, hit bans.ReputationHit, reason string, +) { detail := map[string]any{"source": hit.Source} if hit.Score != nil { detail["score"] = *hit.Score @@ -32,12 +45,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) { rq.line.Reputation = append(rq.line.Reputation, hit.Source) rq.reputation = append(rq.reputation, hit) - rq.h.metrics.ReputationHit(hit.Source) - rq.h.alerts.Raise(alerts.Alert{ + n.metrics.ReputationHit(hit.Source) + n.alerts.Raise(alerts.Alert{ Event: alerts.EventReputationHit, Host: rq.site, Client: rq.client, - Netblock: clientGroup(rq.h.config, rq.client), + Netblock: clientGroup(n.ipv6GroupPrefix, rq.client), ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, diff --git a/internal/waf/request.go b/internal/waf/request.go index 9c6dbc8..a6d68ba 100644 --- a/internal/waf/request.go +++ b/internal/waf/request.go @@ -122,8 +122,11 @@ func (h *handler) startRequest( RequestID: requestID(r, isQuestion), PeerIP: peer.String(), ForwardedFor: strings.Join(forwardedFor, ", "), - ClientGroup: clientGroup(h.config, client).String(), - ContentType: r.Header.Get("Content-Type"), + ClientGroup: clientGroup( + h.config.IPv6GroupPrefix, + client, + ).String(), + ContentType: r.Header.Get("Content-Type"), RequestHeaders: requestHeaders( r, h.config.LogRequestHeaders, @@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict { now := rq.h.now() - v := rq.runHeuristics(ctx, beforeLookup(), now) + v := rq.runHeuristics(ctx, rq.h.beforeLookup, now) if v != nil { return v } @@ -265,7 +268,7 @@ func (rq *request) runAfterLookup( // again: a request counted at the time it arrived could be counted // after a later one of its client's, which the rate limits would take // for the clock set back. - return rq.runHeuristics(ctx, afterLookup(), rq.h.now()) + return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now()) } // pathExempt reports whether r is exempt under prefixes, @@ -365,7 +368,7 @@ func (rq *request) addToHistory() { } rq.h.limiter.AddToHistory( - clientGroup(rq.h.config, rq.client), + clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), rq.h.now(), request, ) @@ -394,11 +397,11 @@ func (rq *request) countAnomalies() { rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ Host: rq.site, Client: rq.client, - ClientGroup: clientGroup(rq.h.config, rq.client), + ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), ASN: answer.ASN, ASName: answer.ASName, Country: answer.Country, - Bytes: rq.countedBytes(), + Bytes: rq.countedBytes(rq.h.config.BytesCounted), }) } @@ -425,7 +428,9 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) { return rq.lookupAnswer, true } - return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client)) + return rq.h.geojs.Kept( + clientGroup(rq.h.config.IPv6GroupPrefix, rq.client), + ) } // responseWriter is waf's answer to a request. It notes the status and diff --git a/internal/waf/routewatchtags.go b/internal/waf/routewatchtags.go index d1a528c..f7383a1 100644 --- a/internal/waf/routewatchtags.go +++ b/internal/waf/routewatchtags.go @@ -8,6 +8,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/routewatch" ) // routeWatchTags adds to the request the tags asn= and prefix= @@ -17,22 +18,26 @@ import ( // SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and // asked for by the request's address, and a route without an origin AS gives only // the prefix= tag. -type routeWatchTags struct{} +type routeWatchTags struct { + // routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset. + routeWatch *routewatch.Client + ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX +} func (routeWatchTags) name() string { return "routewatch_tags" } -func (routeWatchTags) check( +func (r routeWatchTags) check( ctx context.Context, rq *request, _ time.Time, ) *verdict { - if rq.h.routeWatch == nil { + if r.routeWatch == nil { return nil } - route, missing := rq.h.routeWatch.Route( + route, missing := r.routeWatch.Route( ctx, - clientGroup(rq.h.config, rq.client), + clientGroup(r.ipv6GroupPrefix, rq.client), rq.client, ) if missing != "" { diff --git a/internal/waf/rulefiles.go b/internal/waf/rulefiles.go index 22b211f..9240544 100644 --- a/internal/waf/rulefiles.go +++ b/internal/waf/rulefiles.go @@ -9,6 +9,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -22,11 +23,15 @@ import ( // mode raises the alert for the ban // it would have made. Either rule's match is noted as an offence, for the // client's history. Its figures are the rule's id and action. -type ruleFiles struct{} +type ruleFiles struct { + rules *rules.Files + metrics *metrics.Metrics + banMaker banMaker +} func (ruleFiles) name() string { return "rule_files" } -func (ruleFiles) check( +func (r ruleFiles) check( _ context.Context, rq *request, now time.Time, @@ -35,7 +40,7 @@ func (ruleFiles) check( for _, rule := range matched { rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID) - rq.h.metrics.RuleMatched(rule.ID, rule.Action) + r.metrics.RuleMatched(rule.ID, rule.Action) } if len(matched) == 0 { @@ -60,7 +65,8 @@ func (ruleFiles) check( rq.attack = true return &verdict{ - action: rq.banForAttack( + action: r.banMaker.banForAttack( + rq, now, bans.Notes{RuleID: last.ID, Target: last.Target}, ), diff --git a/internal/waf/tagdeny.go b/internal/waf/tagdeny.go index 91ad03a..a28322e 100644 --- a/internal/waf/tagdeny.go +++ b/internal/waf/tagdeny.go @@ -12,13 +12,19 @@ import ( // tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is // the first such tag, in sorted order. -type tagDeny struct{} +type tagDeny struct { + tags []string // SWW_WAF_TAG_DENY +} func (tagDeny) name() string { return "tag_deny" } -func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict { +func (t tagDeny) check( + _ context.Context, + rq *request, + _ time.Time, +) *verdict { for _, tag := range rq.line.Tags { - if slices.Contains(rq.h.config.TagDeny, tag) { + if slices.Contains(t.tags, tag) { return &verdict{ action: requestlog.ActionClientDenied, figures: map[string]any{"tag": tag}, diff --git a/internal/waf/taglists.go b/internal/waf/taglists.go index b4345fc..468c927 100644 --- a/internal/waf/taglists.go +++ b/internal/waf/taglists.go @@ -6,17 +6,25 @@ package waf import ( "context" "time" + + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that // holds the client, the tag being the list's name. It refuses nothing, and runs // first so that every refusal's log line carries these tags. -type tagLists struct{} +type tagLists struct { + lists *reputation.Lists +} func (tagLists) name() string { return "tag_lists" } -func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict { - for _, name := range rq.h.lists.TagsOf(rq.client) { +func (t tagLists) check( + _ context.Context, + rq *request, + _ time.Time, +) *verdict { + for _, name := range t.lists.TagsOf(rq.client) { rq.addTag(name) } diff --git a/internal/waf/trappaths.go b/internal/waf/trappaths.go index 0941500..39ad33b 100644 --- a/internal/waf/trappaths.go +++ b/internal/waf/trappaths.go @@ -16,11 +16,14 @@ import ( // it bans the client's netblock, or in observe mode raises the alert for the // ban it would have made. The ban's notes give the path as sent, and so does // its figure. -type trapPaths struct{} +type trapPaths struct { + paths []string // SWW_WAF_TRAP_PATHS + banMaker banMaker +} func (trapPaths) name() string { return "trap_paths" } -func (trapPaths) check( +func (t trapPaths) check( _ context.Context, rq *request, now time.Time, @@ -33,18 +36,22 @@ func (trapPaths) check( rq.attack = true return &verdict{ - action: rq.banForAttack(now, bans.Notes{TrapPath: path}), + action: t.banMaker.banForAttack( + rq, + now, + bans.Notes{TrapPath: path}, + ), figures: map[string]any{"path": path}, } } -// trapPath returns the request's path as sent, when its path as a path rule -// sees it, without the query, as sent or once percent-decoded, is in +// trapPath returns rq's path as sent, when its path as a path rule sees it, +// without the query, as sent or once percent-decoded, is in // SWW_WAF_TRAP_PATHS, and "" otherwise. -func (rq *request) trapPath() string { +func (t trapPaths) trapPath(rq *request) string { path := rules.Path(rq.in) - if !slices.Contains(rq.h.config.TrapPaths, path) && - !slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) { + if !slices.Contains(t.paths, path) && + !slices.Contains(t.paths, rules.DecodeOnce(path)) { return "" } diff --git a/internal/waf/waf.go b/internal/waf/waf.go index ec8d9ea..f0ca609 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -123,10 +123,10 @@ func New(params Params) *Server { GeoJS: h.geojs, Anomalies: h.anomalies, LookupFile: h.lookupFile, - Lists: h.lists, - DNSBL: h.dnsbl, - AbuseIPDB: h.abuseIPDB, - RouteWatch: h.routeWatch, + Lists: params.Lists, + DNSBL: params.DNSBL, + AbuseIPDB: params.AbuseIPDB, + RouteWatch: params.RouteWatch, Metrics: h.metrics, } } @@ -154,8 +154,10 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool { return false } - return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) || - rq.overAWholeRateLimit(now) + listedBy := rq.h.blocklists.lists.ListedBy(rq.client) + + return rq.h.blocklists.refusedByBlocklists(listedBy) || + rq.h.rateLimits.overAWholeRateLimit(rq, now) } // answerQuestion runs the checks on the visitor's request a question @@ -491,7 +493,7 @@ func (h *handler) finishWith( if rq.allowed { rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS - rq.runBanningHeuristics(ctx, onReport()) + rq.runBanningHeuristics(ctx, rq.h.onReport) rq.countAnomalies() rq.addToHistory()