waf: each heuristic holds its own settings and parts (closes #441) #462

Merged
clawbot merged 1 commits from issue-441-heuristic-structs into next 2026-11-19 21:43:56 +01:00
27 changed files with 647 additions and 334 deletions

No files matched your search

+20 -9
View File
@@ -22,31 +22,42 @@ import (
// an offence, and the request does not wait for the answer. The score is then // an offence, and the request does not wait for the answer. The score is then
// used for each address of the client. Its figures are the score and // used for each address of the client. Its figures are the score and
// SWW_WAF_ABUSEIPDB_MIN_SCORE. // SWW_WAF_ABUSEIPDB_MIN_SCORE.
type abuseIPDB struct{} type abuseIPDB struct {
abuseIPDB *reputation.AbuseIPDB
// limiter is the table of clients the rate limits count, whose histories
// say which clients are offenders.
limiter *ratelimit.Limiter
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
hitNoter reputationHitNoter
}
func (abuseIPDB) name() string { return "abuseipdb" } func (abuseIPDB) name() string { return "abuseipdb" }
func (abuseIPDB) check( func (a abuseIPDB) check(
ctx context.Context, ctx context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
if rq.h.config.AbuseIPDBKey == "" { if !a.keySet {
return nil return nil
} }
client := clientGroup(rq.h.config, rq.client) client := clientGroup(a.ipv6GroupPrefix, rq.client)
held, _ := rq.h.limiter.Client(client) held, _ := a.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{} offender := held.History.Offences != ratelimit.Offences{}
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender) score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
if !hit { if !hit {
return nil return nil
} }
rq.abuseIPDBHit = true rq.abuseIPDBHit = true
rq.addTag(config.AbuseIPDBTag) rq.addTag(config.AbuseIPDBTag)
rq.noteHit( a.hitNoter.noteHit(
rq,
bans.ReputationHit{ bans.ReputationHit{
Source: reputation.AbuseIPDBSource, Source: reputation.AbuseIPDBSource,
Score: &score, Score: &score,
@@ -54,14 +65,14 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE", "scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
) )
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny { if a.action != config.ReputationActionDeny {
return nil return nil
} }
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
figures: map[string]any{ figures: map[string]any{
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore, figureScore: score, "min_score": a.minScore,
}, },
} }
} }
+3 -1
View File
@@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist(
scores := make(map[netip.Prefix]int64, len(listed)) scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed { for _, client := range listed {
netblock := banNetblock(cfg, client.Addr) netblock := banNetblock(
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
)
scores[netblock] = max(scores[netblock], client.Score) scores[netblock] = max(scores[netblock], client.Score)
} }
+8 -2
View File
@@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone) return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
} }
return banNetblock(h.config, addr), nil return banNetblock(
h.config.BanScopeV4Prefix,
h.config.IPv6GroupPrefix,
addr,
), nil
} }
// expiry returns when a ban made at now for duration ends: duration // expiry returns when a ban made at now for duration ends: duration
@@ -295,7 +299,9 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)), Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
} }
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr)) client, seen := rq.h.limiter.Client(
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
)
if seen { if seen {
answer.Client = &client answer.Client = &client
} }
+105 -61
View File
@@ -1,7 +1,7 @@
package waf package waf
// The heuristic active_ban, and the code that makes bans, which several // The heuristic active_ban, and banMaker, which makes the bans of several
// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's. // heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
import ( import (
"context" "context"
@@ -11,6 +11,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -25,18 +26,22 @@ func (rq *request) banResponse(action string) *refusal {
// in, in force at now. It notes for the log line when that ban ends. A request // in, in force at now. It notes for the log line when that ban ends. A request
// that makes the ban permanent, or in observe mode would have, raises the alert // that makes the ban permanent, or in observe mode would have, raises the alert
// for it. Its figures are the ban's netblock and when it ends. // for it. Its figures are the ban's netblock and when it ends.
type activeBan struct{} type activeBan struct {
banMaker banMaker
}
func (activeBan) name() string { return "active_ban" } func (activeBan) name() string { return "active_ban" }
func (activeBan) check( func (a activeBan) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
check := rq.h.ledger.Check ledger := a.banMaker.ledger
if rq.h.config.Observe {
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is check := ledger.Check
if a.banMaker.observe {
check = ledger.Find // the ban refuses nothing, and stays as it is
} }
ban, banned, madePermanent := check(rq.client, now) ban, banned, madePermanent := check(rq.client, now)
@@ -46,7 +51,7 @@ func (activeBan) check(
if madePermanent { if madePermanent {
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
rq.alertBan(ban) a.banMaker.alertBan(rq, ban)
} }
if !banned { if !banned {
@@ -81,11 +86,12 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
// countedBytes returns the request's bytes, once it has ended, as the // countedBytes returns the request's bytes, once it has ended, as the
// byte limits and the anomaly thresholds count them: the response's body // byte limits and the anomaly thresholds count them: the response's body
// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says. // bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED,
func (rq *request) countedBytes() int64 { // says.
func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 {
response, request := rq.out.bytes, rq.requestBytes response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted { switch bytesCounted {
case config.BytesCountedResponse: case config.BytesCountedResponse:
return response return response
case config.BytesCountedRequest: case config.BytesCountedRequest:
@@ -96,7 +102,29 @@ func (rq *request) countedBytes() int64 {
return response + request return response + request
} }
// banForLimit bans the client's netblock at now for a broken limit, the // banMaker makes the bans of the heuristics that ban a client, in the ban
// ledger, and raises the alert for each. In observe mode it makes no ban, and
// raises the alert for the ban it would have made.
type banMaker struct {
ledger *bans.Ledger
// limiter is the table of clients the rate limits count. A ban for a limit
// sets the client's counters back to zero, and a ban's notes give how many
// requests its netblock has sent.
limiter *ratelimit.Limiter
metrics *metrics.Metrics
alerts *alerts.Queue
// observe is true in observe mode (SWW_WAF_MODE).
observe bool
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
// clear sign of attack give.
banResponse int
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
banScopeV4Prefix int
ipv6GroupPrefix int
}
// banForLimit bans the netblock of rq's client at now for a broken limit, the
// one hit names, notes the offence for the log line and counts the hit in // one hit names, notes the offence for the log line and counts the hit in
// the metrics. status is what the client was sent, or is sent: // the metrics. status is what the client was sent, or is sent:
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for // SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
@@ -105,7 +133,8 @@ func (rq *request) countedBytes() int64 {
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets // rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what // the client's counters back to zero. In observe mode, banClient says what
// happens instead. // happens instead.
func (rq *request) banForLimit( func (b banMaker) banForLimit(
rq *request,
now time.Time, now time.Time,
hit ratelimit.Hit, hit ratelimit.Hit,
status int, status int,
@@ -120,7 +149,8 @@ func (rq *request) banForLimit(
} }
rq.line.Offence = requestlog.OffenceLimit rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
b.metrics.LimitHit(hit)
notes := bans.Notes{ notes := bans.Notes{
Kind: hit.Kind, Kind: hit.Kind,
@@ -136,81 +166,94 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged() notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
} }
rq.banClient( b.banClient(
rq,
now, now,
status, status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes}, bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
) )
} }
// banForAttack bans the client's netblock at now for a clear sign of // banForAttack bans the netblock of rq's client at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path // attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient // asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead. // says what happens instead.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { func (b banMaker) banForAttack(
return rq.banClient( rq *request, now time.Time, notes bans.Notes,
) string {
return b.banClient(
rq,
now, now,
rq.h.config.BanResponse, b.banResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes}, bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
) )
} }
// banClient bans the client's netblock at now on grounds, and returns the // banClient bans the netblock of rq's client at now on grounds, and returns
// action for that ban. It adds to the notes what the request tells of the // the action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit // client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero. // sets the client's counters back to zero.
// //
// In observe mode it makes no ban and sets nothing back. It raises the // In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent. // alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient( func (b banMaker) banClient(
rq *request,
now time.Time, now time.Time,
status int, status int,
grounds bans.Grounds, grounds bans.Grounds,
) string { ) string {
netblock := banNetblock(rq.h.config, rq.client) netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
if rq.h.config.Observe { if b.observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) { if !b.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction( return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause), b.ledger.WouldBePermanent(netblock, now, grounds.Cause),
) )
} }
grounds.Notes = rq.withClientNotes( grounds.Notes = b.withClientNotes(
rq,
grounds.Notes, grounds.Notes,
now, now,
status, status,
netblock, netblock,
) )
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds) ban, wouldBan := b.ledger.WouldBan(netblock, now, grounds)
if wouldBan { if wouldBan {
rq.alertBan(ban) b.alertBan(rq, ban)
} }
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock) grounds.Notes = b.withClientNotes(
rq,
grounds.Notes,
now,
status,
netblock,
)
ban, made := rq.h.ledger.Ban(netblock, now, grounds) ban, made := b.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit { if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client)) b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client))
} }
rq.line.BanExpires = banExpires(ban) rq.line.BanExpires = banExpires(ban)
if made { if made {
rq.alertBan(ban) b.alertBan(rq, ban)
} }
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
// withClientNotes is notes with what the request tells of the client // withClientNotes is notes with what rq tells of its client added: its AS,
// added: its AS, country and reputation, the request at now with status, // country and reputation, the request at now with status, and how many
// and how many requests netblock, the client's, has sent. // requests netblock, the client's, has sent.
func (rq *request) withClientNotes( func (b banMaker) withClientNotes(
rq *request,
notes bans.Notes, notes bans.Notes,
now time.Time, now time.Time,
status int, status int,
@@ -221,7 +264,7 @@ func (rq *request) withClientNotes(
notes.Country = rq.line.Country notes.Country = rq.line.Country
notes.Reputation = rq.reputation notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status) notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock) notes.Requests = b.netblockRequests(netblock)
return notes return notes
} }
@@ -231,23 +274,22 @@ func (rq *request) withClientNotes(
// have made is worked out only then, at most once per // have made is worked out only then, at most once per
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the // SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
// netblock's requests, which can mean going through every client. // netblock's requests, which can mean going through every client.
func (rq *request) wouldAlertBan( func (b banMaker) wouldAlertBan(
netblock netip.Prefix, now time.Time, cause string, netblock netip.Prefix, now time.Time, cause string,
) bool { ) bool {
event := alerts.EventBan event := alerts.EventBan
if rq.h.ledger.WouldBePermanent(netblock, now, cause) { if b.ledger.WouldBePermanent(netblock, now, cause) {
event = alerts.EventPermanentBan event = alerts.EventPermanentBan
} }
return rq.h.alerts.WouldSend(event, netblock) return b.alerts.WouldSend(event, netblock)
} }
// alertBan raises the alert for ban, which the request made, or made // alertBan raises the alert for ban, which rq made, or made permanent:
// permanent: permanent_ban for a permanent ban, ban for another. Its // permanent_ban for a permanent ban, ban for another. Its detail gives the
// detail gives the ban's cause, when it ends, and its notes, and in // ban's cause, when it ends, and its notes, and in observe mode, where ban is
// observe mode, where ban is the ban that would have been made, or made // the ban that would have been made, or made permanent, mode, observe.
// permanent, mode, observe. func (b banMaker) alertBan(rq *request, ban bans.Ban) {
func (rq *request) alertBan(ban bans.Ban) {
event := alerts.EventBan event := alerts.EventBan
if ban.Permanent() { if ban.Permanent() {
event = alerts.EventPermanentBan event = alerts.EventPermanentBan
@@ -256,11 +298,11 @@ func (rq *request) alertBan(ban bans.Ban) {
detail := map[string]any{ detail := map[string]any{
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes, "cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
} }
if rq.h.config.Observe { if b.observe {
detail["mode"] = "observe" detail["mode"] = "observe"
} }
rq.h.alerts.Raise(alerts.Alert{ b.alerts.Raise(alerts.Alert{
Event: event, Event: event,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
@@ -273,6 +315,13 @@ func (rq *request) alertBan(ban bans.Ban) {
}) })
} }
// netblockRequests is how many requests netblock has sent since it was
// first seen, the request being banned for included: the histories count it
// only once it has ended.
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
return b.limiter.Requests(netblock) + 1
}
// noted is the request, at now, with status, what the client was sent, or // noted is the request, at now, with status, what the client was sent, or
// in observe mode would have been, as the notes of the ban it makes keep // in observe mode would have been, as the notes of the ban it makes keep
// it. // it.
@@ -287,23 +336,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
} }
} }
// netblockRequests is how many requests netblock has sent since it was
// first seen, this one included: the histories count it only once it has
// ended.
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1
}
// banNetblock is the netblock a ban on client covers: its IPv4 address, // banNetblock is the netblock a ban on client covers: its IPv4 address,
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup // widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group
// counts it in. // clientGroup counts it in, of length ipv6GroupPrefix.
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix { func banNetblock(
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
) netip.Prefix {
addr := client.Unmap() addr := client.Unmap()
if addr.Is4() { if addr.Is4() {
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked() return netip.PrefixFrom(addr, banScopeV4Prefix).Masked()
} }
return clientGroup(cfg, addr) return clientGroup(ipv6GroupPrefix, addr)
} }
// banExpires is when ban ends, as the log line gives it: a time, or // banExpires is when ban ends, as the log line gives it: a time, or
+12 -12
View File
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
) *verdict { ) *verdict {
var running atomic.Pointer[string] var running atomic.Pointer[string]
running.Store(new(trapPaths{}.name())) running.Store(new(rq.h.trapPaths.name()))
found := make(chan *besideLookupFindings, 1) found := make(chan *besideLookupFindings, 1)
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
return v return v
} }
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now()) return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now())
} }
// findBesideLookup does the work of the heuristics of besideLookup on the // findBesideLookup does the work of the heuristics of besideLookup on the
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
) *besideLookupFindings { ) *besideLookupFindings {
findings := &besideLookupFindings{} findings := &besideLookupFindings{}
findings.trapPath = rq.trapPath() findings.trapPath = rq.h.trapPaths.trapPath(rq)
if findings.trapPath != "" { if findings.trapPath != "" {
findings.refuses = true findings.refuses = true
return findings return findings
} }
running.Store(new(ruleFiles{}.name())) running.Store(new(rq.h.ruleFiles.name()))
findings.rules = rq.h.rules.Match(rq.in) findings.rules = rq.h.ruleFiles.rules.Match(rq.in)
if refusedByRule(findings.rules) { if refusedByRule(findings.rules) {
findings.refuses = true findings.refuses = true
return findings return findings
} }
running.Store(new(coreRuleSet{}.name())) coreRuleSet := rq.h.coreRuleSet
running.Store(new(coreRuleSet.name()))
cfg := rq.h.config if coreRuleSet.mode == config.CoreRuleSetModeOff ||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff || pathExempt(rq.in, coreRuleSet.exemptPaths) {
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
return findings return findings
} }
start := time.Now() start := time.Now()
// A question's body has been read into memory, so that reading it here // A question's body has been read into memory, so that reading it here
// cannot fail. // cannot fail.
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect( findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect(
rq.in, rq.client, rq.in.Body, rq.in, rq.client, rq.in.Body,
) )
findings.inspectionTime = time.Since(start) findings.inspectionTime = time.Since(start)
findings.inspected = true findings.inspected = true
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection) findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection)
return findings return findings
} }
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
) *verdict { ) *verdict {
rq.findings = findings rq.findings = findings
return rq.runHeuristics(ctx, besideLookup(), rq.h.now()) return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now())
} }
// timedOut refuses the request for the heuristics of besideLookup not having // timedOut refuses the request for the heuristics of besideLookup not having
+36 -15
View File
@@ -2,6 +2,7 @@ package waf
import ( import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// whole is the percentage of each limit a client gets when no biased // whole is the percentage of each limit a client gets when no biased
@@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
cfg.ASNLimitPercentURL != "" cfg.ASNLimitPercentURL != ""
} }
// biasedThresholds are the settings and the lists by which limitPercentages
// lowers a client's limits.
type biasedThresholds struct {
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
lists *reputation.Lists
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
// and its percentage.
blocklistAction config.ReputationAction
blocklistLimitPercent int64
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
dnsblAbuseIPDBAction config.ReputationAction
dnsblAbuseIPDBLimitPercent int64
}
// limitPercentages returns the client's limit percentages, for the rate // limitPercentages returns the client's limit percentages, for the rate
// limits and for the byte limits, by its tags, its AS number and country // limits and for the byte limits, by its tags, its AS number and country
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and // as looked up, each "" when unknown, and the blocklists, DNSBL zones and
@@ -45,37 +65,38 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and // the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
// an AS number or a country they list. // an AS number or a country they list.
func (rq *request) limitPercentages() (percentage, percentage) { func (b biasedThresholds) limitPercentages(
cfg := rq.h.config rq *request,
) (percentage, percentage) {
asn, country := rq.line.ASN, rq.line.Country asn, country := rq.line.ASN, rq.line.Country
unknown := percentage{percent: whole} unknown := percentage{percent: whole}
if country == "" && !rq.skippedLookupWait { if country == "" && !rq.skippedLookupWait {
unknown = percentage{ unknown = percentage{
cfg.UnknownCountryLimitPercent, b.unknownCountryLimitPercent,
config.UnknownCountryLimitPercentSetting, config.UnknownCountryLimitPercentSetting,
} }
} }
fetched := percentage{percent: whole} fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed { if percent, listed := b.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, config.ASNLimitPercentURLSetting} fetched = percentage{percent, config.ASNLimitPercentURLSetting}
} }
blocklisted := percentage{percent: whole} blocklisted := percentage{percent: whole}
if rq.blocklisted && if rq.blocklisted &&
cfg.BlocklistAction == config.ReputationActionLimit { b.blocklistAction == config.ReputationActionLimit {
blocklisted = percentage{ blocklisted = percentage{
cfg.BlocklistLimitPercent, b.blocklistLimitPercent,
config.BlocklistActionSetting, config.BlocklistActionSetting,
} }
} }
dnsblAbuseIPDBListed := percentage{percent: whole} dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) && if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit { b.dnsblAbuseIPDBAction == config.ReputationActionLimit {
dnsblAbuseIPDBListed = percentage{ dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent, b.dnsblAbuseIPDBLimitPercent,
config.DNSBLAbuseIPDBActionSetting, config.DNSBLAbuseIPDBActionSetting,
} }
} }
@@ -83,11 +104,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
const tagSetting = config.TagLimitPercentSetting const tagSetting = config.TagLimitPercentSetting
asnRequests := lowest( asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting), given(b.tagLimitPercent, asnTag(asn), tagSetting),
fetched, fetched,
) )
countryRequests := given( countryRequests := given(
cfg.TagLimitPercent, b.tagLimitPercent,
countryTag(country), countryTag(country),
tagSetting, tagSetting,
) )
@@ -98,23 +119,23 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if tag != asnTag(asn) && tag != countryTag(country) { if tag != asnTag(asn) && tag != countryTag(country) {
otherTags = lowest( otherTags = lowest(
otherTags, otherTags,
given(cfg.TagLimitPercent, tag, tagSetting), given(b.tagLimitPercent, tag, tagSetting),
) )
} }
} }
asnBytes, countryBytes := asnRequests, countryRequests asnBytes, countryBytes := asnRequests, countryRequests
if _, listed := cfg.ASNBytesLimitPercent[asn]; listed { if _, listed := b.asnBytesLimitPercent[asn]; listed {
asnBytes = given( asnBytes = given(
cfg.ASNBytesLimitPercent, b.asnBytesLimitPercent,
asn, asn,
config.ASNBytesLimitPercentSetting, config.ASNBytesLimitPercentSetting,
) )
} }
if _, listed := cfg.CountryBytesLimitPercent[country]; listed { if _, listed := b.countryBytesLimitPercent[country]; listed {
countryBytes = given( countryBytes = given(
cfg.CountryBytesLimitPercent, b.countryBytesLimitPercent,
country, country,
config.CountryBytesLimitPercentSetting, config.CountryBytesLimitPercentSetting,
) )
+12 -8
View File
@@ -8,6 +8,7 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -15,20 +16,24 @@ import (
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it // and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
// lowers the client's limits instead (see limitPercentages), and being log, it // lowers the client's limits instead (see limitPercentages), and being log, it
// does nothing more. Its figure is the blocklists that list the client. // does nothing more. Its figure is the blocklists that list the client.
type blocklists struct{} type blocklists struct {
lists *reputation.Lists
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
hitNoter reputationHitNoter
}
func (blocklists) name() string { return "blocklists" } func (blocklists) name() string { return "blocklists" }
func (blocklists) check( func (b blocklists) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
listedBy := rq.h.lists.ListedBy(rq.client) listedBy := b.lists.ListedBy(rq.client)
rq.blocklisted = len(listedBy) > 0 rq.blocklisted = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a blocklist") b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist")
if !rq.refusedByBlocklists(listedBy) { if !b.refusedByBlocklists(listedBy) {
return nil return nil
} }
@@ -41,7 +46,6 @@ func (blocklists) check(
// refusedByBlocklists reports whether the heuristic blocklists refuses a // refusedByBlocklists reports whether the heuristic blocklists refuses a
// client, given listedBy, the blocklists that list it: it does when one lists // client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny. // it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool { func (b blocklists) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction == return len(listedBy) > 0 && b.action == config.ReputationActionDeny
config.ReputationActionDeny
} }
+14 -6
View File
@@ -6,6 +6,9 @@ package waf
import ( import (
"context" "context"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
) )
// byteLimits counts the request's bytes, as countedBytes gives them, for the // byteLimits counts the request's bytes, as countedBytes gives them, for the
@@ -16,11 +19,16 @@ import (
// the client over a byte limit, as its limit percentage for the byte limits // the client over a byte limit, as its limit percentage for the byte limits
// lowers it, break it, which bans the client; the response was passed on whole. // lowers it, break it, which bans the client; the response was passed on whole.
// Its figures are the window, the count and the limit. // Its figures are the window, the count and the limit.
type byteLimits struct{} type byteLimits struct {
limiter *ratelimit.Limiter
bytesCounted config.BytesCounted // SWW_WAF_BYTES_COUNTED
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (byteLimits) name() string { return "byte_limits" } func (byteLimits) name() string { return "byte_limits" }
func (byteLimits) check( func (b byteLimits) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
@@ -29,10 +37,10 @@ func (byteLimits) check(
return nil return nil
} }
counts, hit, over := rq.h.limiter.CountBytes( counts, hit, over := b.limiter.CountBytes(
clientGroup(rq.h.config, rq.client), clientGroup(b.ipv6GroupPrefix, rq.client),
now, now,
rq.countedBytes(), rq.countedBytes(b.bytesCounted),
rq.bytesLimitPercent.percent, rq.bytesLimitPercent.percent,
) )
rq.line.Counts.MinuteBytes = counts.MinuteBytes rq.line.Counts.MinuteBytes = counts.MinuteBytes
@@ -43,7 +51,7 @@ func (byteLimits) check(
return nil return nil
} }
rq.banForLimit(now, hit, rq.out.status) b.banMaker.banForLimit(rq, now, hit, rq.out.status)
return &verdict{figures: limitFigures(hit)} return &verdict{figures: limitFigures(hit)}
} }
+5 -6
View File
@@ -5,8 +5,6 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"slices" "slices"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
// peerAddress is the address of the request's TCP peer. // peerAddress is the address of the request's TCP peer.
@@ -45,12 +43,13 @@ func scheme(r *http.Request, isQuestion bool) string {
// clientGroup is the client a request is counted toward: its IPv4 // clientGroup is the client a request is counted toward: its IPv4
// address, or its IPv6 group, the netblock its IPv6 address is in of the // address, or its IPv6 group, the netblock its IPv6 address is in of the
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser // length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. // one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix { // IPv4.
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
addr = addr.Unmap() addr = addr.Unmap()
if addr.Is6() { if addr.Is6() {
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked() return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked()
} }
return netip.PrefixFrom(addr, addr.BitLen()) return netip.PrefixFrom(addr, addr.BitLen())
+30 -18
View File
@@ -10,6 +10,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -22,11 +23,22 @@ import (
// exploitsignature, raises the core_rule_set_match alert, and in block mode // exploitsignature, raises the core_rule_set_match alert, and in block mode
// refuses the request with 403, which is an offence its client's history // refuses the request with 403, which is an offence its client's history
// counts. Its figures are the score and the threshold. // counts. Its figures are the score and the threshold.
type coreRuleSet struct{} type coreRuleSet struct {
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
// off.
inspector CoreRuleSetInspector
metrics *metrics.Metrics
alerts *alerts.Queue
mode config.CoreRuleSetMode // SWW_WAF_CORE_RULE_SET_MODE
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
observe bool // SWW_WAF_MODE is observe
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (coreRuleSet) name() string { return "core_rule_set" } func (coreRuleSet) name() string { return "core_rule_set" }
func (coreRuleSet) check( func (c coreRuleSet) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
@@ -36,7 +48,6 @@ func (coreRuleSet) check(
return nil return nil
} }
cfg := rq.h.config
result := findings.inspection result := findings.inspection
rq.requestBytes = int64(len(findings.read)) rq.requestBytes = int64(len(findings.read))
@@ -47,18 +58,18 @@ func (coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs { for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id) c.metrics.CoreRuleSetMatched(string(c.mode), id)
} }
threshold := cfg.CoreRuleSetScoreThreshold threshold := c.scoreThreshold
if threshold == 0 || result.Score < threshold { if threshold == 0 || result.Score < threshold {
return nil return nil
} }
rq.addTag(config.ExploitSignatureTag) rq.addTag(config.ExploitSignatureTag)
rq.alertCoreRuleSetMatch(result) c.alertCoreRuleSetMatch(rq, result)
if !rq.refusedByCoreRuleSet(result) { if !c.refusedByCoreRuleSet(result) {
return nil return nil
} }
@@ -76,20 +87,21 @@ func (coreRuleSet) check(
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core // refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
// Rule Set found result of: whether the score is at or over // Rule Set found result of: whether the score is at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode. // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool { func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool {
cfg := rq.h.config threshold := c.scoreThreshold
threshold := cfg.CoreRuleSetScoreThreshold
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock && return c.mode == config.CoreRuleSetModeBlock &&
threshold != 0 && result.Score >= threshold threshold != 0 && result.Score >= threshold
} }
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request, // alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the
// which the Core Rule Set scored at result, at or over // Core Rule Set scored at result, at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
// score, the method and the path with the query, and, for a request that is // score, the method and the path with the query, and, for a request that is
// not refused for it, the mode: detect, or observe in observe mode. // not refused for it, the mode: detect, or observe in observe mode.
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) { func (c coreRuleSet) alertCoreRuleSetMatch(
rq *request, result coreruleset.Result,
) {
detail := map[string]any{ detail := map[string]any{
"rule_ids": result.RuleIDs, "rule_ids": result.RuleIDs,
"score": result.Score, "score": result.Score,
@@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
} }
switch { switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect: case c.mode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect) detail["mode"] = string(config.CoreRuleSetModeDetect)
case rq.h.config.Observe: case c.observe:
detail["mode"] = "observe" detail["mode"] = "observe"
} }
rq.h.alerts.Raise(alerts.Alert{ c.alerts.Raise(alerts.Alert{
Event: alerts.EventCoreRuleSetMatch, Event: alerts.EventCoreRuleSetMatch,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client), Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN, ASN: rq.line.ASN,
ASName: rq.line.ASName, ASName: rq.line.ASName,
Country: rq.line.Country, Country: rq.line.Country,
+5 -3
View File
@@ -15,16 +15,18 @@ import (
// does not hold the client's country as it was looked up. A client without a // does not hold the client's country as it was looked up. A client without a
// country, or whose country cannot be found, is refused. Its figures are the // country, or whose country cannot be found, is refused. Its figures are the
// country and the setting that refused it. // country and the setting that refused it.
type countries struct{} type countries struct {
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
}
func (countries) name() string { return "countries" } func (countries) name() string { return "countries" }
func (countries) check( func (c countries) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
allowed := rq.h.config.ExclusivelyAllowedCountries allowed := c.allowed
country := rq.line.Country country := rq.line.Country
if len(allowed) == 0 || slices.Contains(allowed, country) { if len(allowed) == 0 || slices.Contains(allowed, country) {
+10 -3
View File
@@ -4,6 +4,7 @@ package waf
import ( import (
"context" "context"
"net/netip"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -11,12 +12,18 @@ import (
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of // denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
// the setting the client is in. // the setting the client is in.
type denyNets struct{} type denyNets struct {
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
}
func (denyNets) name() string { return "deny_nets" } func (denyNets) name() string { return "deny_nets" }
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict { func (d denyNets) check(
for _, netblock := range rq.h.config.DenyNets { _ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, netblock := range d.netblocks {
if netblock.Contains(rq.client) { if netblock.Contains(rq.client) {
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
+47
View File
@@ -0,0 +1,47 @@
package waf
import (
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
t.Parallel()
heuristic := denyNets{
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
}
// The request has no handler: the heuristic reads only what it holds.
inside := &request{
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
}
v := heuristic.check(t.Context(), inside, time.Time{})
if v == nil {
t.Fatal("a client inside the netblock was let through")
}
if v.action != requestlog.ActionClientDenied {
t.Errorf(
"action = %q, want %q",
v.action,
requestlog.ActionClientDenied,
)
}
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
}
outside := &request{
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
}
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
t.Errorf("a client outside the netblock was refused: %+v", v)
}
}
+14 -6
View File
@@ -9,6 +9,7 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -19,21 +20,28 @@ import (
// being log, it does nothing more. A zone without a verdict on the client is // being log, it does nothing more. A zone without a verdict on the client is
// asked about it in the background, and the request does not wait for the // asked about it in the background, and the request does not wait for the
// answer. Its figure is the zones that list the client, their keys masked. // answer. Its figure is the zones that list the client, their keys masked.
type dnsbl struct{} type dnsbl struct {
dnsbl *reputation.DNSBL
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
hitNoter reputationHitNoter
}
func (dnsbl) name() string { return "dnsbl" } func (dnsbl) name() string { return "dnsbl" }
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict { func (d dnsbl) check(
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client) ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
rq.dnsblListed = len(listedBy) > 0 rq.dnsblListed = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a DNSBL zone") d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone")
for _, zone := range listedBy { for _, zone := range listedBy {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone)) rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
} }
if !rq.dnsblListed || if !rq.dnsblListed || d.action != config.ReputationActionDeny {
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
return nil return nil
} }
+4 -71
View File
@@ -5,9 +5,10 @@ import (
"time" "time"
) )
// To add a heuristic, write one file with a type that implements heuristic, // To add a heuristic, write one file with a struct that implements heuristic
// and add one line for it to one of the lists below. Its settings go in // and holds the settings and parts it reads. Add one line for it to one of the
// config.WAF, and the comment at the top of its file names them. // lists that newHandler builds, which hands it those settings and parts. Its
// settings go in config.WAF, and the comment at the top of its file names them.
// heuristic is one of the checks that decide what waf does with a // heuristic is one of the checks that decide what waf does with a
// request. // request.
@@ -36,74 +37,6 @@ const (
figureNetblock = "netblock" figureNetblock = "netblock"
) )
// The lists of heuristics, each in the order its heuristics run. A client in
// SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number and
// country are looked up, so that a client they refuse is not looked up.
// header_limits comes first: it only measures and counts, and a request it
// refuses causes no other work.
func beforeLookup() []heuristic {
return []heuristic{
headerLimits{},
tagLists{},
routeWatchTags{},
denyNets{},
activeBan{},
}
}
// afterLookup are the heuristics that run once the lookup has been made.
func afterLookup() []heuristic {
return []heuristic{
countries{},
blocklists{},
dnsbl{},
abuseIPDB{},
tagDeny{},
rateLimits{},
}
}
// besideLookup are the heuristics that only read the request. Their work runs
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
// of the request. Their check then applies what that work found, and runs only
// once waf takes it.
func besideLookup() []heuristic {
return []heuristic{
trapPaths{},
ruleFiles{},
coreRuleSet{},
}
}
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second time,
// on the tag rule_files and core_rule_set can add.
func afterBesideLookup() []heuristic {
return []heuristic{
tagDeny{},
}
}
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client, and do
// not change what the request got: their verdict's action is not used.
func afterAnswer() []heuristic {
return []heuristic{
refusalLimit{},
}
}
// onReport are the heuristics that run on an allowed request when rproxy's
// report of it comes, before its report line is written. Like afterAnswer, they
// only ban.
func onReport() []heuristic {
return []heuristic{
byteLimits{},
}
}
// runHeuristics runs list, one of those checkClient runs, on the request at now, // runHeuristics runs list, one of those checkClient runs, on the request at now,
// and returns the first verdict, or nil when none gives one. The log line's // and returns the first verdict, or nil when none gives one. The log line's
// heuristic and heuristic_figures name the heuristic that gave it, and give its // heuristic and heuristic_figures name the heuristic that gave it, and give its
+5 -3
View File
@@ -56,7 +56,7 @@ func (rq *request) startLookUp(
return result return result
} }
client := clientGroup(rq.h.config, rq.client) client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile { if rq.h.config.LookupSource == config.LookupSourceFile {
result <- lookupResult{ result <- lookupResult{
@@ -143,8 +143,10 @@ func AddLookup(
) { ) {
limiter.AddLookup(answer.Client, answer.Answered, limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country) answer.ASN, answer.ASName, answer.Country)
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()), netblock := banNetblock(
answer.ASN, answer.ASName, answer.Country) cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(),
)
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
} }
// GeoJSFailed logs a failure of GeoJS to processLog, raises a // GeoJSFailed logs a failure of GeoJS to processLog, raises a
+168 -31
View File
@@ -133,15 +133,19 @@ type Server struct {
} }
// newHandler returns the handler with what every request shares, the parts // newHandler returns the handler with what every request shares, the parts
// in params among it. // in params among it. It builds the heuristics, each with the settings and
// parts it reads, and the lists they run in.
//
//nolint:funlen // a few lines for each heuristic, a list that grows with them
func newHandler(params Params) *handler { func newHandler(params Params) *handler {
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil { if err != nil {
panic(err) // New fails only for a size below one panic(err) // New fails only for a size below one
} }
return &handler{ cfg := params.Config
config: params.Config, h := &handler{
config: cfg,
requestLog: params.RequestLog, requestLog: params.RequestLog,
processLog: params.ProcessLog, processLog: params.ProcessLog,
now: params.Now, now: params.Now,
@@ -152,47 +156,142 @@ func newHandler(params Params) *handler {
geojs: params.GeoJS, geojs: params.GeoJS,
anomalies: params.Anomalies, anomalies: params.Anomalies,
lookupFile: params.LookupFile, lookupFile: params.LookupFile,
lists: params.Lists,
dnsbl: params.DNSBL,
abuseIPDB: params.AbuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn= // The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before // or country= tag and the biased thresholds act on the answer before
// the request goes on. // the request goes on.
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 || waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders || cfg.AddLookupHeaders ||
params.Config.TagsNeedLookup() || cfg.TagsNeedLookup() ||
biasedThresholdsSet(params.Config), biasedThresholdsSet(cfg),
besideLookupLimit: cmp.Or( besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit, params.BesideLookupTimeLimit,
besideLookupTimeLimit, besideLookupTimeLimit,
), ),
} }
sharedBanMaker := banMaker{
ledger: params.Ledger,
limiter: params.Limiter,
metrics: params.Metrics,
alerts: params.Alerts,
observe: cfg.Observe,
banResponse: cfg.BanResponse,
banScopeV4Prefix: cfg.BanScopeV4Prefix,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
hitNoter := reputationHitNoter{
metrics: params.Metrics,
alerts: params.Alerts,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.blocklists = blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
}
h.rateLimits = rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
metrics: params.Metrics,
banMaker: sharedBanMaker,
}
h.coreRuleSet = coreRuleSet{
inspector: params.CoreRuleSet,
metrics: params.Metrics,
alerts: params.Alerts,
mode: cfg.CoreRuleSetMode,
exemptPaths: cfg.CoreRuleSetExemptPaths,
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
observe: cfg.Observe,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.beforeLookup = []heuristic{
headerLimits{},
tagLists{lists: params.Lists},
routeWatchTags{
routeWatch: params.RouteWatch,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
},
denyNets{netblocks: cfg.DenyNets},
activeBan{banMaker: sharedBanMaker},
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
h.blocklists,
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
hitNoter: hitNoter,
},
abuseIPDB{
abuseIPDB: params.AbuseIPDB,
limiter: params.Limiter,
keySet: cfg.AbuseIPDBKey != "",
minScore: cfg.AbuseIPDBMinScore,
action: cfg.DNSBLAbuseIPDBAction,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
h.rateLimits,
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
h.afterAnswer = []heuristic{
refusalLimit{
limiter: params.Limiter,
perMinute: cfg.RefusalsLimitPerMinute,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.onReport = []heuristic{
byteLimits{
limiter: params.Limiter,
bytesCounted: cfg.BytesCounted,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
return h
} }
// handler is waf's handler. It holds what every request shares; what // handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request. // belongs to one request is in a request.
type handler struct { type handler struct {
config *config.WAF config *config.WAF
requestLog io.Writer requestLog io.Writer
processLog *slog.Logger processLog *slog.Logger
now func() time.Time now func() time.Time
metrics *metrics.Metrics metrics *metrics.Metrics
limiter *ratelimit.Limiter limiter *ratelimit.Limiter
ledger *bans.Ledger ledger *bans.Ledger
geojs *geojs.GeoJS geojs *geojs.GeoJS
anomalies *anomaly.Counters anomalies *anomaly.Counters
lookupFile *lookup.File lookupFile *lookup.File
lists *reputation.Lists
dnsbl *reputation.DNSBL
abuseIPDB *reputation.AbuseIPDB
routeWatch *routewatch.Client
rules *rules.Files
coreRuleSet CoreRuleSetInspector
alerts *alerts.Queue
// waitForGeoJS is true when a setting needs each request's answer from // waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on. // GeoJS before the request goes on.
waitForGeoJS bool waitForGeoJS bool
@@ -202,4 +301,42 @@ type handler struct {
// awaitingReport holds the decision on each answered question by its id // awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled. // until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision] awaitingReport *lru.Cache[string, *decision]
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number
// and country are looked up, so that a client they refuse is not looked
// up. header_limits comes first: it only measures and counts, and a
// request it refuses causes no other work.
beforeLookup []heuristic
// afterLookup are the heuristics that run once the lookup has been made.
afterLookup []heuristic
// besideLookup are the heuristics that only read the request. Their work
// runs beside the lookup and afterLookup, in findBesideLookup, which
// changes nothing of the request. Their check then applies what that work
// found, and runs only once waf takes it.
besideLookup []heuristic
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second
// time, on the tag rule_files and core_rule_set can add.
afterBesideLookup []heuristic
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client,
// and do not change what the request got: their verdict's action is not
// used.
afterAnswer []heuristic
// onReport are the heuristics that run on an allowed request when
// rproxy's report of it comes, before its report line is written. Like
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of the lists that other steps of a request use too:
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
blocklists blocklists
rateLimits rateLimits
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
} }
+34 -22
View File
@@ -6,8 +6,10 @@ package waf
import ( import (
"context" "context"
"net/netip"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -18,28 +20,37 @@ import (
// refused, and bans the client. A request exempt from the rate limits is exempt // refused, and bans the client. A request exempt from the rate limits is exempt
// from the byte limits too. Its figures are the window, the count and the // from the byte limits too. Its figures are the window, the count and the
// limit. // limit.
type rateLimits struct{} type rateLimits struct {
limiter *ratelimit.Limiter
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
biasedThresholds biasedThresholds
banMaker banMaker
}
func (rateLimits) name() string { return "rate_limits" } func (rateLimits) name() string { return "rate_limits" }
func (rateLimits) check( func (r rateLimits) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
rq.counted = rq.countedByRateLimits() rq.counted = r.countedByRateLimits(rq)
if !rq.counted { if !rq.counted {
return nil return nil
} }
rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages() rq.rateLimitPercent, rq.bytesLimitPercent =
r.biasedThresholds.limitPercentages(rq)
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting = rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
rq.rateLimitPercent.logged() rq.rateLimitPercent.logged()
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting = rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged() rq.bytesLimitPercent.logged()
counts, hit, over := rq.h.limiter.Count( counts, hit, over := r.limiter.Count(
clientGroup(rq.h.config, rq.client), clientGroup(r.ipv6GroupPrefix, rq.client),
now, now,
rq.rateLimitPercent.percent, rq.rateLimitPercent.percent,
) )
@@ -49,28 +60,29 @@ func (rateLimits) check(
return nil return nil
} }
rq.banForLimit(now, hit, rq.h.config.BanResponse) r.banMaker.banForLimit(rq, now, hit, r.banResponse)
return &verdict{ return &verdict{
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit), action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
} }
} }
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the // overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
// request at now, whatever its client's limit percentages: the rate limits // at now, whatever its client's limit percentages: the rate limits count it,
// count it, and one more request from its client takes it over a whole rate // and one more request from its client takes it over a whole rate limit. It
// limit. It counts nothing. // counts nothing.
func (rq *request) overAWholeRateLimit(now time.Time) bool { func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
return rq.countedByRateLimits() && return r.countedByRateLimits(rq) &&
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now) r.limiter.OverARateLimit(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
)
} }
// countedByRateLimits reports whether the rate limits count the request: its // countedByRateLimits reports whether the rate limits count rq: its client is
// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt // not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under
// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS. // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
func (rq *request) countedByRateLimits() bool { func (r rateLimits) countedByRateLimits(rq *request) bool {
cfg := rq.h.config return !isInside(rq.client, r.exemptNets) &&
!pathExempt(rq.in, r.exemptPaths)
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
} }
+15 -9
View File
@@ -8,6 +8,7 @@ import (
"net/http" "net/http"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -18,17 +19,22 @@ import (
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the // a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
// heuristics skip, is not counted, and nothing is while the limit is off. // heuristics skip, is not counted, and nothing is while the limit is off.
// Its figures are the window, the count and the limit. // Its figures are the window, the count and the limit.
type refusalLimit struct{} type refusalLimit struct {
limiter *ratelimit.Limiter
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (refusalLimit) name() string { return "refusal_limit" } func (refusalLimit) name() string { return "refusal_limit" }
func (refusalLimit) check( func (r refusalLimit) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
) *verdict { ) *verdict {
cfg := rq.h.config if r.perMinute == 0 {
if cfg.RefusalsLimitPerMinute == 0 {
return nil return nil
} }
@@ -36,10 +42,10 @@ func (refusalLimit) check(
return nil return nil
} }
hit, over := rq.h.limiter.CountRefusal( hit, over := r.limiter.CountRefusal(
clientGroup(rq.h.config, rq.client), clientGroup(r.ipv6GroupPrefix, rq.client),
now, now,
cfg.RefusalsLimitPerMinute, r.perMinute,
) )
if !over { if !over {
return nil return nil
@@ -54,10 +60,10 @@ func (refusalLimit) check(
status = http.StatusForbidden status = http.StatusForbidden
case requestlog.ActionTemporarilyBanned, case requestlog.ActionTemporarilyBanned,
requestlog.ActionPermanentlyBanned: requestlog.ActionPermanentlyBanned:
status = cfg.BanResponse status = r.banResponse
} }
rq.banForLimit(now, hit, status) r.banMaker.banForLimit(rq, now, hit, status)
return &verdict{figures: limitFigures(hit)} return &verdict{figures: limitFigures(hit)}
} }
+21 -8
View File
@@ -5,22 +5,35 @@ package waf
import ( import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/metrics"
) )
// reputationHitNoter notes the reputation sources that list a request's
// client, for the reputation heuristics.
type reputationHitNoter struct {
metrics *metrics.Metrics
alerts *alerts.Queue
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
// noteListed notes each of sources, the URLs of the blocklists or the // noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does, // DNSBL zones, their keys masked, that list rq's client, as noteHit does,
// with reason. // with reason.
func (rq *request) noteListed(sources []string, reason string) { func (n reputationHitNoter) noteListed(
rq *request, sources []string, reason string,
) {
for _, source := range sources { for _, source := range sources {
rq.noteHit(bans.ReputationHit{Source: source}, reason) n.noteHit(rq, bans.ReputationHit{Source: source}, reason)
} }
} }
// noteHit adds hit's source, which lists the client, to the log line's // noteHit adds hit's source, which lists rq's client, to the log line's
// reputation, and hit to the notes of a ban the request makes, counts the // reputation, and hit to the notes of a ban the request makes, counts the
// source in the metrics, and raises a reputation_hit alert with reason, // source in the metrics, and raises a reputation_hit alert with reason,
// whose detail gives hit's source and score. // whose detail gives hit's source and score.
func (rq *request) noteHit(hit bans.ReputationHit, reason string) { func (n reputationHitNoter) noteHit(
rq *request, hit bans.ReputationHit, reason string,
) {
detail := map[string]any{"source": hit.Source} detail := map[string]any{"source": hit.Source}
if hit.Score != nil { if hit.Score != nil {
detail["score"] = *hit.Score detail["score"] = *hit.Score
@@ -28,12 +41,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
rq.line.Reputation = append(rq.line.Reputation, hit.Source) rq.line.Reputation = append(rq.line.Reputation, hit.Source)
rq.reputation = append(rq.reputation, hit) rq.reputation = append(rq.reputation, hit)
rq.h.metrics.ReputationHit(hit.Source) n.metrics.ReputationHit(hit.Source)
rq.h.alerts.Raise(alerts.Alert{ n.alerts.Raise(alerts.Alert{
Event: alerts.EventReputationHit, Event: alerts.EventReputationHit,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client), Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN, ASN: rq.line.ASN,
ASName: rq.line.ASName, ASName: rq.line.ASName,
Country: rq.line.Country, Country: rq.line.Country,
+13 -8
View File
@@ -122,8 +122,11 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion), RequestID: requestID(r, isQuestion),
PeerIP: peer.String(), PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "), ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: clientGroup(h.config, client).String(), ClientGroup: clientGroup(
ContentType: r.Header.Get("Content-Type"), h.config.IPv6GroupPrefix,
client,
).String(),
ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders( RequestHeaders: requestHeaders(
r, r,
h.config.LogRequestHeaders, h.config.LogRequestHeaders,
@@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
now := rq.h.now() now := rq.h.now()
v := rq.runHeuristics(ctx, beforeLookup(), now) v := rq.runHeuristics(ctx, rq.h.beforeLookup, now)
if v != nil { if v != nil {
return v return v
} }
@@ -265,7 +268,7 @@ func (rq *request) runAfterLookup(
// again: a request counted at the time it arrived could be counted // again: a request counted at the time it arrived could be counted
// after a later one of its client's, which the rate limits would take // after a later one of its client's, which the rate limits would take
// for the clock set back. // for the clock set back.
return rq.runHeuristics(ctx, afterLookup(), rq.h.now()) return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now())
} }
// pathExempt reports whether r is exempt under prefixes, // pathExempt reports whether r is exempt under prefixes,
@@ -375,7 +378,7 @@ func (rq *request) addToHistory() {
} }
rq.h.limiter.AddToHistory( rq.h.limiter.AddToHistory(
clientGroup(rq.h.config, rq.client), clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
rq.h.now(), rq.h.now(),
request, request,
) )
@@ -404,11 +407,11 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
ClientGroup: clientGroup(rq.h.config, rq.client), ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
ASN: answer.ASN, ASN: answer.ASN,
ASName: answer.ASName, ASName: answer.ASName,
Country: answer.Country, Country: answer.Country,
Bytes: rq.countedBytes(), Bytes: rq.countedBytes(rq.h.config.BytesCounted),
}) })
} }
@@ -435,7 +438,9 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return rq.lookupAnswer, true return rq.lookupAnswer, true
} }
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client)) return rq.h.geojs.Kept(
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
)
} }
// responseWriter is waf's answer to a request. It notes the status and // responseWriter is waf's answer to a request. It notes the status and
+10 -5
View File
@@ -8,6 +8,7 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
) )
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock> // routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
@@ -17,22 +18,26 @@ import (
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and // SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
// asked for by the request's address, and a route without an origin AS gives only // asked for by the request's address, and a route without an origin AS gives only
// the prefix= tag. // the prefix= tag.
type routeWatchTags struct{} type routeWatchTags struct {
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
routeWatch *routewatch.Client
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (routeWatchTags) name() string { return "routewatch_tags" } func (routeWatchTags) name() string { return "routewatch_tags" }
func (routeWatchTags) check( func (r routeWatchTags) check(
ctx context.Context, ctx context.Context,
rq *request, rq *request,
_ time.Time, _ time.Time,
) *verdict { ) *verdict {
if rq.h.routeWatch == nil { if r.routeWatch == nil {
return nil return nil
} }
route, missing := rq.h.routeWatch.Route( route, missing := r.routeWatch.Route(
ctx, ctx,
clientGroup(rq.h.config, rq.client), clientGroup(r.ipv6GroupPrefix, rq.client),
rq.client, rq.client,
) )
if missing != "" { if missing != "" {
+10 -4
View File
@@ -9,6 +9,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
) )
@@ -22,11 +23,15 @@ import (
// mode raises the alert for the ban // mode raises the alert for the ban
// it would have made. Either rule's match is noted as an offence, for the // it would have made. Either rule's match is noted as an offence, for the
// client's history. Its figures are the rule's id and action. // client's history. Its figures are the rule's id and action.
type ruleFiles struct{} type ruleFiles struct {
rules *rules.Files
metrics *metrics.Metrics
banMaker banMaker
}
func (ruleFiles) name() string { return "rule_files" } func (ruleFiles) name() string { return "rule_files" }
func (ruleFiles) check( func (r ruleFiles) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
@@ -35,7 +40,7 @@ func (ruleFiles) check(
for _, rule := range matched { for _, rule := range matched {
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID) rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
rq.h.metrics.RuleMatched(rule.ID, rule.Action) r.metrics.RuleMatched(rule.ID, rule.Action)
} }
if len(matched) == 0 { if len(matched) == 0 {
@@ -60,7 +65,8 @@ func (ruleFiles) check(
rq.attack = true rq.attack = true
return &verdict{ return &verdict{
action: rq.banForAttack( action: r.banMaker.banForAttack(
rq,
now, now,
bans.Notes{RuleID: last.ID, Target: last.Target}, bans.Notes{RuleID: last.ID, Target: last.Target},
), ),
+9 -3
View File
@@ -12,13 +12,19 @@ import (
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is // tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
// the first such tag, in sorted order. // the first such tag, in sorted order.
type tagDeny struct{} type tagDeny struct {
tags []string // SWW_WAF_TAG_DENY
}
func (tagDeny) name() string { return "tag_deny" } func (tagDeny) name() string { return "tag_deny" }
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict { func (t tagDeny) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, tag := range rq.line.Tags { for _, tag := range rq.line.Tags {
if slices.Contains(rq.h.config.TagDeny, tag) { if slices.Contains(t.tags, tag) {
return &verdict{ return &verdict{
action: requestlog.ActionClientDenied, action: requestlog.ActionClientDenied,
figures: map[string]any{"tag": tag}, figures: map[string]any{"tag": tag},
+11 -3
View File
@@ -6,17 +6,25 @@ package waf
import ( import (
"context" "context"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that // tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
// holds the client, the tag being the list's name. It refuses nothing, and runs // holds the client, the tag being the list's name. It refuses nothing, and runs
// first so that every refusal's log line carries these tags. // first so that every refusal's log line carries these tags.
type tagLists struct{} type tagLists struct {
lists *reputation.Lists
}
func (tagLists) name() string { return "tag_lists" } func (tagLists) name() string { return "tag_lists" }
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict { func (t tagLists) check(
for _, name := range rq.h.lists.TagsOf(rq.client) { _ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, name := range t.lists.TagsOf(rq.client) {
rq.addTag(name) rq.addTag(name)
} }
+15 -8
View File
@@ -16,11 +16,14 @@ import (
// it bans the client's netblock, or in observe mode raises the alert for the // it bans the client's netblock, or in observe mode raises the alert for the
// ban it would have made. The ban's notes give the path as sent, and so does // ban it would have made. The ban's notes give the path as sent, and so does
// its figure. // its figure.
type trapPaths struct{} type trapPaths struct {
paths []string // SWW_WAF_TRAP_PATHS
banMaker banMaker
}
func (trapPaths) name() string { return "trap_paths" } func (trapPaths) name() string { return "trap_paths" }
func (trapPaths) check( func (t trapPaths) check(
_ context.Context, _ context.Context,
rq *request, rq *request,
now time.Time, now time.Time,
@@ -33,18 +36,22 @@ func (trapPaths) check(
rq.attack = true rq.attack = true
return &verdict{ return &verdict{
action: rq.banForAttack(now, bans.Notes{TrapPath: path}), action: t.banMaker.banForAttack(
rq,
now,
bans.Notes{TrapPath: path},
),
figures: map[string]any{"path": path}, figures: map[string]any{"path": path},
} }
} }
// trapPath returns the request's path as sent, when its path as a path rule // trapPath returns rq's path as sent, when its path as a path rule sees it,
// sees it, without the query, as sent or once percent-decoded, is in // without the query, as sent or once percent-decoded, is in
// SWW_WAF_TRAP_PATHS, and "" otherwise. // SWW_WAF_TRAP_PATHS, and "" otherwise.
func (rq *request) trapPath() string { func (t trapPaths) trapPath(rq *request) string {
path := rules.Path(rq.in) path := rules.Path(rq.in)
if !slices.Contains(rq.h.config.TrapPaths, path) && if !slices.Contains(t.paths, path) &&
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) { !slices.Contains(t.paths, rules.DecodeOnce(path)) {
return "" return ""
} }
+11 -9
View File
@@ -123,10 +123,10 @@ func New(params Params) *Server {
GeoJS: h.geojs, GeoJS: h.geojs,
Anomalies: h.anomalies, Anomalies: h.anomalies,
LookupFile: h.lookupFile, LookupFile: h.lookupFile,
Lists: h.lists, Lists: params.Lists,
DNSBL: h.dnsbl, DNSBL: params.DNSBL,
AbuseIPDB: h.abuseIPDB, AbuseIPDB: params.AbuseIPDB,
RouteWatch: h.routeWatch, RouteWatch: params.RouteWatch,
Metrics: h.metrics, Metrics: h.metrics,
} }
} }
@@ -154,8 +154,10 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
return false return false
} }
return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) || listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
rq.overAWholeRateLimit(now)
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
} }
// answerQuestion runs the checks on the visitor's request a question // answerQuestion runs the checks on the visitor's request a question
@@ -194,7 +196,7 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
if refused == nil && !underOwnPrefix { if refused == nil && !underOwnPrefix {
rq.allow() rq.allow()
// In observe mode, what enforce mode would have refused counts here. // In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), afterAnswer()) rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
return return
} }
@@ -213,7 +215,7 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
) )
} }
rq.banAndCount(r.Context(), afterAnswer()) rq.banAndCount(r.Context(), rq.h.afterAnswer)
} }
// readQuestion returns the visitor's address and the body a question carries, // readQuestion returns the visitor's address and the body a question carries,
@@ -486,7 +488,7 @@ func (h *handler) finishWith(
if rq.allowed { if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, onReport()) rq.banAndCount(ctx, rq.h.onReport)
rq.h.metrics.RequestReported( rq.h.metrics.RequestReported(
&rq.line, &rq.line,