waf: each heuristic holds its own settings and parts (closes #441) #462

Open
clawbot wants to merge 1 commits from issue-441-heuristic-structs into next
pull from: issue-441-heuristic-structs
27 changed files with 632 additions and 332 deletions

No files matched your search

+20 -9
View File
@@ -22,31 +22,42 @@ import (
// an offence, and the request does not wait for the answer. The score is then
// used for each address of the client. Its figures are the score and
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
type abuseIPDB struct{}
type abuseIPDB struct {
abuseIPDB *reputation.AbuseIPDB
// limiter is the table of clients the rate limits count, whose histories
// say which clients are offenders.
limiter *ratelimit.Limiter
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
hitNoter reputationHitNoter
}
func (abuseIPDB) name() string { return "abuseipdb" }
func (abuseIPDB) check(
func (a abuseIPDB) check(
ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
if rq.h.config.AbuseIPDBKey == "" {
if !a.keySet {
return nil
}
client := clientGroup(rq.h.config, rq.client)
held, _ := rq.h.limiter.Client(client)
client := clientGroup(a.ipv6GroupPrefix, rq.client)
held, _ := a.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{}
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
if !hit {
return nil
}
rq.abuseIPDBHit = true
rq.addTag(config.AbuseIPDBTag)
rq.noteHit(
a.hitNoter.noteHit(
rq,
bans.ReputationHit{
Source: reputation.AbuseIPDBSource,
Score: &score,
@@ -54,14 +65,14 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
)
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if a.action != config.ReputationActionDeny {
return nil
}
return &verdict{
action: requestlog.ActionClientDenied,
figures: map[string]any{
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
figureScore: score, "min_score": a.minScore,
},
}
}
+3 -1
View File
@@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist(
scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed {
netblock := banNetblock(cfg, client.Addr)
netblock := banNetblock(
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
)
scores[netblock] = max(scores[netblock], client.Score)
}
+8 -2
View File
@@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
}
return banNetblock(h.config, addr), nil
return banNetblock(
h.config.BanScopeV4Prefix,
h.config.IPv6GroupPrefix,
addr,
), nil
}
// expiry returns when a ban made at now for duration ends: duration
@@ -295,7 +299,9 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
}
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
client, seen := rq.h.limiter.Client(
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
)
if seen {
answer.Client = &client
}
+91 -60
View File
@@ -1,7 +1,7 @@
package waf
// The heuristic active_ban, and the code that makes bans, which several
// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
// The heuristic active_ban, and banMaker, which makes the bans of several
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
import (
"context"
@@ -11,6 +11,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -25,18 +26,22 @@ func (rq *request) banResponse(action string) *refusal {
// in, in force at now. It notes for the log line when that ban ends. A request
// that makes the ban permanent, or in observe mode would have, raises the alert
// for it. Its figures are the ban's netblock and when it ends.
type activeBan struct{}
type activeBan struct {
banMaker banMaker
}
func (activeBan) name() string { return "active_ban" }
func (activeBan) check(
func (a activeBan) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
check := rq.h.ledger.Check
if rq.h.config.Observe {
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
ledger := a.banMaker.ledger
check := ledger.Check
if a.banMaker.observe {
check = ledger.Find // the ban refuses nothing, and stays as it is
}
ban, banned, madePermanent := check(rq.client, now)
@@ -46,7 +51,7 @@ func (activeBan) check(
if madePermanent {
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
rq.alertBan(ban)
a.banMaker.alertBan(rq, ban)
}
if !banned {
@@ -81,11 +86,12 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
// countedBytes returns the request's bytes, once it has ended, as the
// byte limits and the anomaly thresholds count them: the response's body
// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says.
func (rq *request) countedBytes() int64 {
// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED,
// says.
func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 {
response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted {
switch bytesCounted {
case config.BytesCountedResponse:
return response
case config.BytesCountedRequest:
@@ -96,7 +102,29 @@ func (rq *request) countedBytes() int64 {
return response + request
}
// banForLimit bans the client's netblock at now for a broken limit, the
// banMaker makes the bans of the heuristics that ban a client, in the ban
// ledger, and raises the alert for each. In observe mode it makes no ban, and
// raises the alert for the ban it would have made.
type banMaker struct {
ledger *bans.Ledger
// limiter is the table of clients the rate limits count. A ban for a limit
// sets the client's counters back to zero, and a ban's notes give how many
// requests its netblock has sent.
limiter *ratelimit.Limiter
metrics *metrics.Metrics
alerts *alerts.Queue
// observe is true in observe mode (SWW_WAF_MODE).
observe bool
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
// clear sign of attack give.
banResponse int
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
banScopeV4Prefix int
ipv6GroupPrefix int
}
// banForLimit bans the netblock of rq's client at now for a broken limit, the
// one hit names, notes the offence for the log line and counts the hit in
// the metrics. status is what the client was sent, or is sent:
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
@@ -106,7 +134,8 @@ func (rq *request) countedBytes() int64 {
// the client's counters back to zero. In observe mode it makes no ban and
// sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (rq *request) banForLimit(
func (b banMaker) banForLimit(
rq *request,
now time.Time,
hit ratelimit.Hit,
status int,
@@ -121,11 +150,11 @@ func (rq *request) banForLimit(
}
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
b.metrics.LimitHit(hit)
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
@@ -139,7 +168,7 @@ func (rq *request) banForLimit(
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
Requests: b.netblockRequests(netblock),
}
switch hit.Kind {
@@ -149,35 +178,36 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
}
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
if b.observe {
ban, wouldBan := b.ledger.WouldBanForLimit(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
b.alertBan(rq, ban)
}
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
ban, made := b.ledger.BanForLimit(netblock, now, notes)
b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
b.alertBan(rq, ban)
}
}
// banForAttack bans the client's netblock at now for a clear sign of
// banForAttack bans the netblock of rq's client at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It fills in the rest of the notes. In observe mode it makes
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
func (b banMaker) banForAttack(
rq *request, now time.Time, notes bans.Notes,
) string {
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
b.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
@@ -185,23 +215,23 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
notes.Request = rq.noted(now, b.banResponse)
notes.Requests = b.netblockRequests(netblock)
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
if b.observe {
ban, wouldBan := b.ledger.WouldBanForAttack(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
b.alertBan(rq, ban)
}
return banAction(ban.Permanent())
}
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
ban, made := b.ledger.BanForAttack(netblock, now, notes)
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
b.alertBan(rq, ban)
}
return banAction(ban.Permanent())
@@ -212,23 +242,22 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
// have made is worked out only then, at most once per
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
// netblock's requests, which can mean going through every client.
func (rq *request) wouldAlertBan(
func (b banMaker) wouldAlertBan(
netblock netip.Prefix, now time.Time, cause string,
) bool {
event := alerts.EventBan
if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
if b.ledger.WouldBePermanent(netblock, now, cause) {
event = alerts.EventPermanentBan
}
return rq.h.alerts.WouldSend(event, netblock)
return b.alerts.WouldSend(event, netblock)
}
// alertBan raises the alert for ban, which the request made, or made
// permanent: permanent_ban for a permanent ban, ban for another. Its
// detail gives the ban's cause, when it ends, and its notes, and in
// observe mode, where ban is the ban that would have been made, or made
// permanent, mode, observe.
func (rq *request) alertBan(ban bans.Ban) {
// alertBan raises the alert for ban, which rq made, or made permanent:
// permanent_ban for a permanent ban, ban for another. Its detail gives the
// ban's cause, when it ends, and its notes, and in observe mode, where ban is
// the ban that would have been made, or made permanent, mode, observe.
func (b banMaker) alertBan(rq *request, ban bans.Ban) {
event := alerts.EventBan
if ban.Permanent() {
event = alerts.EventPermanentBan
@@ -237,11 +266,11 @@ func (rq *request) alertBan(ban bans.Ban) {
detail := map[string]any{
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
}
if rq.h.config.Observe {
if b.observe {
detail["mode"] = "observe"
}
rq.h.alerts.Raise(alerts.Alert{
b.alerts.Raise(alerts.Alert{
Event: event,
Host: rq.site,
Client: rq.client,
@@ -254,6 +283,13 @@ func (rq *request) alertBan(ban bans.Ban) {
})
}
// netblockRequests is how many requests netblock has sent since it was
// first seen, the request being banned for included: the histories count it
// only once it has ended.
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
return b.limiter.Requests(netblock) + 1
}
// noted is the request, at now, with status, what the client was sent, or
// in observe mode would have been, as the notes of the ban it makes keep
// it.
@@ -268,23 +304,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
}
}
// netblockRequests is how many requests netblock has sent since it was
// first seen, this one included: the histories count it only once it has
// ended.
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1
}
// banNetblock is the netblock a ban on client covers: its IPv4 address,
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
// counts it in.
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group
// clientGroup counts it in, of length ipv6GroupPrefix.
func banNetblock(
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
) netip.Prefix {
addr := client.Unmap()
if addr.Is4() {
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
return netip.PrefixFrom(addr, banScopeV4Prefix).Masked()
}
return clientGroup(cfg, addr)
return clientGroup(ipv6GroupPrefix, addr)
}
// banExpires is when ban ends, as the log line gives it: a time, or
+12 -12
View File
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
) *verdict {
var running atomic.Pointer[string]
running.Store(new(trapPaths{}.name()))
running.Store(new(rq.h.trapPaths.name()))
found := make(chan *besideLookupFindings, 1)
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
return v
}
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now())
}
// findBesideLookup does the work of the heuristics of besideLookup on the
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
) *besideLookupFindings {
findings := &besideLookupFindings{}
findings.trapPath = rq.trapPath()
findings.trapPath = rq.h.trapPaths.trapPath(rq)
if findings.trapPath != "" {
findings.refuses = true
return findings
}
running.Store(new(ruleFiles{}.name()))
running.Store(new(rq.h.ruleFiles.name()))
findings.rules = rq.h.rules.Match(rq.in)
findings.rules = rq.h.ruleFiles.rules.Match(rq.in)
if refusedByRule(findings.rules) {
findings.refuses = true
return findings
}
running.Store(new(coreRuleSet{}.name()))
coreRuleSet := rq.h.coreRuleSet
running.Store(new(coreRuleSet.name()))
cfg := rq.h.config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
if coreRuleSet.mode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, coreRuleSet.exemptPaths) {
return findings
}
start := time.Now()
// A question's body has been read into memory, so that reading it here
// cannot fail.
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect(
rq.in, rq.client, rq.in.Body,
)
findings.inspectionTime = time.Since(start)
findings.inspected = true
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection)
return findings
}
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
) *verdict {
rq.findings = findings
return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now())
}
// timedOut refuses the request for the heuristics of besideLookup not having
+36 -15
View File
@@ -2,6 +2,7 @@ package waf
import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// whole is the percentage of each limit a client gets when no biased
@@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
cfg.ASNLimitPercentURL != ""
}
// biasedThresholds are the settings and the lists by which limitPercentages
// lowers a client's limits.
type biasedThresholds struct {
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
lists *reputation.Lists
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
// and its percentage.
blocklistAction config.ReputationAction
blocklistLimitPercent int64
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
dnsblAbuseIPDBAction config.ReputationAction
dnsblAbuseIPDBLimitPercent int64
}
// limitPercentages returns the client's limit percentages, for the rate
// limits and for the byte limits, by its tags, its AS number and country
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
@@ -45,37 +65,38 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
// an AS number or a country they list.
func (rq *request) limitPercentages() (percentage, percentage) {
cfg := rq.h.config
func (b biasedThresholds) limitPercentages(
rq *request,
) (percentage, percentage) {
asn, country := rq.line.ASN, rq.line.Country
unknown := percentage{percent: whole}
if country == "" && !rq.skippedLookupWait {
unknown = percentage{
cfg.UnknownCountryLimitPercent,
b.unknownCountryLimitPercent,
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
}
}
fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
if percent, listed := b.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
}
blocklisted := percentage{percent: whole}
if rq.blocklisted &&
cfg.BlocklistAction == config.ReputationActionLimit {
b.blocklistAction == config.ReputationActionLimit {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
b.blocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION",
}
}
dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
b.dnsblAbuseIPDBAction == config.ReputationActionLimit {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
b.dnsblAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
}
}
@@ -83,11 +104,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
given(b.tagLimitPercent, asnTag(asn), tagSetting),
fetched,
)
countryRequests := given(
cfg.TagLimitPercent,
b.tagLimitPercent,
countryTag(country),
tagSetting,
)
@@ -98,23 +119,23 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if tag != asnTag(asn) && tag != countryTag(country) {
otherTags = lowest(
otherTags,
given(cfg.TagLimitPercent, tag, tagSetting),
given(b.tagLimitPercent, tag, tagSetting),
)
}
}
asnBytes, countryBytes := asnRequests, countryRequests
if _, listed := cfg.ASNBytesLimitPercent[asn]; listed {
if _, listed := b.asnBytesLimitPercent[asn]; listed {
asnBytes = given(
cfg.ASNBytesLimitPercent,
b.asnBytesLimitPercent,
asn,
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
)
}
if _, listed := cfg.CountryBytesLimitPercent[country]; listed {
if _, listed := b.countryBytesLimitPercent[country]; listed {
countryBytes = given(
cfg.CountryBytesLimitPercent,
b.countryBytesLimitPercent,
country,
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
)
+12 -8
View File
@@ -8,6 +8,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -15,20 +16,24 @@ import (
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
// lowers the client's limits instead (see limitPercentages), and being log, it
// does nothing more. Its figure is the blocklists that list the client.
type blocklists struct{}
type blocklists struct {
lists *reputation.Lists
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
hitNoter reputationHitNoter
}
func (blocklists) name() string { return "blocklists" }
func (blocklists) check(
func (b blocklists) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
listedBy := rq.h.lists.ListedBy(rq.client)
listedBy := b.lists.ListedBy(rq.client)
rq.blocklisted = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a blocklist")
b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist")
if !rq.refusedByBlocklists(listedBy) {
if !b.refusedByBlocklists(listedBy) {
return nil
}
@@ -41,7 +46,6 @@ func (blocklists) check(
// refusedByBlocklists reports whether the heuristic blocklists refuses a
// client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
config.ReputationActionDeny
func (b blocklists) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && b.action == config.ReputationActionDeny
}
+14 -6
View File
@@ -6,6 +6,9 @@ package waf
import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)
// byteLimits counts the request's bytes, as countedBytes gives them, for the
@@ -16,11 +19,16 @@ import (
// the client over a byte limit, as its limit percentage for the byte limits
// lowers it, break it, which bans the client; the response was passed on whole.
// Its figures are the window, the count and the limit.
type byteLimits struct{}
type byteLimits struct {
limiter *ratelimit.Limiter
bytesCounted config.BytesCounted // SWW_WAF_BYTES_COUNTED
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (byteLimits) name() string { return "byte_limits" }
func (byteLimits) check(
func (b byteLimits) check(
_ context.Context,
rq *request,
now time.Time,
@@ -29,10 +37,10 @@ func (byteLimits) check(
return nil
}
counts, hit, over := rq.h.limiter.CountBytes(
clientGroup(rq.h.config, rq.client),
counts, hit, over := b.limiter.CountBytes(
clientGroup(b.ipv6GroupPrefix, rq.client),
now,
rq.countedBytes(),
rq.countedBytes(b.bytesCounted),
rq.bytesLimitPercent.percent,
)
rq.line.Counts.MinuteBytes = counts.MinuteBytes
@@ -43,7 +51,7 @@ func (byteLimits) check(
return nil
}
rq.banForLimit(now, hit, rq.out.status)
b.banMaker.banForLimit(rq, now, hit, rq.out.status)
return &verdict{figures: limitFigures(hit)}
}
+5 -6
View File
@@ -5,8 +5,6 @@ import (
"net/http"
"net/netip"
"slices"
"sneak.berlin/go/smallwebwaf/internal/config"
)
// peerAddress is the address of the request's TCP peer.
@@ -45,12 +43,13 @@ func scheme(r *http.Request, isQuestion bool) string {
// clientGroup is the client a request is counted toward: its IPv4
// address, or its IPv6 group, the netblock its IPv6 address is in of the
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since
// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as
// IPv4.
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
addr = addr.Unmap()
if addr.Is6() {
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked()
}
return netip.PrefixFrom(addr, addr.BitLen())
+30 -18
View File
@@ -10,6 +10,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -22,11 +23,22 @@ import (
// exploitsignature, raises the core_rule_set_match alert, and in block mode
// refuses the request with 403, which is an offence its client's history
// counts. Its figures are the score and the threshold.
type coreRuleSet struct{}
type coreRuleSet struct {
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
// off.
inspector CoreRuleSetInspector
metrics *metrics.Metrics
alerts *alerts.Queue
mode config.CoreRuleSetMode // SWW_WAF_CORE_RULE_SET_MODE
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
observe bool // SWW_WAF_MODE is observe
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (coreRuleSet) name() string { return "core_rule_set" }
func (coreRuleSet) check(
func (c coreRuleSet) check(
_ context.Context,
rq *request,
_ time.Time,
@@ -36,7 +48,6 @@ func (coreRuleSet) check(
return nil
}
cfg := rq.h.config
result := findings.inspection
rq.requestBytes = int64(len(findings.read))
@@ -47,18 +58,18 @@ func (coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
c.metrics.CoreRuleSetMatched(string(c.mode), id)
}
threshold := cfg.CoreRuleSetScoreThreshold
threshold := c.scoreThreshold
if threshold == 0 || result.Score < threshold {
return nil
}
rq.addTag(config.ExploitSignatureTag)
rq.alertCoreRuleSetMatch(result)
c.alertCoreRuleSetMatch(rq, result)
if !rq.refusedByCoreRuleSet(result) {
if !c.refusedByCoreRuleSet(result) {
return nil
}
@@ -76,20 +87,21 @@ func (coreRuleSet) check(
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
// Rule Set found result of: whether the score is at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
cfg := rq.h.config
threshold := cfg.CoreRuleSetScoreThreshold
func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool {
threshold := c.scoreThreshold
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
return c.mode == config.CoreRuleSetModeBlock &&
threshold != 0 && result.Score >= threshold
}
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
// which the Core Rule Set scored at result, at or over
// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the
// Core Rule Set scored at result, at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
// score, the method and the path with the query, and, for a request that is
// not refused for it, the mode: detect, or observe in observe mode.
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
func (c coreRuleSet) alertCoreRuleSetMatch(
rq *request, result coreruleset.Result,
) {
detail := map[string]any{
"rule_ids": result.RuleIDs,
"score": result.Score,
@@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
}
switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
case c.mode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect)
case rq.h.config.Observe:
case c.observe:
detail["mode"] = "observe"
}
rq.h.alerts.Raise(alerts.Alert{
c.alerts.Raise(alerts.Alert{
Event: alerts.EventCoreRuleSetMatch,
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client),
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+5 -3
View File
@@ -14,16 +14,18 @@ import (
// does not hold the client's country as it was looked up. A client without a
// country, or whose country cannot be found, is refused. Its figures are the
// country and the setting that refused it.
type countries struct{}
type countries struct {
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
}
func (countries) name() string { return "countries" }
func (countries) check(
func (c countries) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
allowed := rq.h.config.ExclusivelyAllowedCountries
allowed := c.allowed
country := rq.line.Country
if len(allowed) == 0 || slices.Contains(allowed, country) {
+10 -3
View File
@@ -4,6 +4,7 @@ package waf
import (
"context"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -11,12 +12,18 @@ import (
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
// the setting the client is in.
type denyNets struct{}
type denyNets struct {
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
}
func (denyNets) name() string { return "deny_nets" }
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
for _, netblock := range rq.h.config.DenyNets {
func (d denyNets) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, netblock := range d.netblocks {
if netblock.Contains(rq.client) {
return &verdict{
action: requestlog.ActionClientDenied,
+47
View File
@@ -0,0 +1,47 @@
package waf
import (
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
t.Parallel()
heuristic := denyNets{
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
}
// The request has no handler: the heuristic reads only what it holds.
inside := &request{
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
}
v := heuristic.check(t.Context(), inside, time.Time{})
if v == nil {
t.Fatal("a client inside the netblock was let through")
}
if v.action != requestlog.ActionClientDenied {
t.Errorf(
"action = %q, want %q",
v.action,
requestlog.ActionClientDenied,
)
}
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
}
outside := &request{
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
}
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
t.Errorf("a client outside the netblock was refused: %+v", v)
}
}
+14 -6
View File
@@ -9,6 +9,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -19,21 +20,28 @@ import (
// being log, it does nothing more. A zone without a verdict on the client is
// asked about it in the background, and the request does not wait for the
// answer. Its figure is the zones that list the client, their keys masked.
type dnsbl struct{}
type dnsbl struct {
dnsbl *reputation.DNSBL
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
hitNoter reputationHitNoter
}
func (dnsbl) name() string { return "dnsbl" }
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
func (d dnsbl) check(
ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
rq.dnsblListed = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a DNSBL zone")
d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone")
for _, zone := range listedBy {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
}
if !rq.dnsblListed ||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if !rq.dnsblListed || d.action != config.ReputationActionDeny {
return nil
}
+5 -72
View File
@@ -5,9 +5,10 @@ import (
"time"
)
// To add a heuristic, write one file with a type that implements heuristic,
// and add one line for it to one of the lists below. Its settings go in
// config.WAF, and the comment at the top of its file names them.
// To add a heuristic, write one file with a struct that implements heuristic
// and holds the settings and parts it reads. Add one line for it to one of the
// lists that newHandler builds, which hands it those settings and parts. Its
// settings go in config.WAF, and the comment at the top of its file names them.
// heuristic is one of the checks that decide what waf does with a
// request.
@@ -36,78 +37,10 @@ const (
figureNetblock = "netblock"
)
// The lists of heuristics, each in the order its heuristics run. A client in
// SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number and
// country are looked up, so that a client they refuse is not looked up.
// header_limits comes first: it only measures and counts, and a request it
// refuses causes no other work.
func beforeLookup() []heuristic {
return []heuristic{
headerLimits{},
tagLists{},
routeWatchTags{},
denyNets{},
activeBan{},
}
}
// afterLookup are the heuristics that run once the lookup has been made.
func afterLookup() []heuristic {
return []heuristic{
countries{},
blocklists{},
dnsbl{},
abuseIPDB{},
tagDeny{},
rateLimits{},
}
}
// besideLookup are the heuristics that only read the request. Their work runs
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
// of the request. Their check then applies what that work found, and runs only
// once waf takes it.
func besideLookup() []heuristic {
return []heuristic{
trapPaths{},
ruleFiles{},
coreRuleSet{},
}
}
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second time,
// on the tag rule_files and core_rule_set can add.
func afterBesideLookup() []heuristic {
return []heuristic{
tagDeny{},
}
}
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client, and do
// not change what the request got: their verdict's action is not used.
func afterAnswer() []heuristic {
return []heuristic{
refusalLimit{},
}
}
// onReport are the heuristics that run on an allowed request when rproxy's
// report of it comes, before its report line is written. Like afterAnswer, they
// only ban.
func onReport() []heuristic {
return []heuristic{
byteLimits{},
}
}
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
rq.runBanningHeuristics(ctx, rq.h.afterAnswer)
}
// runHeuristics runs list, one of those checkClient runs, on the request at now,
+5 -3
View File
@@ -56,7 +56,7 @@ func (rq *request) startLookUp(
return result
}
client := clientGroup(rq.h.config, rq.client)
client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile {
result <- lookupResult{
@@ -143,8 +143,10 @@ func AddLookup(
) {
limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country)
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country)
netblock := banNetblock(
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(),
)
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
}
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
+168 -31
View File
@@ -133,15 +133,19 @@ type Server struct {
}
// newHandler returns the handler with what every request shares, the parts
// in params among it.
// in params among it. It builds the heuristics, each with the settings and
// parts it reads, and the lists they run in.
//
//nolint:funlen // a few lines for each heuristic, a list that grows with them
func newHandler(params Params) *handler {
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil {
panic(err) // New fails only for a size below one
}
return &handler{
config: params.Config,
cfg := params.Config
h := &handler{
config: cfg,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
now: params.Now,
@@ -152,47 +156,142 @@ func newHandler(params Params) *handler {
geojs: params.GeoJS,
anomalies: params.Anomalies,
lookupFile: params.LookupFile,
lists: params.Lists,
dnsbl: params.DNSBL,
abuseIPDB: params.AbuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders ||
params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config),
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
cfg.AddLookupHeaders ||
cfg.TagsNeedLookup() ||
biasedThresholdsSet(cfg),
besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit,
besideLookupTimeLimit,
),
}
sharedBanMaker := banMaker{
ledger: params.Ledger,
limiter: params.Limiter,
metrics: params.Metrics,
alerts: params.Alerts,
observe: cfg.Observe,
banResponse: cfg.BanResponse,
banScopeV4Prefix: cfg.BanScopeV4Prefix,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
hitNoter := reputationHitNoter{
metrics: params.Metrics,
alerts: params.Alerts,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.blocklists = blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
}
h.rateLimits = rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
metrics: params.Metrics,
banMaker: sharedBanMaker,
}
h.coreRuleSet = coreRuleSet{
inspector: params.CoreRuleSet,
metrics: params.Metrics,
alerts: params.Alerts,
mode: cfg.CoreRuleSetMode,
exemptPaths: cfg.CoreRuleSetExemptPaths,
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
observe: cfg.Observe,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.beforeLookup = []heuristic{
headerLimits{},
tagLists{lists: params.Lists},
routeWatchTags{
routeWatch: params.RouteWatch,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
},
denyNets{netblocks: cfg.DenyNets},
activeBan{banMaker: sharedBanMaker},
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
h.blocklists,
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
hitNoter: hitNoter,
},
abuseIPDB{
abuseIPDB: params.AbuseIPDB,
limiter: params.Limiter,
keySet: cfg.AbuseIPDBKey != "",
minScore: cfg.AbuseIPDBMinScore,
action: cfg.DNSBLAbuseIPDBAction,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
h.rateLimits,
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
h.afterAnswer = []heuristic{
refusalLimit{
limiter: params.Limiter,
perMinute: cfg.RefusalsLimitPerMinute,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.onReport = []heuristic{
byteLimits{
limiter: params.Limiter,
bytesCounted: cfg.BytesCounted,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
return h
}
// handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request.
type handler struct {
config *config.WAF
requestLog io.Writer
processLog *slog.Logger
now func() time.Time
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
lists *reputation.Lists
dnsbl *reputation.DNSBL
abuseIPDB *reputation.AbuseIPDB
routeWatch *routewatch.Client
rules *rules.Files
coreRuleSet CoreRuleSetInspector
alerts *alerts.Queue
config *config.WAF
requestLog io.Writer
processLog *slog.Logger
now func() time.Time
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
// waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on.
waitForGeoJS bool
@@ -202,4 +301,42 @@ type handler struct {
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number
// and country are looked up, so that a client they refuse is not looked
// up. header_limits comes first: it only measures and counts, and a
// request it refuses causes no other work.
beforeLookup []heuristic
// afterLookup are the heuristics that run once the lookup has been made.
afterLookup []heuristic
// besideLookup are the heuristics that only read the request. Their work
// runs beside the lookup and afterLookup, in findBesideLookup, which
// changes nothing of the request. Their check then applies what that work
// found, and runs only once waf takes it.
besideLookup []heuristic
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second
// time, on the tag rule_files and core_rule_set can add.
afterBesideLookup []heuristic
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client,
// and do not change what the request got: their verdict's action is not
// used.
afterAnswer []heuristic
// onReport are the heuristics that run on an allowed request when
// rproxy's report of it comes, before its report line is written. Like
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of the lists that other steps of a request use too:
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
blocklists blocklists
rateLimits rateLimits
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
}
+34 -22
View File
@@ -6,8 +6,10 @@ package waf
import (
"context"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -18,28 +20,37 @@ import (
// refused, and bans the client. A request exempt from the rate limits is exempt
// from the byte limits too. Its figures are the window, the count and the
// limit.
type rateLimits struct{}
type rateLimits struct {
limiter *ratelimit.Limiter
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
biasedThresholds biasedThresholds
banMaker banMaker
}
func (rateLimits) name() string { return "rate_limits" }
func (rateLimits) check(
func (r rateLimits) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
rq.counted = rq.countedByRateLimits()
rq.counted = r.countedByRateLimits(rq)
if !rq.counted {
return nil
}
rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages()
rq.rateLimitPercent, rq.bytesLimitPercent =
r.biasedThresholds.limitPercentages(rq)
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
rq.rateLimitPercent.logged()
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged()
counts, hit, over := rq.h.limiter.Count(
clientGroup(rq.h.config, rq.client),
counts, hit, over := r.limiter.Count(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
rq.rateLimitPercent.percent,
)
@@ -49,28 +60,29 @@ func (rateLimits) check(
return nil
}
rq.banForLimit(now, hit, rq.h.config.BanResponse)
r.banMaker.banForLimit(rq, now, hit, r.banResponse)
return &verdict{
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
}
}
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
// request at now, whatever its client's limit percentages: the rate limits
// count it, and one more request from its client takes it over a whole rate
// limit. It counts nothing.
func (rq *request) overAWholeRateLimit(now time.Time) bool {
return rq.countedByRateLimits() &&
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
// at now, whatever its client's limit percentages: the rate limits count it,
// and one more request from its client takes it over a whole rate limit. It
// counts nothing.
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
return r.countedByRateLimits(rq) &&
r.limiter.OverARateLimit(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
)
}
// countedByRateLimits reports whether the rate limits count the request: its
// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
func (rq *request) countedByRateLimits() bool {
cfg := rq.h.config
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
// countedByRateLimits reports whether the rate limits count rq: its client is
// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
func (r rateLimits) countedByRateLimits(rq *request) bool {
return !isInside(rq.client, r.exemptNets) &&
!pathExempt(rq.in, r.exemptPaths)
}
+15 -9
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -18,17 +19,22 @@ import (
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
// heuristics skip, is not counted, and nothing is while the limit is off.
// Its figures are the window, the count and the limit.
type refusalLimit struct{}
type refusalLimit struct {
limiter *ratelimit.Limiter
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (refusalLimit) name() string { return "refusal_limit" }
func (refusalLimit) check(
func (r refusalLimit) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
cfg := rq.h.config
if cfg.RefusalsLimitPerMinute == 0 {
if r.perMinute == 0 {
return nil
}
@@ -36,10 +42,10 @@ func (refusalLimit) check(
return nil
}
hit, over := rq.h.limiter.CountRefusal(
clientGroup(rq.h.config, rq.client),
hit, over := r.limiter.CountRefusal(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
cfg.RefusalsLimitPerMinute,
r.perMinute,
)
if !over {
return nil
@@ -54,10 +60,10 @@ func (refusalLimit) check(
status = http.StatusForbidden
case requestlog.ActionTemporarilyBanned,
requestlog.ActionPermanentlyBanned:
status = cfg.BanResponse
status = r.banResponse
}
rq.banForLimit(now, hit, status)
r.banMaker.banForLimit(rq, now, hit, status)
return &verdict{figures: limitFigures(hit)}
}
+21 -8
View File
@@ -5,22 +5,35 @@ package waf
import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/metrics"
)
// reputationHitNoter notes the reputation sources that list a request's
// client, for the reputation heuristics.
type reputationHitNoter struct {
metrics *metrics.Metrics
alerts *alerts.Queue
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
// noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does,
// DNSBL zones, their keys masked, that list rq's client, as noteHit does,
// with reason.
func (rq *request) noteListed(sources []string, reason string) {
func (n reputationHitNoter) noteListed(
rq *request, sources []string, reason string,
) {
for _, source := range sources {
rq.noteHit(bans.ReputationHit{Source: source}, reason)
n.noteHit(rq, bans.ReputationHit{Source: source}, reason)
}
}
// noteHit adds hit's source, which lists the client, to the log line's
// noteHit adds hit's source, which lists rq's client, to the log line's
// reputation, and hit to the notes of a ban the request makes, counts the
// source in the metrics, and raises a reputation_hit alert with reason,
// whose detail gives hit's source and score.
func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
func (n reputationHitNoter) noteHit(
rq *request, hit bans.ReputationHit, reason string,
) {
detail := map[string]any{"source": hit.Source}
if hit.Score != nil {
detail["score"] = *hit.Score
@@ -28,12 +41,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
rq.reputation = append(rq.reputation, hit)
rq.h.metrics.ReputationHit(hit.Source)
rq.h.alerts.Raise(alerts.Alert{
n.metrics.ReputationHit(hit.Source)
n.alerts.Raise(alerts.Alert{
Event: alerts.EventReputationHit,
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client),
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+13 -8
View File
@@ -122,8 +122,11 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion),
PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: clientGroup(h.config, client).String(),
ContentType: r.Header.Get("Content-Type"),
ClientGroup: clientGroup(
h.config.IPv6GroupPrefix,
client,
).String(),
ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders(
r,
h.config.LogRequestHeaders,
@@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
now := rq.h.now()
v := rq.runHeuristics(ctx, beforeLookup(), now)
v := rq.runHeuristics(ctx, rq.h.beforeLookup, now)
if v != nil {
return v
}
@@ -265,7 +268,7 @@ func (rq *request) runAfterLookup(
// again: a request counted at the time it arrived could be counted
// after a later one of its client's, which the rate limits would take
// for the clock set back.
return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now())
}
// pathExempt reports whether r is exempt under prefixes,
@@ -365,7 +368,7 @@ func (rq *request) addToHistory() {
}
rq.h.limiter.AddToHistory(
clientGroup(rq.h.config, rq.client),
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
rq.h.now(),
request,
)
@@ -394,11 +397,11 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site,
Client: rq.client,
ClientGroup: clientGroup(rq.h.config, rq.client),
ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
ASN: answer.ASN,
ASName: answer.ASName,
Country: answer.Country,
Bytes: rq.countedBytes(),
Bytes: rq.countedBytes(rq.h.config.BytesCounted),
})
}
@@ -425,7 +428,9 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return rq.lookupAnswer, true
}
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
return rq.h.geojs.Kept(
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
)
}
// responseWriter is waf's answer to a request. It notes the status and
+10 -5
View File
@@ -8,6 +8,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
)
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
@@ -17,22 +18,26 @@ import (
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
// asked for by the request's address, and a route without an origin AS gives only
// the prefix= tag.
type routeWatchTags struct{}
type routeWatchTags struct {
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
routeWatch *routewatch.Client
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (routeWatchTags) name() string { return "routewatch_tags" }
func (routeWatchTags) check(
func (r routeWatchTags) check(
ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
if rq.h.routeWatch == nil {
if r.routeWatch == nil {
return nil
}
route, missing := rq.h.routeWatch.Route(
route, missing := r.routeWatch.Route(
ctx,
clientGroup(rq.h.config, rq.client),
clientGroup(r.ipv6GroupPrefix, rq.client),
rq.client,
)
if missing != "" {
+10 -4
View File
@@ -9,6 +9,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
)
@@ -22,11 +23,15 @@ import (
// mode raises the alert for the ban
// it would have made. Either rule's match is noted as an offence, for the
// client's history. Its figures are the rule's id and action.
type ruleFiles struct{}
type ruleFiles struct {
rules *rules.Files
metrics *metrics.Metrics
banMaker banMaker
}
func (ruleFiles) name() string { return "rule_files" }
func (ruleFiles) check(
func (r ruleFiles) check(
_ context.Context,
rq *request,
now time.Time,
@@ -35,7 +40,7 @@ func (ruleFiles) check(
for _, rule := range matched {
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
r.metrics.RuleMatched(rule.ID, rule.Action)
}
if len(matched) == 0 {
@@ -60,7 +65,8 @@ func (ruleFiles) check(
rq.attack = true
return &verdict{
action: rq.banForAttack(
action: r.banMaker.banForAttack(
rq,
now,
bans.Notes{RuleID: last.ID, Target: last.Target},
),
+9 -3
View File
@@ -12,13 +12,19 @@ import (
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
// the first such tag, in sorted order.
type tagDeny struct{}
type tagDeny struct {
tags []string // SWW_WAF_TAG_DENY
}
func (tagDeny) name() string { return "tag_deny" }
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
func (t tagDeny) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, tag := range rq.line.Tags {
if slices.Contains(rq.h.config.TagDeny, tag) {
if slices.Contains(t.tags, tag) {
return &verdict{
action: requestlog.ActionClientDenied,
figures: map[string]any{"tag": tag},
+11 -3
View File
@@ -6,17 +6,25 @@ package waf
import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
// holds the client, the tag being the list's name. It refuses nothing, and runs
// first so that every refusal's log line carries these tags.
type tagLists struct{}
type tagLists struct {
lists *reputation.Lists
}
func (tagLists) name() string { return "tag_lists" }
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
for _, name := range rq.h.lists.TagsOf(rq.client) {
func (t tagLists) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, name := range t.lists.TagsOf(rq.client) {
rq.addTag(name)
}
+15 -8
View File
@@ -16,11 +16,14 @@ import (
// it bans the client's netblock, or in observe mode raises the alert for the
// ban it would have made. The ban's notes give the path as sent, and so does
// its figure.
type trapPaths struct{}
type trapPaths struct {
paths []string // SWW_WAF_TRAP_PATHS
banMaker banMaker
}
func (trapPaths) name() string { return "trap_paths" }
func (trapPaths) check(
func (t trapPaths) check(
_ context.Context,
rq *request,
now time.Time,
@@ -33,18 +36,22 @@ func (trapPaths) check(
rq.attack = true
return &verdict{
action: rq.banForAttack(now, bans.Notes{TrapPath: path}),
action: t.banMaker.banForAttack(
rq,
now,
bans.Notes{TrapPath: path},
),
figures: map[string]any{"path": path},
}
}
// trapPath returns the request's path as sent, when its path as a path rule
// sees it, without the query, as sent or once percent-decoded, is in
// trapPath returns rq's path as sent, when its path as a path rule sees it,
// without the query, as sent or once percent-decoded, is in
// SWW_WAF_TRAP_PATHS, and "" otherwise.
func (rq *request) trapPath() string {
func (t trapPaths) trapPath(rq *request) string {
path := rules.Path(rq.in)
if !slices.Contains(rq.h.config.TrapPaths, path) &&
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
if !slices.Contains(t.paths, path) &&
!slices.Contains(t.paths, rules.DecodeOnce(path)) {
return ""
}
+9 -7
View File
@@ -123,10 +123,10 @@ func New(params Params) *Server {
GeoJS: h.geojs,
Anomalies: h.anomalies,
LookupFile: h.lookupFile,
Lists: h.lists,
DNSBL: h.dnsbl,
AbuseIPDB: h.abuseIPDB,
RouteWatch: h.routeWatch,
Lists: params.Lists,
DNSBL: params.DNSBL,
AbuseIPDB: params.AbuseIPDB,
RouteWatch: params.RouteWatch,
Metrics: h.metrics,
}
}
@@ -154,8 +154,10 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
return false
}
return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
rq.overAWholeRateLimit(now)
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
}
// answerQuestion runs the checks on the visitor's request a question
@@ -491,7 +493,7 @@ func (h *handler) finishWith(
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.runBanningHeuristics(ctx, onReport())
rq.runBanningHeuristics(ctx, rq.h.onReport)
rq.countAnomalies()
rq.addToHistory()