Compare commits

..
Author SHA1 Message Date
sneak ce87531fec waf: each heuristic holds its own settings and parts (closes #441)
check / check (push) Waiting to run
Every heuristic is a struct built once in newHandler with the settings
and collaborators it reads; no check reads the handler. The six lists
are handler fields built there. Code several heuristics share takes what
it needs as arguments or lives on a struct they hold: banMaker for bans
and their alerts, reputationHitNoter for reputation hits,
biasedThresholds for limit percentages. The handler keeps blocklists,
rateLimits, trapPaths, ruleFiles and coreRuleSet as fields too, since
refusedWhateverTheAnswer and findBesideLookup use them.

Model: opus-5-5
2026-11-19 16:46:56 +00:00
41 changed files with 980 additions and 888 deletions

No files matched your search

+4 -8
View File
@@ -54,11 +54,7 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
now := midnight()
scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made)
@@ -123,7 +119,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(limited, now, bans.Notes{})
// listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
@@ -148,7 +144,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
}
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(limited, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made)
@@ -158,7 +154,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
// The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes.
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
ledger.BanForAttack(attacker, now, bans.Notes{})
want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores {
+23 -64
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one.
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(second, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
}
// And the next drops that one.
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(third, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes.
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(other, second.Start, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
kept := first
kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams())
limit, _ := ledger.Ban(
limit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
)
byteLimit, _ := ledger.Ban(
byteLimit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.3/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
)
attack, _ := ledger.Ban(
attack, _ := ledger.BanForAttack(
netip.MustParsePrefix("203.0.113.2/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
bans.Notes{RuleID: "git-dir", Target: "path"},
)
for _, tc := range []struct{ got, want string }{
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses")
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
}
// The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf(
"the next ban for an attack ends at %s, want seven days on",
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
made, _ := ledger.Ban(
made, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams())
// An hour's ban for a broken limit.
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"),
+82 -54
View File
@@ -300,50 +300,81 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest
}
// Grounds are why a request makes a ban: its cause, CauseLimit or
// CauseAttack, and its notes.
type Grounds struct {
Cause string
Notes Notes
// BanForLimit bans netblock at now for a broken limit, with notes, and
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
// made within LimitBanRepeatWindow after the netblock's ban that ended
// last, other than one for a clear sign of attack or a lifted one, lasts
// repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
}
// Ban bans netblock at now on grounds, and returns the ban, and true. If
// a ban on netblock is still active, as when two of its requests break a
// limit at once, that ban is returned with false, and no other is made.
// The ledger fills in the notes' Refused and EarlierBans itself.
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
// WouldBanForLimit returns what BanForLimit would, without making the ban:
// what observe mode would have done.
func (l *Ledger) WouldBanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, true)
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
}
// WouldBan returns what Ban would, without making the ban: what observe
// mode would have done.
func (l *Ledger) WouldBan(
netblock netip.Prefix, now time.Time, grounds Grounds,
// BanForAttack bans netblock at now for a clear sign of attack, with
// notes, and returns the ban, and whether it made it, as BanForLimit
// does. A first ban lasts AttackBanDuration; once the netblock has had
// one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, false)
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
}
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as Ban would make it. It
// works out nothing else of the ban.
// or CauseAttack, made at now would be permanent, as BanForLimit or
// BanForAttack would make it. It works out nothing else of the ban.
func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause string,
) bool {
@@ -380,9 +411,10 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
// on netblock is active, since the admin asked for this one. The ledger
// fills in the notes' EarlierBans, and counts the ban among those made.
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// ban even while another on netblock is active, since the admin asked
// for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string,
) Ban {
@@ -727,13 +759,15 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
return forgot
}
// ban bans netblock at now on grounds, as Ban describes, and returns the
// ban, and whether it made it. Unless keep is true, the ban is not made,
// only returned: it is the ban that would have been made.
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
func (l *Ledger) ban(
netblock netip.Prefix,
now time.Time,
grounds Grounds,
cause, reason string,
notes Notes,
keep bool,
) (Ban, bool) {
l.mu.Lock()
@@ -742,8 +776,6 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active.
var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock)
if found {
active := activeBan(*bans, now)
@@ -759,19 +791,15 @@ func (l *Ledger) ban(
ban := Ban{
Netblock: netblock,
Start: now,
Cause: grounds.Cause,
Cause: cause,
Reason: reason,
Notes: notes,
}
switch grounds.Cause {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
if cause == CauseAttack {
ban.Expires = l.attackExpiry(held, now)
default:
panic("bans: a request's ban has the cause " + grounds.Cause)
} else {
ban.Expires = l.limitExpiry(held, now)
}
if !keep {
@@ -779,7 +807,7 @@ func (l *Ledger) ban(
}
l.add(ban)
l.made[grounds.Cause]++
l.made[cause]++
l.markChanged()
return ban, true
+51 -121
View File
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) ||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends.
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
}
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netblock,
first.Expires.Add(tc.gap),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if second.Expires.Sub(second.Start) != tc.want ||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params)
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight()
for i := range 14 {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Expires.After(ban.Start) {
t.Fatalf(
"ban %d starts at %s and ends at %s",
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
}
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
if !made {
t.Error("the first ban was not made")
}
again, made := ledger.Ban(
again, made := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) ||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
)
}
again, made = ledger.Ban(
again, made = ledger.BanForAttack(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) {
t.Errorf(
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
for range 3 {
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
got, banned, _ := ledger.Find(
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64")
now := midnight()
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.BanForLimit(b, now, bans.Notes{})
ledger.BanForLimit(c, now, bans.Notes{})
// A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's.
ledger.Check(a.Addr(), now)
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(d, now, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(a, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes
// first.
ledger.Check(d.Addr(), first.Expires)
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(b, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" ||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit.
third, _ := ledger.Ban(
third, _ := ledger.BanForAttack(
netblock,
second.Expires.Add(30*day),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// While the first ban lasts, none would be made.
during, would := ledger.WouldBan(
during, would := ledger.WouldBanForAttack(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if would || !reflect.DeepEqual(during, first) {
t.Errorf(
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made.
limitGrounds := bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
}
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
bans.Notes{RuleID: "git-dir"})
limit, wouldLimit := ledger.WouldBanForLimit(
netblock,
first.Expires,
limitGrounds,
limitNotes,
)
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false)
// The ban made is the one that would have been.
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit)
}
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1)
}
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
now = ban.Expires
}
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
}
// Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent")
}
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit {
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
}
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
bans.Notes{Request: request},
)
cut := long[:256]
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock.
ledger.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
ledger.BanForLimit(netblock, midnight(), bans.Notes{
ASN: "AS64497", ASName: "Other Net", Country: "FR",
})
ledger.Ban(
netblock,
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
ledger.BanForLimit(other, midnight(), bans.Notes{})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+21 -36
View File
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false)
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write.
ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false)
// Two bans before the value is read leave one.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.11/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
wantChanged(t, ledger, true)
wantChanged(t, ledger, false)
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
v6,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
ledger.BanForLimit(high, midnight(), bans.Notes{})
ledger.BanForLimit(low, midnight(), bans.Notes{})
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
snapshot := ledger.Snapshot()
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
// Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse")
}
again, _ := after.Ban(
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(),
},
})
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
for client, want := range map[string]bool{
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
}
// A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
nineHours.Expires.Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
}
// The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.8/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
want := []bans.Ban{first, second, kept}
+32 -93
View File
@@ -75,7 +75,7 @@ type WAF struct {
BytesLimitPerMinute int64
BytesLimitPerHour int64
BytesLimitPerDay int64
BytesCounted BytesCounted
BytesCounted string
// LookupSource is where each client's AS number and country are looked
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
@@ -86,7 +86,7 @@ type WAF struct {
// AddLookupHeaders is true when the app is passed the client's AS number
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
// LookupSource is off.
LookupSource LookupSource
LookupSource string
LookupDBPath string
ASNAndCountryLookupTimeout time.Duration
AddLookupHeaders bool
@@ -131,7 +131,7 @@ type WAF struct {
// BlocklistLimitPercent is the percentage of every limit it gets.
BlocklistURLs []string
BlocklistRefresh time.Duration
BlocklistAction ReputationAction
BlocklistAction string
BlocklistLimitPercent int64
// DNSBLZones are the DNSBL zones clients are asked about
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
@@ -154,7 +154,7 @@ type WAF struct {
AbuseIPDBKey string
AbuseIPDBMinScore int64
AbuseIPDBDailyBudget int
DNSBLAbuseIPDBAction ReputationAction
DNSBLAbuseIPDBAction string
DNSBLAbuseIPDBLimitPercent int64
IPReputationCacheTTL time.Duration
DNSBLAbuseIPDBTimeout time.Duration
@@ -237,7 +237,7 @@ type WAF struct {
// reads none.
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
CoreRuleSetMode CoreRuleSetMode
CoreRuleSetMode string
CoreRuleSetParanoiaLevel int
CoreRuleSetScoreThreshold int
CoreRuleSetDisabledRules []int
@@ -316,66 +316,20 @@ type WAF struct {
settings []slog.Attr
}
// BytesCounted is which body bytes count toward the byte limits and the
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
type BytesCounted string
// The values of SWW_WAF_BYTES_COUNTED.
const (
// BytesCountedResponse counts the response's body bytes.
BytesCountedResponse BytesCounted = "response"
// BytesCountedRequest counts the request's body bytes.
BytesCountedRequest BytesCounted = "request"
// BytesCountedBoth counts the body bytes of both.
BytesCountedBoth BytesCounted = "both"
)
// LookupSource is where each client's AS number and country are looked up,
// a value of SWW_WAF_LOOKUP_SOURCE.
type LookupSource string
// The values of SWW_WAF_LOOKUP_SOURCE.
const (
// LookupSourceGeoJS asks GeoJS.
LookupSourceGeoJS LookupSource = "geojs"
// LookupSourceFile looks clients up in the lookup database, the file
// SWW_WAF_LOOKUP_DB_PATH names.
LookupSourceFile LookupSource = "file"
// LookupSourceOff looks no client up.
LookupSourceOff LookupSource = "off"
)
// ReputationAction is what is done with a client a reputation source
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
type ReputationAction string
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
const (
// ReputationActionDeny refuses the client's requests.
ReputationActionDeny ReputationAction = "deny"
// ReputationActionLimit gives the client a percentage of every limit,
// written limit:<percent> in the setting.
ReputationActionLimit ReputationAction = "limit"
// ReputationActionLog only notes the hit in the log line and raises an
// alert.
ReputationActionLog ReputationAction = "log"
)
// CoreRuleSetMode is what the Core Rule Set does, a value of
// SWW_WAF_CORE_RULE_SET_MODE.
type CoreRuleSetMode string
// The values of SWW_WAF_CORE_RULE_SET_MODE.
const (
// CoreRuleSetModeOff runs no request through the Core Rule Set.
CoreRuleSetModeOff CoreRuleSetMode = "off"
CoreRuleSetModeOff = off
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
CoreRuleSetModeDetect = "detect"
// CoreRuleSetModeBlock refuses a match with 403.
CoreRuleSetModeBlock CoreRuleSetMode = "block"
CoreRuleSetModeBlock = "block"
)
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
const fileSource = "file"
const (
day = 24 * time.Hour
ipv4Bits = 32
@@ -604,13 +558,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
),
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
BytesCounted: env.bytesCounted(
"SWW_WAF_BYTES_COUNTED",
BytesCountedBoth,
),
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
LookupSource: env.lookupSource(
"SWW_WAF_LOOKUP_SOURCE",
LookupSourceGeoJS,
"geojs",
),
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
ASNAndCountryLookupTimeout: env.milliseconds(
@@ -807,9 +758,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.InstanceName, cfg.LogRemoteURL != nil)
// The app gets the headers by default, unless no client is looked up.
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
strconv.FormatBool(cfg.LookupSource != off))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
"SWW_WAF_BLOCKLIST_ACTION", "deny")
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
@@ -907,12 +858,9 @@ func (e *environment) count(name, defaultValue string) int64 {
// bytesCounted reads the setting that is which body bytes count toward the
// byte limits and the byte anomaly thresholds: response, request or both.
func (e *environment) bytesCounted(
name string, defaultValue BytesCounted,
) BytesCounted {
value := BytesCounted(e.value(name, string(defaultValue)))
if value != BytesCountedResponse && value != BytesCountedRequest &&
value != BytesCountedBoth {
func (e *environment) bytesCounted(name, defaultValue string) string {
value := e.value(name, defaultValue)
if value != "response" && value != "request" && value != "both" {
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
}
@@ -957,10 +905,8 @@ func (e *environment) trapPaths(name string) []string {
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
// detect or block.
func (e *environment) coreRuleSetMode(
name string, defaultValue CoreRuleSetMode,
) CoreRuleSetMode {
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
mode := e.value(name, defaultValue)
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
mode != CoreRuleSetModeBlock {
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
@@ -1110,24 +1056,20 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
// action reads a setting that is what is done with a client a blocklist
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
// as limit and the percentage.
func (e *environment) action(
name, defaultValue string,
) (ReputationAction, int64) {
value := ReputationAction(e.value(name, defaultValue))
if value == ReputationActionDeny || value == ReputationActionLog {
func (e *environment) action(name, defaultValue string) (string, int64) {
value := e.value(name, defaultValue)
if value == "deny" || value == "log" {
return value, 0
}
percentText, isLimit := strings.CutPrefix(
string(value), string(ReputationActionLimit)+":",
)
percentText, isLimit := strings.CutPrefix(value, "limit:")
percent, err := ParsePercent(percentText)
if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
}
return ReputationActionLimit, percent
return "limit", percent
}
// zones reads the setting that is the list of DNSBL zones. It is empty by
@@ -1178,12 +1120,9 @@ func (e *environment) routeWatchURL(name string) string {
// lookupSource reads the setting that is where clients are looked up:
// geojs, file, or off.
func (e *environment) lookupSource(
name string, defaultValue LookupSource,
) LookupSource {
source := LookupSource(e.value(name, string(defaultValue)))
if source != LookupSourceGeoJS && source != LookupSourceFile &&
source != LookupSourceOff {
func (e *environment) lookupSource(name, defaultValue string) string {
source := e.value(name, defaultValue)
if source != "geojs" && source != fileSource && source != off {
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
}
@@ -1195,12 +1134,12 @@ func (e *environment) lookupSource(
// one source at a time.
func (e *environment) checkLookupDBPath(cfg *WAF) {
switch {
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
errNeedsDBPath,
))
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
cfg.LookupSource,
@@ -1225,7 +1164,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
}
}
if cfg.LookupSource != LookupSourceOff {
if cfg.LookupSource != off {
return
}
+5 -6
View File
@@ -614,7 +614,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
},
{
environment{
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
coreRuleSetMode: config.CoreRuleSetModeDetect,
coreRuleSetParanoiaLevel: "4",
coreRuleSetScoreThreshold: "10",
coreRuleSetDisabledRules: "942100, 920350",
@@ -2033,9 +2033,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
}
for _, tc := range []struct {
value string
action config.ReputationAction
percent int64
value, action string
percent int64
}{
{actionDeny, actionDeny, 0},
{actionLog, actionLog, 0},
@@ -2168,7 +2167,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
// want are the zones, resolver, action and percent config.WAF gives.
want []string
wantResolver string
wantAction config.ReputationAction
wantAction string
wantPercent int64
}{
{
@@ -3050,7 +3049,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
logLevel: "info",
ruleFilesDir: "/etc/sww/rules.d",
ruleFilesEnabled: "true",
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
coreRuleSetMode: config.CoreRuleSetModeBlock,
coreRuleSetParanoiaLevel: "1",
coreRuleSetScoreThreshold: "5",
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
+18 -30
View File
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it.
first, _ := params.Ledger.Ban(
first, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5 * time.Second)
second, _ := params.Ledger.Ban(
second, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run)
// A ban, and bans.json written with it.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay)
synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and
// another added.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
err = files.WriteAll()
@@ -2135,11 +2135,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned {
@@ -2514,15 +2510,11 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.Ban(client, now, bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
params.Ledger.BanForLimit(client, now, bans.Notes{
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
})
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Notes{RuleID: "env-file", Target: "path"})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2708,11 +2700,7 @@ func wantLiftedBanKept(
}
// Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.Ban(
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf(
"the next ban lasts %s, want 1h",
+1 -3
View File
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
status := run(t.Context(), map[string]string{
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
"SWW_WAF_CORE_RULE_SET_MODE": string(
config.CoreRuleSetModeOff,
),
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
}, out)
if status != 1 {
+4 -29
View File
@@ -2,7 +2,6 @@ package sww
import (
"context"
"fmt"
"io"
"log/slog"
"net"
@@ -176,35 +175,11 @@ func newRuleFiles(
return ruleFiles, nil
}
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
crs, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
crs, err := waf.NewCoreRuleSet(cfg)
if err != nil {
return nil, startError{
"cannot use the Core Rule Set",
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
}
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
return nil, startError{"cannot use the Core Rule Set", err}
}
return crs, nil
@@ -217,7 +192,7 @@ func newLookupFile(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*lookup.File, error) {
if cfg.LookupSource != config.LookupSourceFile {
if cfg.LookupSource != "file" {
return nil, nil //nolint:nilnil // no lookup database, and no error
}
+20 -9
View File
@@ -22,31 +22,42 @@ import (
// an offence, and the request does not wait for the answer. The score is then
// used for each address of the client. Its figures are the score and
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
type abuseIPDB struct{}
type abuseIPDB struct {
abuseIPDB *reputation.AbuseIPDB
// limiter is the table of clients the rate limits count, whose histories
// say which clients are offenders.
limiter *ratelimit.Limiter
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
action string // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
hitNoter reputationHitNoter
}
func (abuseIPDB) name() string { return "abuseipdb" }
func (abuseIPDB) check(
func (a abuseIPDB) check(
ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
if rq.h.config.AbuseIPDBKey == "" {
if !a.keySet {
return nil
}
client := clientGroup(rq.h.config, rq.client)
held, _ := rq.h.limiter.Client(client)
client := clientGroup(a.ipv6GroupPrefix, rq.client)
held, _ := a.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{}
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
if !hit {
return nil
}
rq.abuseIPDBHit = true
rq.addTag(config.AbuseIPDBTag)
rq.noteHit(
a.hitNoter.noteHit(
rq,
bans.ReputationHit{
Source: reputation.AbuseIPDBSource,
Score: &score,
@@ -54,14 +65,14 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
)
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if a.action != deny {
return nil
}
return &verdict{
action: requestlog.ActionClientDenied,
figures: map[string]any{
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
figureScore: score, "min_score": a.minScore,
},
}
}
+3 -1
View File
@@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist(
scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed {
netblock := banNetblock(cfg, client.Addr)
netblock := banNetblock(
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
)
scores[netblock] = max(scores[netblock], client.Score)
}
+8 -2
View File
@@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
}
return banNetblock(h.config, addr), nil
return banNetblock(
h.config.BanScopeV4Prefix,
h.config.IPv6GroupPrefix,
addr,
), nil
}
// expiry returns when a ban made at now for duration ends: duration
@@ -295,7 +299,9 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
}
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
client, seen := rq.h.limiter.Client(
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
)
if seen {
answer.Client = &client
}
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"",
map[string]string{metricsToken: token},
)
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(otherClient+"/32"),
clk.Now(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
before := server.Ledger.Snapshot()
+5 -3
View File
@@ -130,9 +130,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make
// permanent.
group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
})
attackBan, _ := server.Ledger.BanForAttack(
group,
start,
bans.Notes{RuleID: "probe"},
)
// The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack.
+2 -2
View File
@@ -686,10 +686,10 @@ func askDirectly(
// banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(client+"/32"),
now,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
}
+132 -122
View File
@@ -1,7 +1,7 @@
package waf
// The heuristic active_ban, and the code that makes bans, which several
// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
// The heuristic active_ban, and banMaker, which makes the bans of several
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
import (
"context"
@@ -10,7 +10,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -25,18 +25,22 @@ func (rq *request) banResponse(action string) *refusal {
// in, in force at now. It notes for the log line when that ban ends. A request
// that makes the ban permanent, or in observe mode would have, raises the alert
// for it. Its figures are the ban's netblock and when it ends.
type activeBan struct{}
type activeBan struct {
banMaker banMaker
}
func (activeBan) name() string { return "active_ban" }
func (activeBan) check(
func (a activeBan) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
check := rq.h.ledger.Check
if rq.h.config.Observe {
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
ledger := a.banMaker.ledger
check := ledger.Check
if a.banMaker.observe {
check = ledger.Find // the ban refuses nothing, and stays as it is
}
ban, banned, madePermanent := check(rq.client, now)
@@ -46,7 +50,7 @@ func (activeBan) check(
if madePermanent {
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
rq.alertBan(ban)
a.banMaker.alertBan(rq, ban)
}
if !banned {
@@ -81,31 +85,55 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
// countedBytes returns the request's bytes, once it has ended, as the
// byte limits and the anomaly thresholds count them: the response's body
// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says.
func (rq *request) countedBytes() int64 {
// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED,
// says.
func (rq *request) countedBytes(bytesCounted string) int64 {
response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted {
case config.BytesCountedResponse:
switch bytesCounted {
case "response":
return response
case config.BytesCountedRequest:
case "request":
return request
case config.BytesCountedBoth:
default: // both
return response + request
}
return response + request
}
// banForLimit bans the client's netblock at now for a broken limit, the
// banMaker makes the bans of the heuristics that ban a client, in the ban
// ledger, and raises the alert for each. In observe mode it makes no ban, and
// raises the alert for the ban it would have made.
type banMaker struct {
ledger *bans.Ledger
// limiter is the table of clients the rate limits count. A ban for a limit
// sets the client's counters back to zero, and a ban's notes give how many
// requests its netblock has sent.
limiter *ratelimit.Limiter
metrics *metrics.Metrics
alerts *alerts.Queue
// observe is true in observe mode (SWW_WAF_MODE).
observe bool
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
// clear sign of attack give.
banResponse int
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
banScopeV4Prefix int
ipv6GroupPrefix int
}
// banForLimit bans the netblock of rq's client at now for a broken limit, the
// one hit names, notes the offence for the log line and counts the hit in
// the metrics. status is what the client was sent, or is sent:
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
// one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what
// happens instead.
func (rq *request) banForLimit(
// the client's counters back to zero. In observe mode it makes no ban and
// sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (b banMaker) banForLimit(
rq *request,
now time.Time,
hit ratelimit.Hit,
status int,
@@ -120,13 +148,25 @@ func (rq *request) banForLimit(
}
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
b.metrics.LimitHit(hit)
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: b.netblockRequests(netblock),
}
switch hit.Kind {
@@ -136,118 +176,86 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
}
rq.banClient(
now,
status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
)
}
// banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) string {
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
if b.observe {
ban, wouldBan := b.ledger.WouldBanForLimit(netblock, now, notes)
if wouldBan {
b.alertBan(rq, ban)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
return
}
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
ban, made := b.ledger.BanForLimit(netblock, now, notes)
b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
b.alertBan(rq, ban)
}
}
// banForAttack bans the netblock of rq's client at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It fills in the rest of the notes. In observe mode it makes
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (b banMaker) banForAttack(
rq *request, now time.Time, notes bans.Notes,
) string {
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
b.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, b.banResponse)
notes.Requests = b.netblockRequests(netblock)
if b.observe {
ban, wouldBan := b.ledger.WouldBanForAttack(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
b.alertBan(rq, ban)
}
return banAction(ban.Permanent())
}
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
ban, made := b.ledger.BanForAttack(netblock, now, notes)
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
b.alertBan(rq, ban)
}
return banAction(ban.Permanent())
}
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
// netblock's requests, which can mean going through every client.
func (rq *request) wouldAlertBan(
func (b banMaker) wouldAlertBan(
netblock netip.Prefix, now time.Time, cause string,
) bool {
event := alerts.EventBan
if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
if b.ledger.WouldBePermanent(netblock, now, cause) {
event = alerts.EventPermanentBan
}
return rq.h.alerts.WouldSend(event, netblock)
return b.alerts.WouldSend(event, netblock)
}
// alertBan raises the alert for ban, which the request made, or made
// permanent: permanent_ban for a permanent ban, ban for another. Its
// detail gives the ban's cause, when it ends, and its notes, and in
// observe mode, where ban is the ban that would have been made, or made
// permanent, mode, observe.
func (rq *request) alertBan(ban bans.Ban) {
// alertBan raises the alert for ban, which rq made, or made permanent:
// permanent_ban for a permanent ban, ban for another. Its detail gives the
// ban's cause, when it ends, and its notes, and in observe mode, where ban is
// the ban that would have been made, or made permanent, mode, observe.
func (b banMaker) alertBan(rq *request, ban bans.Ban) {
event := alerts.EventBan
if ban.Permanent() {
event = alerts.EventPermanentBan
@@ -256,11 +264,11 @@ func (rq *request) alertBan(ban bans.Ban) {
detail := map[string]any{
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
}
if rq.h.config.Observe {
if b.observe {
detail["mode"] = "observe"
}
rq.h.alerts.Raise(alerts.Alert{
b.alerts.Raise(alerts.Alert{
Event: event,
Host: rq.site,
Client: rq.client,
@@ -273,6 +281,13 @@ func (rq *request) alertBan(ban bans.Ban) {
})
}
// netblockRequests is how many requests netblock has sent since it was
// first seen, the request being banned for included: the histories count it
// only once it has ended.
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
return b.limiter.Requests(netblock) + 1
}
// noted is the request, at now, with status, what the client was sent, or
// in observe mode would have been, as the notes of the ban it makes keep
// it.
@@ -287,23 +302,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
}
}
// netblockRequests is how many requests netblock has sent since it was
// first seen, this one included: the histories count it only once it has
// ended.
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1
}
// banNetblock is the netblock a ban on client covers: its IPv4 address,
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
// counts it in.
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group
// clientGroup counts it in, of length ipv6GroupPrefix.
func banNetblock(
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
) netip.Prefix {
addr := client.Unmap()
if addr.Is4() {
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
return netip.PrefixFrom(addr, banScopeV4Prefix).Masked()
}
return clientGroup(cfg, addr)
return clientGroup(ipv6GroupPrefix, addr)
}
// banExpires is when ban ends, as the log line gives it: a time, or
+12 -12
View File
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
) *verdict {
var running atomic.Pointer[string]
running.Store(new(trapPaths{}.name()))
running.Store(new(rq.h.trapPaths.name()))
found := make(chan *besideLookupFindings, 1)
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
return v
}
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now())
}
// findBesideLookup does the work of the heuristics of besideLookup on the
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
) *besideLookupFindings {
findings := &besideLookupFindings{}
findings.trapPath = rq.trapPath()
findings.trapPath = rq.h.trapPaths.trapPath(rq)
if findings.trapPath != "" {
findings.refuses = true
return findings
}
running.Store(new(ruleFiles{}.name()))
running.Store(new(rq.h.ruleFiles.name()))
findings.rules = rq.h.rules.Match(rq.in)
findings.rules = rq.h.ruleFiles.rules.Match(rq.in)
if refusedByRule(findings.rules) {
findings.refuses = true
return findings
}
running.Store(new(coreRuleSet{}.name()))
coreRuleSet := rq.h.coreRuleSet
running.Store(new(coreRuleSet.name()))
cfg := rq.h.config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
if coreRuleSet.mode == config.CoreRuleSetModeOff ||
pathExempt(rq.in, coreRuleSet.exemptPaths) {
return findings
}
start := time.Now()
// A question's body has been read into memory, so that reading it here
// cannot fail.
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect(
rq.in, rq.client, rq.in.Body,
)
findings.inspectionTime = time.Since(start)
findings.inspected = true
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection)
return findings
}
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
) *verdict {
rq.findings = findings
return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now())
}
// timedOut refuses the request for the heuristics of besideLookup not having
+36 -16
View File
@@ -2,6 +2,7 @@ package waf
import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// whole is the percentage of each limit a client gets when no biased
@@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
cfg.ASNLimitPercentURL != ""
}
// biasedThresholds are the settings and the lists by which limitPercentages
// lowers a client's limits.
type biasedThresholds struct {
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
lists *reputation.Lists
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
// and its percentage.
blocklistAction string
blocklistLimitPercent int64
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
dnsblAbuseIPDBAction string
dnsblAbuseIPDBLimitPercent int64
}
// limitPercentages returns the client's limit percentages, for the rate
// limits and for the byte limits, by its tags, its AS number and country
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
@@ -45,37 +65,37 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
// an AS number or a country they list.
func (rq *request) limitPercentages() (percentage, percentage) {
cfg := rq.h.config
func (b biasedThresholds) limitPercentages(
rq *request,
) (percentage, percentage) {
asn, country := rq.line.ASN, rq.line.Country
unknown := percentage{percent: whole}
if country == "" && !rq.skippedLookupWait {
unknown = percentage{
cfg.UnknownCountryLimitPercent,
b.unknownCountryLimitPercent,
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
}
}
fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
if percent, listed := b.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
}
blocklisted := percentage{percent: whole}
if rq.blocklisted &&
cfg.BlocklistAction == config.ReputationActionLimit {
if rq.blocklisted && b.blocklistAction == "limit" {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
b.blocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION",
}
}
dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
b.dnsblAbuseIPDBAction == "limit" {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
b.dnsblAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
}
}
@@ -83,11 +103,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
given(b.tagLimitPercent, asnTag(asn), tagSetting),
fetched,
)
countryRequests := given(
cfg.TagLimitPercent,
b.tagLimitPercent,
countryTag(country),
tagSetting,
)
@@ -98,23 +118,23 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if tag != asnTag(asn) && tag != countryTag(country) {
otherTags = lowest(
otherTags,
given(cfg.TagLimitPercent, tag, tagSetting),
given(b.tagLimitPercent, tag, tagSetting),
)
}
}
asnBytes, countryBytes := asnRequests, countryRequests
if _, listed := cfg.ASNBytesLimitPercent[asn]; listed {
if _, listed := b.asnBytesLimitPercent[asn]; listed {
asnBytes = given(
cfg.ASNBytesLimitPercent,
b.asnBytesLimitPercent,
asn,
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
)
}
if _, listed := cfg.CountryBytesLimitPercent[country]; listed {
if _, listed := b.countryBytesLimitPercent[country]; listed {
countryBytes = given(
cfg.CountryBytesLimitPercent,
b.countryBytesLimitPercent,
country,
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
)
+12 -9
View File
@@ -7,7 +7,7 @@ import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -15,20 +15,24 @@ import (
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
// lowers the client's limits instead (see limitPercentages), and being log, it
// does nothing more. Its figure is the blocklists that list the client.
type blocklists struct{}
type blocklists struct {
lists *reputation.Lists
action string // SWW_WAF_BLOCKLIST_ACTION
hitNoter reputationHitNoter
}
func (blocklists) name() string { return "blocklists" }
func (blocklists) check(
func (b blocklists) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
listedBy := rq.h.lists.ListedBy(rq.client)
listedBy := b.lists.ListedBy(rq.client)
rq.blocklisted = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a blocklist")
b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist")
if !rq.refusedByBlocklists(listedBy) {
if !b.refusedByBlocklists(listedBy) {
return nil
}
@@ -41,7 +45,6 @@ func (blocklists) check(
// refusedByBlocklists reports whether the heuristic blocklists refuses a
// client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
config.ReputationActionDeny
func (b blocklists) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && b.action == deny
}
+13 -6
View File
@@ -6,6 +6,8 @@ package waf
import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)
// byteLimits counts the request's bytes, as countedBytes gives them, for the
@@ -16,11 +18,16 @@ import (
// the client over a byte limit, as its limit percentage for the byte limits
// lowers it, break it, which bans the client; the response was passed on whole.
// Its figures are the window, the count and the limit.
type byteLimits struct{}
type byteLimits struct {
limiter *ratelimit.Limiter
bytesCounted string // SWW_WAF_BYTES_COUNTED
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (byteLimits) name() string { return "byte_limits" }
func (byteLimits) check(
func (b byteLimits) check(
_ context.Context,
rq *request,
now time.Time,
@@ -29,10 +36,10 @@ func (byteLimits) check(
return nil
}
counts, hit, over := rq.h.limiter.CountBytes(
clientGroup(rq.h.config, rq.client),
counts, hit, over := b.limiter.CountBytes(
clientGroup(b.ipv6GroupPrefix, rq.client),
now,
rq.countedBytes(),
rq.countedBytes(b.bytesCounted),
rq.bytesLimitPercent.percent,
)
rq.line.Counts.MinuteBytes = counts.MinuteBytes
@@ -43,7 +50,7 @@ func (byteLimits) check(
return nil
}
rq.banForLimit(now, hit, rq.out.status)
b.banMaker.banForLimit(rq, now, hit, rq.out.status)
return &verdict{figures: limitFigures(hit)}
}
+5 -6
View File
@@ -5,8 +5,6 @@ import (
"net/http"
"net/netip"
"slices"
"sneak.berlin/go/smallwebwaf/internal/config"
)
// peerAddress is the address of the request's TCP peer.
@@ -45,12 +43,13 @@ func scheme(r *http.Request, isQuestion bool) string {
// clientGroup is the client a request is counted toward: its IPv4
// address, or its IPv6 group, the netblock its IPv6 address is in of the
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since
// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as
// IPv4.
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
addr = addr.Unmap()
if addr.Is6() {
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked()
}
return netip.PrefixFrom(addr, addr.BitLen())
+31 -19
View File
@@ -10,6 +10,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -22,11 +23,22 @@ import (
// exploitsignature, raises the core_rule_set_match alert, and in block mode
// refuses the request with 403, which is an offence its client's history
// counts. Its figures are the score and the threshold.
type coreRuleSet struct{}
type coreRuleSet struct {
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
// off.
inspector CoreRuleSetInspector
metrics *metrics.Metrics
alerts *alerts.Queue
mode string // SWW_WAF_CORE_RULE_SET_MODE
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
observe bool // SWW_WAF_MODE is observe
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (coreRuleSet) name() string { return "core_rule_set" }
func (coreRuleSet) check(
func (c coreRuleSet) check(
_ context.Context,
rq *request,
_ time.Time,
@@ -36,7 +48,6 @@ func (coreRuleSet) check(
return nil
}
cfg := rq.h.config
result := findings.inspection
rq.requestBytes = int64(len(findings.read))
@@ -47,18 +58,18 @@ func (coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
c.metrics.CoreRuleSetMatched(c.mode, id)
}
threshold := cfg.CoreRuleSetScoreThreshold
threshold := c.scoreThreshold
if threshold == 0 || result.Score < threshold {
return nil
}
rq.addTag(config.ExploitSignatureTag)
rq.alertCoreRuleSetMatch(result)
c.alertCoreRuleSetMatch(rq, result)
if !rq.refusedByCoreRuleSet(result) {
if !c.refusedByCoreRuleSet(result) {
return nil
}
@@ -76,20 +87,21 @@ func (coreRuleSet) check(
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
// Rule Set found result of: whether the score is at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
cfg := rq.h.config
threshold := cfg.CoreRuleSetScoreThreshold
func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool {
threshold := c.scoreThreshold
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
return c.mode == config.CoreRuleSetModeBlock &&
threshold != 0 && result.Score >= threshold
}
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
// which the Core Rule Set scored at result, at or over
// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the
// Core Rule Set scored at result, at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
// score, the method and the path with the query, and, for a request that is
// not refused for it, the mode: detect, or observe in observe mode.
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
func (c coreRuleSet) alertCoreRuleSetMatch(
rq *request, result coreruleset.Result,
) {
detail := map[string]any{
"rule_ids": result.RuleIDs,
"score": result.Score,
@@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
}
switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect)
case rq.h.config.Observe:
case c.mode == config.CoreRuleSetModeDetect:
detail["mode"] = config.CoreRuleSetModeDetect
case c.observe:
detail["mode"] = "observe"
}
rq.h.alerts.Raise(alerts.Alert{
c.alerts.Raise(alerts.Alert{
Event: alerts.EventCoreRuleSetMatch,
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client),
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+5 -3
View File
@@ -14,16 +14,18 @@ import (
// does not hold the client's country as it was looked up. A client without a
// country, or whose country cannot be found, is refused. Its figures are the
// country and the setting that refused it.
type countries struct{}
type countries struct {
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
}
func (countries) name() string { return "countries" }
func (countries) check(
func (c countries) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
allowed := rq.h.config.ExclusivelyAllowedCountries
allowed := c.allowed
country := rq.line.Country
if len(allowed) == 0 || slices.Contains(allowed, country) {
+10 -3
View File
@@ -4,6 +4,7 @@ package waf
import (
"context"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -11,12 +12,18 @@ import (
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
// the setting the client is in.
type denyNets struct{}
type denyNets struct {
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
}
func (denyNets) name() string { return "deny_nets" }
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
for _, netblock := range rq.h.config.DenyNets {
func (d denyNets) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, netblock := range d.netblocks {
if netblock.Contains(rq.client) {
return &verdict{
action: requestlog.ActionClientDenied,
+47
View File
@@ -0,0 +1,47 @@
package waf
import (
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
t.Parallel()
heuristic := denyNets{
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
}
// The request has no handler: the heuristic reads only what it holds.
inside := &request{
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
}
v := heuristic.check(t.Context(), inside, time.Time{})
if v == nil {
t.Fatal("a client inside the netblock was let through")
}
if v.action != requestlog.ActionClientDenied {
t.Errorf(
"action = %q, want %q",
v.action,
requestlog.ActionClientDenied,
)
}
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
}
outside := &request{
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
}
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
t.Errorf("a client outside the netblock was refused: %+v", v)
}
}
+14 -6
View File
@@ -9,6 +9,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -19,21 +20,28 @@ import (
// being log, it does nothing more. A zone without a verdict on the client is
// asked about it in the background, and the request does not wait for the
// answer. Its figure is the zones that list the client, their keys masked.
type dnsbl struct{}
type dnsbl struct {
dnsbl *reputation.DNSBL
action string // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
hitNoter reputationHitNoter
}
func (dnsbl) name() string { return "dnsbl" }
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
func (d dnsbl) check(
ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
rq.dnsblListed = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a DNSBL zone")
d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone")
for _, zone := range listedBy {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
}
if !rq.dnsblListed ||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if !rq.dnsblListed || d.action != deny {
return nil
}
+5 -72
View File
@@ -5,9 +5,10 @@ import (
"time"
)
// To add a heuristic, write one file with a type that implements heuristic,
// and add one line for it to one of the lists below. Its settings go in
// config.WAF, and the comment at the top of its file names them.
// To add a heuristic, write one file with a struct that implements heuristic
// and holds the settings and parts it reads. Add one line for it to one of the
// lists that newHandler builds, which hands it those settings and parts. Its
// settings go in config.WAF, and the comment at the top of its file names them.
// heuristic is one of the checks that decide what waf does with a
// request.
@@ -36,78 +37,10 @@ const (
figureNetblock = "netblock"
)
// The lists of heuristics, each in the order its heuristics run. A client in
// SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number and
// country are looked up, so that a client they refuse is not looked up.
// header_limits comes first: it only measures and counts, and a request it
// refuses causes no other work.
func beforeLookup() []heuristic {
return []heuristic{
headerLimits{},
tagLists{},
routeWatchTags{},
denyNets{},
activeBan{},
}
}
// afterLookup are the heuristics that run once the lookup has been made.
func afterLookup() []heuristic {
return []heuristic{
countries{},
blocklists{},
dnsbl{},
abuseIPDB{},
tagDeny{},
rateLimits{},
}
}
// besideLookup are the heuristics that only read the request. Their work runs
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
// of the request. Their check then applies what that work found, and runs only
// once waf takes it.
func besideLookup() []heuristic {
return []heuristic{
trapPaths{},
ruleFiles{},
coreRuleSet{},
}
}
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second time,
// on the tag rule_files and core_rule_set can add.
func afterBesideLookup() []heuristic {
return []heuristic{
tagDeny{},
}
}
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client, and do
// not change what the request got: their verdict's action is not used.
func afterAnswer() []heuristic {
return []heuristic{
refusalLimit{},
}
}
// onReport are the heuristics that run on an allowed request when rproxy's
// report of it comes, before its report line is written. Like afterAnswer, they
// only ban.
func onReport() []heuristic {
return []heuristic{
byteLimits{},
}
}
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
rq.runBanningHeuristics(ctx, rq.h.afterAnswer)
}
// runHeuristics runs list, one of those checkClient runs, on the request at now,
+7 -6
View File
@@ -49,16 +49,15 @@ func (rq *request) startLookUp(
) <-chan lookupResult {
result := make(chan lookupResult, 1)
if rq.h.config.LookupSource == config.LookupSourceOff ||
!canBePlaced(rq.client) {
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
result <- lookupResult{}
return result
}
client := clientGroup(rq.h.config, rq.client)
client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
result <- lookupResult{
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
}
@@ -143,8 +142,10 @@ func AddLookup(
) {
limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country)
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country)
netblock := banNetblock(
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(),
)
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
}
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
+205 -33
View File
@@ -6,6 +6,7 @@ package waf
import (
"cmp"
"fmt"
"io"
"log/slog"
"net/http"
@@ -82,8 +83,8 @@ type Params struct {
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
// CoreRuleSet is what inspects requests while
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
@@ -133,15 +134,19 @@ type Server struct {
}
// newHandler returns the handler with what every request shares, the parts
// in params among it.
// in params among it. It builds the heuristics, each with the settings and
// parts it reads, and the lists they run in.
//
//nolint:funlen // a few lines for each heuristic, a list that grows with them
func newHandler(params Params) *handler {
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil {
panic(err) // New fails only for a size below one
}
return &handler{
config: params.Config,
cfg := params.Config
h := &handler{
config: cfg,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
now: params.Now,
@@ -152,47 +157,176 @@ func newHandler(params Params) *handler {
geojs: params.GeoJS,
anomalies: params.Anomalies,
lookupFile: params.LookupFile,
lists: params.Lists,
dnsbl: params.DNSBL,
abuseIPDB: params.AbuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders ||
params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config),
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
cfg.AddLookupHeaders ||
cfg.TagsNeedLookup() ||
biasedThresholdsSet(cfg),
besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit,
besideLookupTimeLimit,
),
}
sharedBanMaker := banMaker{
ledger: params.Ledger,
limiter: params.Limiter,
metrics: params.Metrics,
alerts: params.Alerts,
observe: cfg.Observe,
banResponse: cfg.BanResponse,
banScopeV4Prefix: cfg.BanScopeV4Prefix,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
hitNoter := reputationHitNoter{
metrics: params.Metrics,
alerts: params.Alerts,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.blocklists = blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
}
h.rateLimits = rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
metrics: params.Metrics,
banMaker: sharedBanMaker,
}
h.coreRuleSet = coreRuleSet{
inspector: params.CoreRuleSet,
metrics: params.Metrics,
alerts: params.Alerts,
mode: cfg.CoreRuleSetMode,
exemptPaths: cfg.CoreRuleSetExemptPaths,
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
observe: cfg.Observe,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.beforeLookup = []heuristic{
headerLimits{},
tagLists{lists: params.Lists},
routeWatchTags{
routeWatch: params.RouteWatch,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
},
denyNets{netblocks: cfg.DenyNets},
activeBan{banMaker: sharedBanMaker},
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
h.blocklists,
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
hitNoter: hitNoter,
},
abuseIPDB{
abuseIPDB: params.AbuseIPDB,
limiter: params.Limiter,
keySet: cfg.AbuseIPDBKey != "",
minScore: cfg.AbuseIPDBMinScore,
action: cfg.DNSBLAbuseIPDBAction,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
h.rateLimits,
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
h.afterAnswer = []heuristic{
refusalLimit{
limiter: params.Limiter,
perMinute: cfg.RefusalsLimitPerMinute,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.onReport = []heuristic{
byteLimits{
limiter: params.Limiter,
bytesCounted: cfg.BytesCounted,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
return h
}
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, fmt.Errorf(
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
err,
)
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return coreRuleSet, nil
}
// handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request.
type handler struct {
config *config.WAF
requestLog io.Writer
processLog *slog.Logger
now func() time.Time
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
lists *reputation.Lists
dnsbl *reputation.DNSBL
abuseIPDB *reputation.AbuseIPDB
routeWatch *routewatch.Client
rules *rules.Files
coreRuleSet CoreRuleSetInspector
alerts *alerts.Queue
config *config.WAF
requestLog io.Writer
processLog *slog.Logger
now func() time.Time
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
// waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on.
waitForGeoJS bool
@@ -202,4 +336,42 @@ type handler struct {
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number
// and country are looked up, so that a client they refuse is not looked
// up. header_limits comes first: it only measures and counts, and a
// request it refuses causes no other work.
beforeLookup []heuristic
// afterLookup are the heuristics that run once the lookup has been made.
afterLookup []heuristic
// besideLookup are the heuristics that only read the request. Their work
// runs beside the lookup and afterLookup, in findBesideLookup, which
// changes nothing of the request. Their check then applies what that work
// found, and runs only once waf takes it.
besideLookup []heuristic
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second
// time, on the tag rule_files and core_rule_set can add.
afterBesideLookup []heuristic
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client,
// and do not change what the request got: their verdict's action is not
// used.
afterAnswer []heuristic
// onReport are the heuristics that run on an allowed request when
// rproxy's report of it comes, before its report line is written. Like
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of the lists that other steps of a request use too:
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
blocklists blocklists
rateLimits rateLimits
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
}
+13 -34
View File
@@ -20,7 +20,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
@@ -505,20 +504,24 @@ func newWAFWith(
}
}
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params := waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
// The tests of the limit set it back, and hold the Core Rule Set
// with a stand-in.
BesideLookupTimeLimit: time.Hour,
}
addCoreRuleSet(t, &params)
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
@@ -527,30 +530,6 @@ func newWAFWith(
return server, out, alertQueue
}
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
func addCoreRuleSet(t *testing.T, params *waf.Params) {
t.Helper()
cfg := params.Config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params.CoreRuleSet = coreRuleSet
}
// addCounters adds to params the metrics, the table of clients, the ledger
// and the anomaly counters, as sww builds them from params' settings. The
// metrics read the ledger, the table of clients and the rules.
+34 -22
View File
@@ -6,8 +6,10 @@ package waf
import (
"context"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -18,28 +20,37 @@ import (
// refused, and bans the client. A request exempt from the rate limits is exempt
// from the byte limits too. Its figures are the window, the count and the
// limit.
type rateLimits struct{}
type rateLimits struct {
limiter *ratelimit.Limiter
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
biasedThresholds biasedThresholds
banMaker banMaker
}
func (rateLimits) name() string { return "rate_limits" }
func (rateLimits) check(
func (r rateLimits) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
rq.counted = rq.countedByRateLimits()
rq.counted = r.countedByRateLimits(rq)
if !rq.counted {
return nil
}
rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages()
rq.rateLimitPercent, rq.bytesLimitPercent =
r.biasedThresholds.limitPercentages(rq)
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
rq.rateLimitPercent.logged()
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged()
counts, hit, over := rq.h.limiter.Count(
clientGroup(rq.h.config, rq.client),
counts, hit, over := r.limiter.Count(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
rq.rateLimitPercent.percent,
)
@@ -49,28 +60,29 @@ func (rateLimits) check(
return nil
}
rq.banForLimit(now, hit, rq.h.config.BanResponse)
r.banMaker.banForLimit(rq, now, hit, r.banResponse)
return &verdict{
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
}
}
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
// request at now, whatever its client's limit percentages: the rate limits
// count it, and one more request from its client takes it over a whole rate
// limit. It counts nothing.
func (rq *request) overAWholeRateLimit(now time.Time) bool {
return rq.countedByRateLimits() &&
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
// at now, whatever its client's limit percentages: the rate limits count it,
// and one more request from its client takes it over a whole rate limit. It
// counts nothing.
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
return r.countedByRateLimits(rq) &&
r.limiter.OverARateLimit(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
)
}
// countedByRateLimits reports whether the rate limits count the request: its
// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
func (rq *request) countedByRateLimits() bool {
cfg := rq.h.config
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
// countedByRateLimits reports whether the rate limits count rq: its client is
// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
func (r rateLimits) countedByRateLimits(rq *request) bool {
return !isInside(rq.client, r.exemptNets) &&
!pathExempt(rq.in, r.exemptPaths)
}
+15 -9
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -18,17 +19,22 @@ import (
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
// heuristics skip, is not counted, and nothing is while the limit is off.
// Its figures are the window, the count and the limit.
type refusalLimit struct{}
type refusalLimit struct {
limiter *ratelimit.Limiter
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
banResponse int // SWW_WAF_BAN_RESPONSE
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
banMaker banMaker
}
func (refusalLimit) name() string { return "refusal_limit" }
func (refusalLimit) check(
func (r refusalLimit) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
cfg := rq.h.config
if cfg.RefusalsLimitPerMinute == 0 {
if r.perMinute == 0 {
return nil
}
@@ -36,10 +42,10 @@ func (refusalLimit) check(
return nil
}
hit, over := rq.h.limiter.CountRefusal(
clientGroup(rq.h.config, rq.client),
hit, over := r.limiter.CountRefusal(
clientGroup(r.ipv6GroupPrefix, rq.client),
now,
cfg.RefusalsLimitPerMinute,
r.perMinute,
)
if !over {
return nil
@@ -54,10 +60,10 @@ func (refusalLimit) check(
status = http.StatusForbidden
case requestlog.ActionTemporarilyBanned,
requestlog.ActionPermanentlyBanned:
status = cfg.BanResponse
status = r.banResponse
}
rq.banForLimit(now, hit, status)
r.banMaker.banForLimit(rq, now, hit, status)
return &verdict{figures: limitFigures(hit)}
}
+25 -8
View File
@@ -5,22 +5,39 @@ package waf
import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/metrics"
)
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
// refuses the requests of a client a source lists.
const deny = "deny"
// reputationHitNoter notes the reputation sources that list a request's
// client, for the reputation heuristics.
type reputationHitNoter struct {
metrics *metrics.Metrics
alerts *alerts.Queue
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
// noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does,
// DNSBL zones, their keys masked, that list rq's client, as noteHit does,
// with reason.
func (rq *request) noteListed(sources []string, reason string) {
func (n reputationHitNoter) noteListed(
rq *request, sources []string, reason string,
) {
for _, source := range sources {
rq.noteHit(bans.ReputationHit{Source: source}, reason)
n.noteHit(rq, bans.ReputationHit{Source: source}, reason)
}
}
// noteHit adds hit's source, which lists the client, to the log line's
// noteHit adds hit's source, which lists rq's client, to the log line's
// reputation, and hit to the notes of a ban the request makes, counts the
// source in the metrics, and raises a reputation_hit alert with reason,
// whose detail gives hit's source and score.
func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
func (n reputationHitNoter) noteHit(
rq *request, hit bans.ReputationHit, reason string,
) {
detail := map[string]any{"source": hit.Source}
if hit.Score != nil {
detail["score"] = *hit.Score
@@ -28,12 +45,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
rq.reputation = append(rq.reputation, hit)
rq.h.metrics.ReputationHit(hit.Source)
rq.h.alerts.Raise(alerts.Alert{
n.metrics.ReputationHit(hit.Source)
n.alerts.Raise(alerts.Alert{
Event: alerts.EventReputationHit,
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client),
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+14 -9
View File
@@ -122,8 +122,11 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion),
PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: clientGroup(h.config, client).String(),
ContentType: r.Header.Get("Content-Type"),
ClientGroup: clientGroup(
h.config.IPv6GroupPrefix,
client,
).String(),
ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders(
r,
h.config.LogRequestHeaders,
@@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
now := rq.h.now()
v := rq.runHeuristics(ctx, beforeLookup(), now)
v := rq.runHeuristics(ctx, rq.h.beforeLookup, now)
if v != nil {
return v
}
@@ -265,7 +268,7 @@ func (rq *request) runAfterLookup(
// again: a request counted at the time it arrived could be counted
// after a later one of its client's, which the rate limits would take
// for the clock set back.
return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now())
}
// pathExempt reports whether r is exempt under prefixes,
@@ -365,7 +368,7 @@ func (rq *request) addToHistory() {
}
rq.h.limiter.AddToHistory(
clientGroup(rq.h.config, rq.client),
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
rq.h.now(),
request,
)
@@ -394,11 +397,11 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site,
Client: rq.client,
ClientGroup: clientGroup(rq.h.config, rq.client),
ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
ASN: answer.ASN,
ASName: answer.ASName,
Country: answer.Country,
Bytes: rq.countedBytes(),
Bytes: rq.countedBytes(rq.h.config.BytesCounted),
})
}
@@ -421,11 +424,13 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return geojs.Answer{}, false
}
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
return rq.lookupAnswer, true
}
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
return rq.h.geojs.Kept(
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
)
}
// responseWriter is waf's answer to a request. It notes the status and
+10 -5
View File
@@ -8,6 +8,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
)
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
@@ -17,22 +18,26 @@ import (
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
// asked for by the request's address, and a route without an origin AS gives only
// the prefix= tag.
type routeWatchTags struct{}
type routeWatchTags struct {
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
routeWatch *routewatch.Client
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
}
func (routeWatchTags) name() string { return "routewatch_tags" }
func (routeWatchTags) check(
func (r routeWatchTags) check(
ctx context.Context,
rq *request,
_ time.Time,
) *verdict {
if rq.h.routeWatch == nil {
if r.routeWatch == nil {
return nil
}
route, missing := rq.h.routeWatch.Route(
route, missing := r.routeWatch.Route(
ctx,
clientGroup(rq.h.config, rq.client),
clientGroup(r.ipv6GroupPrefix, rq.client),
rq.client,
)
if missing != "" {
+10 -4
View File
@@ -9,6 +9,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
)
@@ -22,11 +23,15 @@ import (
// mode raises the alert for the ban
// it would have made. Either rule's match is noted as an offence, for the
// client's history. Its figures are the rule's id and action.
type ruleFiles struct{}
type ruleFiles struct {
rules *rules.Files
metrics *metrics.Metrics
banMaker banMaker
}
func (ruleFiles) name() string { return "rule_files" }
func (ruleFiles) check(
func (r ruleFiles) check(
_ context.Context,
rq *request,
now time.Time,
@@ -35,7 +40,7 @@ func (ruleFiles) check(
for _, rule := range matched {
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
r.metrics.RuleMatched(rule.ID, rule.Action)
}
if len(matched) == 0 {
@@ -60,7 +65,8 @@ func (ruleFiles) check(
rq.attack = true
return &verdict{
action: rq.banForAttack(
action: r.banMaker.banForAttack(
rq,
now,
bans.Notes{RuleID: last.ID, Target: last.Target},
),
+9 -3
View File
@@ -12,13 +12,19 @@ import (
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
// the first such tag, in sorted order.
type tagDeny struct{}
type tagDeny struct {
tags []string // SWW_WAF_TAG_DENY
}
func (tagDeny) name() string { return "tag_deny" }
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
func (t tagDeny) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, tag := range rq.line.Tags {
if slices.Contains(rq.h.config.TagDeny, tag) {
if slices.Contains(t.tags, tag) {
return &verdict{
action: requestlog.ActionClientDenied,
figures: map[string]any{"tag": tag},
+11 -3
View File
@@ -6,17 +6,25 @@ package waf
import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
// holds the client, the tag being the list's name. It refuses nothing, and runs
// first so that every refusal's log line carries these tags.
type tagLists struct{}
type tagLists struct {
lists *reputation.Lists
}
func (tagLists) name() string { return "tag_lists" }
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
for _, name := range rq.h.lists.TagsOf(rq.client) {
func (t tagLists) check(
_ context.Context,
rq *request,
_ time.Time,
) *verdict {
for _, name := range t.lists.TagsOf(rq.client) {
rq.addTag(name)
}
+15 -8
View File
@@ -16,11 +16,14 @@ import (
// it bans the client's netblock, or in observe mode raises the alert for the
// ban it would have made. The ban's notes give the path as sent, and so does
// its figure.
type trapPaths struct{}
type trapPaths struct {
paths []string // SWW_WAF_TRAP_PATHS
banMaker banMaker
}
func (trapPaths) name() string { return "trap_paths" }
func (trapPaths) check(
func (t trapPaths) check(
_ context.Context,
rq *request,
now time.Time,
@@ -33,18 +36,22 @@ func (trapPaths) check(
rq.attack = true
return &verdict{
action: rq.banForAttack(now, bans.Notes{TrapPath: path}),
action: t.banMaker.banForAttack(
rq,
now,
bans.Notes{TrapPath: path},
),
figures: map[string]any{"path": path},
}
}
// trapPath returns the request's path as sent, when its path as a path rule
// sees it, without the query, as sent or once percent-decoded, is in
// trapPath returns rq's path as sent, when its path as a path rule sees it,
// without the query, as sent or once percent-decoded, is in
// SWW_WAF_TRAP_PATHS, and "" otherwise.
func (rq *request) trapPath() string {
func (t trapPaths) trapPath(rq *request) string {
path := rules.Path(rq.in)
if !slices.Contains(rq.h.config.TrapPaths, path) &&
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
if !slices.Contains(t.paths, path) &&
!slices.Contains(t.paths, rules.DecodeOnce(path)) {
return ""
}
+9 -7
View File
@@ -123,10 +123,10 @@ func New(params Params) *Server {
GeoJS: h.geojs,
Anomalies: h.anomalies,
LookupFile: h.lookupFile,
Lists: h.lists,
DNSBL: h.dnsbl,
AbuseIPDB: h.abuseIPDB,
RouteWatch: h.routeWatch,
Lists: params.Lists,
DNSBL: params.DNSBL,
AbuseIPDB: params.AbuseIPDB,
RouteWatch: params.RouteWatch,
Metrics: h.metrics,
}
}
@@ -154,8 +154,10 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
return false
}
return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
rq.overAWholeRateLimit(now)
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
}
// answerQuestion runs the checks on the visitor's request a question
@@ -491,7 +493,7 @@ func (h *handler) finishWith(
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.runBanningHeuristics(ctx, onReport())
rq.runBanningHeuristics(ctx, rq.h.onReport)
rq.countAnomalies()
rq.addToHistory()