Compare commits

..
Author SHA1 Message Date
clawbot 89f2918d25 bans: one ledger method makes a request's ban, one place decides observe mode (closes #444)
check / check (push) Waiting to run
The ledger's BanForLimit, WouldBanForLimit, BanForAttack and
WouldBanForAttack become Ban and WouldBan, which take the cause and the
notes in one bans.Grounds. The waf's banForLimit and banForAttack fill in
what differs and call banClient, the one function that tests observe
mode on the way to a ban. Behaviour is unchanged; tests change only
where they called the four methods.

Model: opus-5-5
2026-11-19 17:39:32 +00:00
26 changed files with 648 additions and 599 deletions

No files matched your search

+3 -5
View File
@@ -906,11 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each.
- `internal/state`: the state and cache files.
- `internal/requestlog`: the request log.
- `internal/processlog`: the process's own messages.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
`internal/metrics`: the state and cache files, the log lines, the syslog
sender and the metrics.
## Rationale
+2 -4
View File
@@ -104,10 +104,8 @@ never as a time.
its own.
- `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop.
- `internal/requestlog`: the request log, as JSON lines on stdout.
- `internal/processlog`: the process lines on stdout, which hold the
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
sending them to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its
+8 -4
View File
@@ -54,7 +54,11 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
now := midnight()
scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
limitBan, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made)
@@ -119,7 +123,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.BanForLimit(limited, now, bans.Notes{})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
// listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
@@ -144,7 +148,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
}
ledger.BanForLimit(limited, now, bans.Notes{})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made)
@@ -154,7 +158,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
// The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes.
ledger.BanForAttack(attacker, now, bans.Notes{})
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores {
+64 -23
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one.
ledger.BanForLimit(second, midnight(), bans.Notes{})
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
wantBans(
t,
ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
}
// And the next drops that one.
ledger.BanForLimit(third, midnight(), bans.Notes{})
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
wantBans(
t,
ledger,
@@ -98,20 +98,28 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins")
first, _ := ledger.BanForLimit(
first, _ := ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes.
ledger.BanForLimit(other, second.Start, bans.Notes{})
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it.
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -127,15 +135,27 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban.
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
kept := first
kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -147,20 +167,37 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams())
limit, _ := ledger.BanForLimit(
limit, _ := ledger.Ban(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
)
byteLimit, _ := ledger.BanForLimit(
byteLimit, _ := ledger.Ban(
netip.MustParsePrefix("203.0.113.3/32"),
midnight(),
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
)
attack, _ := ledger.BanForAttack(
attack, _ := ledger.Ban(
netip.MustParsePrefix("203.0.113.2/32"),
midnight(),
bans.Notes{RuleID: "git-dir", Target: "path"},
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
)
for _, tc := range []struct{ got, want string }{
@@ -200,7 +237,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses")
}
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -251,7 +288,11 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
}
// The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf(
"the next ban for an attack ends at %s, want seven days on",
@@ -264,10 +305,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
made, _ := ledger.BanForLimit(
made, _ := ledger.Ban(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -311,7 +352,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams())
// An hour's ban for a broken limit.
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an
@@ -431,10 +472,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first.
ledger.BanForLimit(
ledger.Ban(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"),
+54 -82
View File
@@ -300,81 +300,50 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest
}
// BanForLimit bans netblock at now for a broken limit, with notes, and
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
// made within LimitBanRepeatWindow after the netblock's ban that ended
// last, other than one for a clear sign of attack or a lifted one, lasts
// repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
// Grounds are why a request makes a ban: its cause, CauseLimit or
// CauseAttack, and its notes.
type Grounds struct {
Cause string
Notes Notes
}
// WouldBanForLimit returns what BanForLimit would, without making the ban:
// what observe mode would have done.
func (l *Ledger) WouldBanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
// Ban bans netblock at now on grounds, and returns the ban, and true. If
// a ban on netblock is still active, as when two of its requests break a
// limit at once, that ban is returned with false, and no other is made.
// The ledger fills in the notes' Refused and EarlierBans itself.
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
return l.ban(netblock, now, grounds, true)
}
// BanForAttack bans netblock at now for a clear sign of attack, with
// notes, and returns the ban, and whether it made it, as BanForLimit
// does. A first ban lasts AttackBanDuration; once the netblock has had
// one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
// WouldBan returns what Ban would, without making the ban: what observe
// mode would have done.
func (l *Ledger) WouldBan(
netblock netip.Prefix, now time.Time, grounds Grounds,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
return l.ban(netblock, now, grounds, false)
}
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as BanForLimit or
// BanForAttack would make it. It works out nothing else of the ban.
// or CauseAttack, made at now would be permanent, as Ban would make it. It
// works out nothing else of the ban.
func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause string,
) bool {
@@ -411,10 +380,9 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// ban even while another on netblock is active, since the admin asked
// for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
// on netblock is active, since the admin asked for this one. The ledger
// fills in the notes' EarlierBans, and counts the ban among those made.
func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string,
) Ban {
@@ -759,15 +727,13 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
return forgot
}
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
// ban bans netblock at now on grounds, as Ban describes, and returns the
// ban, and whether it made it. Unless keep is true, the ban is not made,
// only returned: it is the ban that would have been made.
func (l *Ledger) ban(
netblock netip.Prefix,
now time.Time,
cause, reason string,
notes Notes,
grounds Grounds,
keep bool,
) (Ban, bool) {
l.mu.Lock()
@@ -776,6 +742,8 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active.
var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock)
if found {
active := activeBan(*bans, now)
@@ -791,15 +759,19 @@ func (l *Ledger) ban(
ban := Ban{
Netblock: netblock,
Start: now,
Cause: cause,
Reason: reason,
Cause: grounds.Cause,
Notes: notes,
}
if cause == CauseAttack {
ban.Expires = l.attackExpiry(held, now)
} else {
switch grounds.Cause {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
ban.Expires = l.attackExpiry(held, now)
default:
panic("bans: a request's ban has the cause " + grounds.Cause)
}
if !keep {
@@ -807,7 +779,7 @@ func (l *Ledger) ban(
}
l.add(ban)
l.made[cause]++
l.made[grounds.Cause]++
l.markChanged()
return ban, true
+121 -51
View File
@@ -22,7 +22,11 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) ||
@@ -42,7 +46,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends.
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
}
@@ -71,15 +75,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.BanForLimit(
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.BanForLimit(
second, _ := ledger.Ban(
netblock,
first.Expires.Add(tc.gap),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
if second.Expires.Sub(second.Start) != tc.want ||
@@ -104,10 +108,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params)
ban, _ := ledger.BanForLimit(
ban, _ := ledger.Ban(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -127,7 +131,11 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight()
for i := range 14 {
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if !ban.Expires.After(ban.Start) {
t.Fatalf(
"ban %d starts at %s and ends at %s",
@@ -140,7 +148,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires
}
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
}
@@ -152,15 +160,19 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
first, made := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
if !made {
t.Error("the first ban was not made")
}
again, made := ledger.BanForLimit(
again, made := ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
if made || !reflect.DeepEqual(again, first) ||
@@ -175,10 +187,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
)
}
again, made = ledger.BanForAttack(
again, made = ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Notes{},
bans.Grounds{Cause: bans.CauseAttack},
)
if made || !reflect.DeepEqual(again, first) {
t.Errorf(
@@ -195,10 +207,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.BanForLimit(
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Notes{Requests: 5},
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
)
for range 3 {
@@ -237,10 +249,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.BanForLimit(
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Notes{Requests: 5},
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
)
got, banned, _ := ledger.Find(
@@ -287,24 +299,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64")
now := midnight()
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.BanForLimit(b, now, bans.Notes{})
ledger.BanForLimit(c, now, bans.Notes{})
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
// A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's.
ledger.Check(a.Addr(), now)
ledger.BanForLimit(d, now, bans.Notes{})
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.BanForLimit(a, first.Expires, bans.Notes{})
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes
// first.
ledger.Check(d.Addr(), first.Expires)
ledger.BanForLimit(b, first.Expires, bans.Notes{})
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -325,8 +337,16 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -343,7 +363,11 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" ||
@@ -418,8 +442,16 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count.
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -435,10 +467,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit.
third, _ := ledger.BanForAttack(
third, _ := ledger.Ban(
netblock,
second.Expires.Add(30*day),
bans.Notes{},
bans.Grounds{Cause: bans.CauseAttack},
)
if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -452,14 +484,18 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
wantChanged(t, ledger, true)
// While the first ban lasts, none would be made.
during, would := ledger.WouldBanForAttack(
during, would := ledger.WouldBan(
netblock,
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseAttack},
)
if would || !reflect.DeepEqual(during, first) {
t.Errorf(
@@ -472,13 +508,19 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made.
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
bans.Notes{RuleID: "git-dir"})
limit, wouldLimit := ledger.WouldBanForLimit(
limitGrounds := bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
}
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
netblock,
first.Expires,
limitNotes,
limitGrounds,
)
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -505,7 +547,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false)
// The ban made is the one that would have been.
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit)
}
@@ -526,7 +568,11 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1)
}
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
now = ban.Expires
}
@@ -539,7 +585,11 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
}
// Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
attack, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent")
}
@@ -553,8 +603,16 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
attack, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit {
@@ -578,10 +636,13 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
}
ban, _ := ledger.BanForLimit(
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Notes{Request: request},
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
)
cut := long[:256]
@@ -607,11 +668,20 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock.
ledger.BanForLimit(netblock, midnight(), bans.Notes{
ASN: "AS64497", ASName: "Other Net", Country: "FR",
ledger.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
})
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
ledger.BanForLimit(other, midnight(), bans.Notes{})
ledger.Ban(
netblock,
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+36 -21
View File
@@ -19,25 +19,29 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false)
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write.
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false)
// Two bans before the value is read leave one.
ledger.BanForLimit(
ledger.Ban(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.BanForLimit(
ledger.Ban(
netip.MustParsePrefix("203.0.113.11/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
wantChanged(t, ledger, true)
wantChanged(t, ledger, false)
@@ -51,10 +55,14 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
ledger.BanForLimit(high, midnight(), bans.Notes{})
ledger.BanForLimit(low, midnight(), bans.Notes{})
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
first, _ := ledger.Ban(
v6,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
snapshot := ledger.Snapshot()
@@ -80,7 +88,10 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
// Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three
@@ -96,7 +107,11 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse")
}
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
again, _ := after.Ban(
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -124,10 +139,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(),
},
})
ledger.BanForLimit(
ledger.Ban(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
for client, want := range map[string]bool{
@@ -200,7 +215,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
}
// A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+
@@ -236,10 +251,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.BanForLimit(
ban, _ := ledger.Ban(
netblock,
nineHours.Expires.Add(time.Hour),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -313,15 +328,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
}
// The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.BanForLimit(
first, _ := ledger.Ban(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.BanForLimit(
second, _ := ledger.Ban(
netip.MustParsePrefix("198.51.100.8/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
want := []bans.Ban{first, second, kept}
+24 -14
View File
@@ -84,7 +84,12 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct {
params Params
url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure.
httpClient *http.Client
@@ -135,7 +140,12 @@ func New(params Params) *GeoJS {
}
return &GeoJS{
params: params,
url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
@@ -178,7 +188,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client)
if !found {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
}
w, waiting := g.waiting[client]
@@ -241,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge()
now := g.params.Now()
now := g.now()
for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor {
@@ -278,7 +288,7 @@ func (g *GeoJS) answerOrWait(
}
if w == nil {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
return Answer{}, nil // too many clients wait already
}
@@ -289,7 +299,7 @@ func (g *GeoJS) answerOrWait(
}
if w.late {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
return Answer{}, nil
}
@@ -300,7 +310,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.params.Now()
now := g.now()
kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor {
@@ -317,7 +327,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
// GeoJS are for every client waiting, so they go on when the caller whose
// ctx is given stops waiting.
func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.params.Now().Before(g.retryAt) {
if g.asking || g.now().Before(g.retryAt) {
return
}
@@ -339,9 +349,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
given, err := g.request(ctx, clients)
kept, answered := g.keep(clients, given, err)
if g.params.Answered != nil {
if g.answered != nil {
for _, answer := range kept {
g.params.Answered(answer)
g.answered(answer)
}
}
@@ -390,7 +400,7 @@ func (g *GeoJS) keep(
g.mu.Lock()
defer g.mu.Unlock()
now := g.params.Now()
now := g.now()
kept := make([]Answer, 0, len(clients))
leftOut := 0
@@ -434,7 +444,7 @@ func (g *GeoJS) keep(
w.late = true
}
g.params.Failed(err, g.retryDelay)
g.failed(err, g.retryDelay)
return kept, false
}
@@ -464,7 +474,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext(
ctx,
http.MethodGet,
g.params.URL,
g.url,
http.NoBody,
)
if err != nil {
@@ -473,7 +483,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.params.Asked()
g.asked()
res, err := g.httpClient.Do(req)
if err != nil {
-44
View File
@@ -1,44 +0,0 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
-105
View File
@@ -1,105 +0,0 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+4 -7
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight()
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for client, requests := range map[string]int{
"198.51.100.9/32": 2,
+6 -18
View File
@@ -156,28 +156,16 @@ type Request struct {
BlockedByCoreRuleSet bool
}
// Params are what New needs.
type Params struct {
// Limits are what each client's requests and bytes are counted against.
Limits Limits
// MaxClients is the most clients the table holds,
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
// New returns a Limiter for limits, with no client counted yet, whose
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
// it, the least recently seen client is dropped, with its history, and
// starts afresh if it comes back.
func New(limits Limits, maxClients int) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
if err != nil {
panic(err) // NewLRU fails only for a size below one
}
limits := params.Limits
return &Limiter{
windows: [3]window{
{
+39 -63
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
tableSize,
)
if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
off := ratelimit.New(ratelimit.Limits{}, tableSize)
off.Count(client, now, whole)
if off.OverARateLimit(client, now) {
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
limiter := ratelimit.New(tc.limits, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
quarter := tc.length / 4
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerHour: limit},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: 5,
BytesPerDay: math.MaxInt64,
},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel()
// Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
limiter := ratelimit.New(tc.limits, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it.
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight()
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerDay: limit},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
tableSize,
)
refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight()
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
now := midnight()
clients := make([]netip.Prefix, maxClients+1)
+6 -12
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64",
}
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
}
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
before := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
for range limit {
wantCount(t, before, client, start, "")
}
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no
// fresh allowance.
later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour)
}
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit)
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client {
t.Helper()
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
after := ratelimit.New(ratelimit.Limits{}, tableSize)
after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0]
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next()
}
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
limiter.Load(clients, midnight())
got := limiter.Snapshot()
+32 -3
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the request log sww prints on stdout: one
// Package requestlog writes the lines sww prints on stdout: one
// JSON object per request, two from waf and a short one from rproxy,
// marked "type":"request". Package processlog writes the process's own
// messages.
// marked "type":"request", and the process's own messages as JSON lines marked
// "type":"process".
package requestlog
import (
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond,
)
}
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
+87
View File
@@ -3,6 +3,8 @@ package requestlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want)
}
}
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+6 -6
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{
Output: io.Discard,
InstanceName: "test",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
io.Discard,
"test",
slog.LevelInfo,
),
})
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
+6 -6
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -370,11 +370,11 @@ func startServerLogging(
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: processlog.New(processlog.Params{
Output: processLog,
InstanceName: "test",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
processLog,
"test",
slog.LevelInfo,
),
})
listener, err := (&net.ListenConfig{}).Listen(
+31 -19
View File
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it.
first, _ := params.Ledger.BanForLimit(
first, _ := params.Ledger.Ban(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
time.Sleep(5 * time.Second)
second, _ := params.Ledger.BanForLimit(
second, _ := params.Ledger.Ban(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run)
// A ban, and bans.json written with it.
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Notes{})
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
time.Sleep(params.BansWriteDelay)
synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and
// another added.
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Notes{})
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.BanForLimit(
params.Ledger.Ban(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.BanForLimit(
params.Ledger.Ban(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
err = files.WriteAll()
@@ -2135,7 +2135,11 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
params.Ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned {
@@ -2448,7 +2452,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000,
}),
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{
@@ -2510,11 +2514,15 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.BanForLimit(client, now, bans.Notes{
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
params.Ledger.Ban(client, now, bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
})
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Notes{RuleID: "env-file", Target: "path"})
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2700,7 +2708,11 @@ func wantLiftedBanKept(
}
// Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
ban, _ := ledger.Ban(
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf(
"the next ban lasts %s, want 1h",
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
)
@@ -23,11 +23,11 @@ import (
// setting for either.
func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: hostname,
Level: slog.LevelInfo,
})
processLog := requestlog.NewProcessLogger(
params.Stdout,
hostname,
slog.LevelInfo,
)
cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil {
+16 -22
View File
@@ -19,10 +19,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,11 +43,8 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
processLog := requestlog.NewProcessLogger(params.Stdout,
config.InstanceName(params.Viper), slog.LevelError)
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
@@ -66,11 +63,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = processlog.New(processlog.Params{
Output: stdout,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
processLog = requestlog.NewProcessLogger(
stdout,
cfg.InstanceName,
cfg.LogLevel,
)
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
@@ -241,17 +238,14 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
return ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
}
// newLedger returns the ledger of the bans, with the settings for them.
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"",
map[string]string{metricsToken: token},
)
server.Ledger.BanForLimit(
server.Ledger.Ban(
netip.MustParsePrefix(otherClient+"/32"),
clk.Now(),
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
before := server.Ledger.Snapshot()
+3 -5
View File
@@ -130,11 +130,9 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make
// permanent.
group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.BanForAttack(
group,
start,
bans.Notes{RuleID: "probe"},
)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
})
// The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack.
+2 -2
View File
@@ -686,10 +686,10 @@ func askDirectly(
// banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.BanForLimit(
server.Ledger.Ban(
netip.MustParsePrefix(client+"/32"),
now,
bans.Notes{},
bans.Grounds{Cause: bans.CauseLimit},
)
}
+72 -53
View File
@@ -103,9 +103,8 @@ func (rq *request) countedBytes() int64 {
// one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode it makes no ban and
// sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
// the client's counters back to zero. In observe mode, banClient says what
// happens instead.
func (rq *request) banForLimit(
now time.Time,
hit ratelimit.Hit,
@@ -123,23 +122,11 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
}
switch hit.Kind {
@@ -149,47 +136,54 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
}
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
}
rq.banClient(
now,
status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
)
}
// banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It fills in the rest of the notes. In observe mode it makes
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
// asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) string {
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
if wouldBan {
rq.alertBan(ban)
}
@@ -197,7 +191,13 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return banAction(ban.Permanent())
}
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
rq.line.BanExpires = banExpires(ban)
if made {
@@ -207,6 +207,25 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return banAction(ban.Permanent())
}
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per
+18 -22
View File
@@ -24,7 +24,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -399,11 +398,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{
Output: io.Discard,
InstanceName: "rproxy",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
io.Discard,
"rproxy",
slog.LevelInfo,
),
})
listener, err := (&net.ListenConfig{}).Listen(
@@ -472,11 +471,11 @@ func newWAFWith(
}
out := &output{}
processLog := processlog.New(processlog.Params{
Output: out,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
processLog := requestlog.NewProcessLogger(
out,
cfg.InstanceName,
cfg.LogLevel,
)
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -562,17 +561,14 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName,
)
params.Limiter = ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
params.Limiter = ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,