Compare commits

..
Author SHA1 Message Date
sneak e52edb9f62 requestlog: the package doc no longer says one line per request
check / check (push) Waiting to run
It said the request log is one JSON object per request, then listed
three for a request: two from waf and one from rproxy.

Model: opus-5-5
2026-11-19 18:43:32 +00:00
sneak 5667661dd8 SPEC: the request log is not one line per request
check / check (push) Waiting to run
The package list said requestlog writes one JSON line per request. SPEC's
"The log" section gives a request waf answers three request log lines.

Model: opus-5-5
2026-11-19 18:23:52 +00:00
sneak 5ce35505bc Constructors take Params: ratelimit.New and processlog.New (closes #450)
ratelimit.New takes ratelimit.Params (Limits, MaxClients) in place of
two arguments by position. The process logger moves out of requestlog
into internal/processlog, whose New takes Params (Output, InstanceName,
Level); its two tests move with it unchanged. geojs.GeoJS keeps its
params whole, as DNSBL, AbuseIPDB and routewatch.Client do. SPEC and
README list the new package.

Model: opus-5-5
2026-11-19 18:23:52 +00:00
clawbot e7cc0adbdb waf: end the asks to GeoJS, the DNSBL zones, AbuseIPDB and routewatch at the stop (closes #439)
check / check (push) Waiting to run
Each of the four keeps the cancel function of every ask it started, and a
wait group of their goroutines. Its new method (GeoJS.StopAsking,
DNSBL.StopAsking, AbuseIPDB.StopChecking, routewatch Client.StopAsking)
refuses further asks, ends those under way, and returns once their
goroutines have. An ask ended that way is no failure: no alert, no failure
count, no wait before the next. Each provider in internal/sww adds the
method as a stop hook, so the last write of the state files follows it.

Model: opus-5-5
2026-11-19 19:02:12 +01:00
33 changed files with 979 additions and 715 deletions

No files matched your search

+5 -3
View File
@@ -906,9 +906,11 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each. `internal/alerts`: one check or source each.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and - `internal/state`: the state and cache files.
`internal/metrics`: the state and cache files, the log lines, the syslog - `internal/requestlog`: the request log.
sender and the metrics. - `internal/processlog`: the process's own messages.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
## Rationale ## Rationale
+4 -2
View File
@@ -104,8 +104,10 @@ never as a time.
its own. its own.
- `internal/state`: reads the state and cache files at start, takes in edits - `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop. while running, and writes the files when due and at the stop.
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and - `internal/requestlog`: the request log, as JSON lines on stdout.
sending them to the syslog server. - `internal/processlog`: the process lines on stdout, which hold the
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format. - `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image. - `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its `deploy/example-app` is an app built on the image, with `rproxy` in its
+4 -8
View File
@@ -54,11 +54,7 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
now := midnight() now := midnight()
scores := map[netip.Prefix]int64{netblock: 100} scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.Ban( limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 { if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made) t.Errorf("made %d bans over a ban for a limit, want none", made)
@@ -123,7 +119,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
expires := now.Add(30 * day) expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100}) ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(limited, now, bans.Notes{})
// listed is off the blacklist when it is fetched after the ban's end. // listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 { if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
@@ -148,7 +144,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100 scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
} }
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(limited, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 { if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made) t.Errorf("made %d bans for 3 netblocks, want 3", made)
@@ -158,7 +154,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
// The ban for an attack takes the place of the one for a limit, and no // The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes. // blacklist ban goes.
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack}) ledger.BanForAttack(attacker, now, bans.Notes{})
want := map[netip.Prefix]int{limited: 0, attacker: 1} want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores { for netblock := range scores {
+23 -64
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2}) wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one. // A new ban drops the ban sww made, and only that one.
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(second, midnight(), bans.Notes{})
wantBans( wantBans(
t, t,
ledger, ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
} }
// And the next drops that one. // And the next drops that one.
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(third, midnight(), bans.Notes{})
wantBans( wantBans(
t, t,
ledger, ledger,
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended. // twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute), ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins") "probes for logins")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(
netblock, netblock,
midnight().Add(time.Minute), midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
) )
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first // A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes. // ban for a limit, its earliest that is not an admin's, goes.
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(other, second.Start, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1}) wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it. // The netblock's next ban still counts it.
third, _ := ledger.Ban( third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) { if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+ t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans) "want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first // The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin // ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban. // then keeps the first ban.
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock, second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
kept := first kept := first
kept.Cause = bans.CauseAdmin kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second}) ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's. // The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.Ban( third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) { if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+ t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans) "want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
limit, _ := ledger.Ban( limit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"), netip.MustParsePrefix("203.0.113.1/32"),
midnight(), midnight(),
bans.Grounds{ bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
) )
byteLimit, _ := ledger.Ban( byteLimit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.3/32"), netip.MustParsePrefix("203.0.113.3/32"),
midnight(), midnight(),
bans.Grounds{ bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
) )
attack, _ := ledger.Ban( attack, _ := ledger.BanForAttack(
netip.MustParsePrefix("203.0.113.2/32"), netip.MustParsePrefix("203.0.113.2/32"),
midnight(), midnight(),
bans.Grounds{ bans.Notes{RuleID: "git-dir", Target: "path"},
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
) )
for _, tc := range []struct{ got, want string }{ for _, tc := range []struct{ got, want string }{
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses") t.Error("the lifted ban refuses")
} }
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour || if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf( t.Errorf(
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
} }
// The next clear sign of attack bans for seven days, as a first does. // The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.Ban( ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
if ban.Expires.Sub(ban.Start) != 7*day { if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf( t.Errorf(
"the next ban for an attack ends at %s, want seven days on", "the next ban for an attack ends at %s, want seven days on",
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel() t.Parallel()
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
made, _ := ledger.Ban( made, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"), netip.MustParsePrefix("203.0.113.1/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
atStart := bans.Ban{ atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"), Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
// An hour's ban for a broken limit. // An hour's ban for a broken limit.
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an // A minute later an admin bans the netblock for good, named by an
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first. // The temporary ban is made first, so that its netblock is looked at first.
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
ledger.BanForAdmin( ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"), netip.MustParsePrefix("203.0.113.0/24"),
+82 -54
View File
@@ -300,50 +300,81 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest return newest
} }
// Grounds are why a request makes a ban: its cause, CauseLimit or // BanForLimit bans netblock at now for a broken limit, with notes, and
// CauseAttack, and its notes. // returns the ban, and true. A first ban lasts LimitBanDuration. A ban
type Grounds struct { // made within LimitBanRepeatWindow after the netblock's ban that ended
Cause string // last, other than one for a clear sign of attack or a lifted one, lasts
Notes Notes // repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
} }
// Ban bans netblock at now on grounds, and returns the ban, and true. If // WouldBanForLimit returns what BanForLimit would, without making the ban:
// a ban on netblock is still active, as when two of its requests break a // what observe mode would have done.
// limit at once, that ban is returned with false, and no other is made. func (l *Ledger) WouldBanForLimit(
// The ledger fills in the notes' Refused and EarlierBans itself. netblock netip.Prefix, now time.Time, notes Notes,
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
) (Ban, bool) { ) (Ban, bool) {
return l.ban(netblock, now, grounds, true) return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
} }
// WouldBan returns what Ban would, without making the ban: what observe // BanForAttack bans netblock at now for a clear sign of attack, with
// mode would have done. // notes, and returns the ban, and whether it made it, as BanForLimit
func (l *Ledger) WouldBan( // does. A first ban lasts AttackBanDuration; once the netblock has had
netblock netip.Prefix, now time.Time, grounds Grounds, // one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) { ) (Ban, bool) {
return l.ban(netblock, now, grounds, false) return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
} }
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit // WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as Ban would make it. It // or CauseAttack, made at now would be permanent, as BanForLimit or
// works out nothing else of the ban. // BanForAttack would make it. It works out nothing else of the ban.
func (l *Ledger) WouldBePermanent( func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause string, netblock netip.Prefix, now time.Time, cause string,
) bool { ) bool {
@@ -380,9 +411,10 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until // BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose // expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another // cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// on netblock is active, since the admin asked for this one. The ledger // ban even while another on netblock is active, since the admin asked
// fills in the notes' EarlierBans, and counts the ban among those made. // for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
func (l *Ledger) BanForAdmin( func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string, netblock netip.Prefix, now, expires time.Time, reason string,
) Ban { ) Ban {
@@ -727,13 +759,15 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
return forgot return forgot
} }
// ban bans netblock at now on grounds, as Ban describes, and returns the // ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// ban, and whether it made it. Unless keep is true, the ban is not made, // reason and notes, as BanForLimit and BanForAttack describe, and returns
// only returned: it is the ban that would have been made. // the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
func (l *Ledger) ban( func (l *Ledger) ban(
netblock netip.Prefix, netblock netip.Prefix,
now time.Time, now time.Time,
grounds Grounds, cause, reason string,
notes Notes,
keep bool, keep bool,
) (Ban, bool) { ) (Ban, bool) {
l.mu.Lock() l.mu.Lock()
@@ -742,8 +776,6 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active. // held are the netblock's bans, none of them active.
var held []Ban var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock) bans, found := l.netblocks.Get(netblock)
if found { if found {
active := activeBan(*bans, now) active := activeBan(*bans, now)
@@ -759,19 +791,15 @@ func (l *Ledger) ban(
ban := Ban{ ban := Ban{
Netblock: netblock, Netblock: netblock,
Start: now, Start: now,
Cause: grounds.Cause, Cause: cause,
Reason: reason,
Notes: notes, Notes: notes,
} }
switch grounds.Cause { if cause == CauseAttack {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
ban.Expires = l.attackExpiry(held, now) ban.Expires = l.attackExpiry(held, now)
default: } else {
panic("bans: a request's ban has the cause " + grounds.Cause) ban.Expires = l.limitExpiry(held, now)
} }
if !keep { if !keep {
@@ -779,7 +807,7 @@ func (l *Ledger) ban(
} }
l.add(ban) l.add(ban)
l.made[grounds.Cause]++ l.made[cause]++
l.markChanged() l.markChanged()
return ban, true return ban, true
+51 -121
View File
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and // Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours. // 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} { for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
length := time.Duration(hours) * time.Hour length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) || if !ban.Expires.Equal(now.Add(length)) ||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is // The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends. // permanent, and never ends.
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() { if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires) t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
} }
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
second, _ := ledger.Ban( second, _ := ledger.BanForLimit(
netblock, netblock,
first.Expires.Add(tc.gap), first.Expires.Add(tc.gap),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if second.Expires.Sub(second.Start) != tc.want || if second.Expires.Sub(second.Start) != tc.want ||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params) ledger := bans.New(params)
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if !ban.Permanent() { if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires) t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight() now := midnight()
for i := range 14 { for i := range 14 {
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if !ban.Expires.After(ban.Start) { if !ban.Expires.After(ban.Start) {
t.Fatalf( t.Fatalf(
"ban %d starts at %s and ends at %s", "ban %d starts at %s and ends at %s",
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires now = ban.Expires
} }
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() { if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires) t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
} }
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.Ban( first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
if !made { if !made {
t.Error("the first ban was not made") t.Error("the first ban was not made")
} }
again, made := ledger.Ban( again, made := ledger.BanForLimit(
netblock, netblock,
midnight().Add(time.Minute), midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if made || !reflect.DeepEqual(again, first) || if made || !reflect.DeepEqual(again, first) ||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
) )
} }
again, made = ledger.Ban( again, made = ledger.BanForAttack(
netblock, netblock,
midnight().Add(time.Minute), midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseAttack}, bans.Notes{},
) )
if made || !reflect.DeepEqual(again, first) { if made || !reflect.DeepEqual(again, first) {
t.Errorf( t.Errorf(
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}}, bans.Notes{Requests: 5},
) )
for range 3 { for range 3 {
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}}, bans.Notes{Requests: 5},
) )
got, banned, _ := ledger.Find( got, banned, _ := ledger.Find(
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64") d := netip.MustParsePrefix("2001:db8::/64")
now := midnight() now := midnight()
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit}) first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(b, now, bans.Notes{})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(c, now, bans.Notes{})
// A request from a makes b the netblock seen longest ago, and its ban // A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's. // goes to make room for d's.
ledger.Check(a.Addr(), now) ledger.Check(a.Addr(), now)
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(d, now, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1}) wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes. // a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(a, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1}) wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes // With d seen since, a is seen longest ago, and its earlier ban goes
// first. // first.
ledger.Check(d.Addr(), first.Expires) ledger.Check(d.Addr(), first.Expires)
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(b, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1}) wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) { if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params) ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock, second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
held := ledger.Bans(netblock) held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) || if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"} notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.Ban( ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
if !ban.Expires.Equal(midnight().Add(7*day)) || if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack || ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" || ban.Notes.RuleID != "env-file" ||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count. // A ban for a broken limit before does not count.
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock, second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
if second.Expires.Sub(second.Start) != 7*day { if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days", t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent, // Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit. // beside the one for a limit.
third, _ := ledger.Ban( third, _ := ledger.BanForAttack(
netblock, netblock,
second.Expires.Add(30*day), second.Expires.Add(30*day),
bans.Grounds{Cause: bans.CauseAttack}, bans.Notes{},
) )
if !third.Permanent() || if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) { third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
// While the first ban lasts, none would be made. // While the first ban lasts, none would be made.
during, would := ledger.WouldBan( during, would := ledger.WouldBanForAttack(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseAttack}, bans.Notes{},
) )
if would || !reflect.DeepEqual(during, first) { if would || !reflect.DeepEqual(during, first) {
t.Errorf( t.Errorf(
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a // As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made. // limit broken again for three hours, but neither is made.
limitGrounds := bans.Grounds{ limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
Cause: bans.CauseLimit, attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}, bans.Notes{RuleID: "git-dir"})
} limit, wouldLimit := ledger.WouldBanForLimit(
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
netblock, netblock,
first.Expires, first.Expires,
limitGrounds, limitNotes,
) )
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) || if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
// The ban made is the one that would have been. // The ban made is the one that would have been.
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds) made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
if !reflect.DeepEqual(made, limit) { if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit) t.Errorf("the ban made is %+v, want %+v", made, limit)
} }
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1) t.Fatalf("ban %d for a limit would be permanent", i+1)
} }
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
now = ban.Expires now = ban.Expires
} }
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
} }
// Once a first ban for an attack has ended, the next would be permanent. // Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.Ban( attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) { if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent") t.Error("a second ban for an attack would not be permanent")
} }
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the // Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does. // ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.Ban( attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
netblock, limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if limit.Expires.Sub(limit.Start) != time.Hour || if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit { limit.Cause != bans.CauseLimit {
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long, Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
} }
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{ bans.Notes{Request: request},
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
) )
cut := long[:256] cut := long[:256]
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after // A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock. // the first ended, and one on another netblock.
ledger.Ban(netblock, midnight(), bans.Grounds{ ledger.BanForLimit(netblock, midnight(), bans.Notes{
Cause: bans.CauseLimit, ASN: "AS64497", ASName: "Other Net", Country: "FR",
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
}) })
ledger.Ban( ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
netblock, ledger.BanForLimit(other, midnight(), bans.Notes{})
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE") ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+21 -36
View File
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes // A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write. // only the counts in the notes, which wait for the interval's write.
ledger.Ban( ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Check(netblock.Addr(), midnight().Add(time.Minute)) ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
// Two bans before the value is read leave one. // Two bans before the value is read leave one.
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"), netip.MustParsePrefix("203.0.113.10/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.11/32"), netip.MustParsePrefix("203.0.113.11/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32") high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32") low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
v6, ledger.BanForLimit(high, midnight(), bans.Notes{})
midnight(), ledger.BanForLimit(low, midnight(), bans.Notes{})
bans.Grounds{Cause: bans.CauseLimit}, ledger.BanForLimit(v6, first.Expires, bans.Notes{})
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
snapshot := ledger.Snapshot() snapshot := ledger.Snapshot()
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams()) before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{ ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
// Loaded into a new ledger, as across a restart, the ban still refuses // Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three // while it lasts, and once it has ended a broken limit bans for three
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse") t.Error("the loaded ban does not refuse")
} }
again, _ := after.Ban( again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if again.Expires.Sub(again.Start) != 3*time.Hour || if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf( t.Errorf(
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(), Start: midnight(),
}, },
}) })
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"), netip.MustParsePrefix("198.51.100.7/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
for client, want := range map[string]bool{ for client, want := range map[string]bool{
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
} }
// A limit broken now makes no shorter ban over the permanent one. // A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 { if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf( t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+ "a broken limit returned %+v and left the netblock %d bans, "+
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans // Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans // for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's. // before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
nineHours.Expires.Add(time.Hour), nineHours.Expires.Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if ban.Expires.Sub(ban.Start) != 27*time.Hour || if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) { ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
} }
// The ledger holds one ban, so it makes two more without dropping any. // The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"), netip.MustParsePrefix("198.51.100.7/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
second, _ := ledger.Ban( second, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.8/32"), netip.MustParsePrefix("198.51.100.8/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
want := []bans.Ban{first, second, kept} want := []bans.Ban{first, second, kept}
+60 -37
View File
@@ -84,12 +84,7 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every // one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use. // client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct { type GeoJS struct {
url string params Params
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to // httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure. // GeoJS alone: a redirect is a failure.
httpClient *http.Client httpClient *http.Client
@@ -99,8 +94,13 @@ type GeoJS struct {
// waiting are the clients without an answer: those to ask GeoJS about, // waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about. // and those it is being asked about.
waiting map[netip.Prefix]*wait waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way. // asking is true while a request to GeoJS is under way, and endAsking
asking bool // ends it. stopped is true once StopAsking has been called, and
// askGoroutines waits for the goroutine that makes the requests.
asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// retryDelay is how long GeoJS is left alone after its last failure, // retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again. // zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration retryDelay time.Duration
@@ -140,12 +140,7 @@ func New(params Params) *GeoJS {
} }
return &GeoJS{ return &GeoJS{
url: params.URL, params: params,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{ httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse return http.ErrUseLastResponse
@@ -188,7 +183,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client) answer, found := g.kept(client)
if !found { if !found {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
} }
w, waiting := g.waiting[client] w, waiting := g.waiting[client]
@@ -210,6 +205,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
return answer return answer
} }
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS. // Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock() g.mu.Lock()
@@ -251,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge() g.answers.Purge()
now := g.now() now := g.params.Now()
for _, answer := range answers { for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor { if now.Sub(answer.Answered) < keepFor {
@@ -288,18 +300,19 @@ func (g *GeoJS) answerOrWait(
} }
if w == nil { if w == nil {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
return Answer{}, nil // too many clients wait already return Answer{}, nil // too many clients wait already
} }
if !g.asking { if !g.asking {
// GeoJS is left alone after a failure, so no answer can come. // GeoJS is left alone after a failure, or asked nothing after
// StopAsking, so no answer can come.
w.late = true w.late = true
} }
if w.late { if w.late {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
return Answer{}, nil return Answer{}, nil
} }
@@ -310,7 +323,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago, // kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used. // and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.now() now := g.params.Now()
kept, found := g.answers.Get(client) kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor { if !found || now.Sub(kept.Answered) >= keepFor {
@@ -323,22 +336,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
} }
// ask starts asking GeoJS about the waiting clients, unless a request to // ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way or it is left alone after a failure. The requests to // it is under way, it is left alone after a failure, or StopAsking has
// GeoJS are for every client waiting, so they go on when the caller whose // been called. The requests to GeoJS are for every client waiting, so they
// ctx is given stops waiting. // go on when the caller whose ctx is given stops waiting, until StopAsking.
func (g *GeoJS) ask(ctx context.Context) { func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.now().Before(g.retryAt) { if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
return return
} }
g.asking = true askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
g.asking, g.endAsking = true, end
go g.askAboutWaiting(context.WithoutCancel(ctx)) g.askGoroutines.Go(func() {
defer end()
g.askAboutWaiting(askCtx)
})
} }
// askAboutWaiting asks GeoJS about the waiting clients, one request at a // askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left or GeoJS fails. Each answer kept is given to // time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
// Answered, outside the lock, since Answered takes locks of its own. // answer kept is given to Answered, outside the lock, since Answered takes
// locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) { func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for { for {
clients := g.nextClients() clients := g.nextClients()
@@ -347,11 +366,15 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
} }
given, err := g.request(ctx, clients) given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err) kept, answered := g.keep(clients, given, err)
if g.answered != nil { if g.params.Answered != nil {
for _, answer := range kept { for _, answer := range kept {
g.answered(answer) g.params.Answered(answer)
} }
} }
@@ -362,13 +385,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
} }
// nextClients returns up to maxPerRequest of the waiting clients. When // nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, it returns none and notes that no request to GeoJS is // none is waiting, or StopAsking has been called, it returns none and
// under way. // notes that no request to GeoJS is under way.
func (g *GeoJS) nextClients() []netip.Prefix { func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
if len(g.waiting) == 0 { if len(g.waiting) == 0 || g.stopped {
g.asking = false g.asking = false
return nil return nil
@@ -400,7 +423,7 @@ func (g *GeoJS) keep(
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
now := g.now() now := g.params.Now()
kept := make([]Answer, 0, len(clients)) kept := make([]Answer, 0, len(clients))
leftOut := 0 leftOut := 0
@@ -444,7 +467,7 @@ func (g *GeoJS) keep(
w.late = true w.late = true
} }
g.failed(err, g.retryDelay) g.params.Failed(err, g.retryDelay)
return kept, false return kept, false
} }
@@ -474,7 +497,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext( req, err := http.NewRequestWithContext(
ctx, ctx,
http.MethodGet, http.MethodGet,
g.url, g.params.URL,
http.NoBody, http.NoBody,
) )
if err != nil { if err != nil {
@@ -483,7 +506,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.asked() g.params.Asked()
res, err := g.httpClient.Do(req) res, err := g.httpClient.Do(req)
if err != nil { if err != nil {
+38
View File
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
}) })
} }
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
service, _, g, reports := startWithReports()
clients := newClients()
service.set(hanging)
g.LookUpWithoutWaiting(t.Context(), clients())
waitForRequests(t, service, 1)
// The request is ended, not waited out, and is no failure.
began := time.Now()
g.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
reports.wantFailures(t)
// A new client is then not asked about, and its lookup does not
// wait.
service.set(answering)
wantCountry(t, g, clients(), "")
if waited := time.Since(began); waited != 0 {
t.Errorf("the lookup returned after %s, want at once", waited)
}
waitForRequests(t, service, 1)
reports.wantFailures(t)
})
}
func TestRequestOver10SecondsIsAFailure(t *testing.T) { func TestRequestOver10SecondsIsAFailure(t *testing.T) {
t.Parallel() t.Parallel()
+44
View File
@@ -0,0 +1,44 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
+105
View File
@@ -0,0 +1,105 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+7 -4
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) { func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32") other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight() start := midnight()
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
for client, requests := range map[string]int{ for client, requests := range map[string]int{
"198.51.100.9/32": 2, "198.51.100.9/32": 2,
+18 -6
View File
@@ -156,16 +156,28 @@ type Request struct {
BlockedByCoreRuleSet bool BlockedByCoreRuleSet bool
} }
// New returns a Limiter for limits, with no client counted yet, whose // Params are what New needs.
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past type Params struct {
// it, the least recently seen client is dropped, with its history, and // Limits are what each client's requests and bytes are counted against.
// starts afresh if it comes back. Limits Limits
func New(limits Limits, maxClients int) *Limiter { // MaxClients is the most clients the table holds,
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil) // SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
if err != nil { if err != nil {
panic(err) // NewLRU fails only for a size below one panic(err) // NewLRU fails only for a size below one
} }
limits := params.Limits
return &Limiter{ return &Limiter{
windows: [3]window{ windows: [3]window{
{ {
+63 -39
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight() now := midnight()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
tableSize, MaxClients: tableSize,
) })
if limiter.OverARateLimit(client, now) { if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit") t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute) wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over. // A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Limits{}, tableSize) off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
off.Count(client, now, whole) off.Count(client, now, whole)
if off.OverARateLimit(client, now) { if off.OverARateLimit(client, now) {
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(tc.limits, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
quarter := tc.length / 4 quarter := tc.length / 4
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerHour: limit}, Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}, Limits: ratelimit.Limits{
tableSize, PerMinute: 5,
) BytesPerDay: math.MaxInt64,
},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel() t.Parallel()
// Only the hour has limits: the minute's and the day's are off. // Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(tc.limits, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it. // 600 bytes are within the limit, 600 more over it.
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32") other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) { ) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerDay: limit}, Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
tableSize, MaxClients: tableSize,
) })
refused := netip.MustParsePrefix("203.0.113.9/32") refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32") within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3 const maxClients = 3
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
now := midnight() now := midnight()
clients := make([]netip.Prefix, maxClients+1) clients := make([]netip.Prefix, maxClients+1)
+12 -6
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64", "2001:db8::/64",
} }
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
for _, i := range []int{2, 3, 0, 1} { for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
} }
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) before := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
for range limit { for range limit {
wantCount(t, before, client, start, "") wantCount(t, before, client, start, "")
} }
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no // Loaded into a new limiter, as across a restart, the client has no
// fresh allowance. // fresh allowance.
later := start.Add(time.Minute) later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) after := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after.Load(before.Snapshot(), later) after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour) wantCount(t, after, client, later, hour)
} }
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter.Count(client, start, whole) limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole) limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit) limiter.CountRefusal(client, start, limit)
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client { loaded := func(now time.Time) ratelimit.Client {
t.Helper() t.Helper()
after := ratelimit.New(ratelimit.Limits{}, tableSize) after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
after.Load(limiter.Snapshot(), now) after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0] return after.Snapshot()[0]
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next() addr = addr.Next()
} }
limiter := ratelimit.New(ratelimit.Limits{}, maxClients) limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
limiter.Load(clients, midnight()) limiter.Load(clients, midnight())
got := limiter.Snapshot() got := limiter.Snapshot()
+48 -16
View File
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
// scores are by client. Each is added as it is fetched and never moved // scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped. // up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Prefix, Score] scores *simplelru.LRU[netip.Prefix, Score]
// checking are the clients whose check is under way. // checking are the clients whose check is under way, each with the
checking map[netip.Prefix]bool // function that ends it. stopped is true once StopChecking has been
// called, and checkGoroutines waits for the goroutine of each check.
checking map[netip.Prefix]context.CancelFunc
stopped bool
checkGoroutines sync.WaitGroup
// day is the day, in UTC, of the checks spent counts. // day is the day, in UTC, of the checks spent counts.
day civil.Date day civil.Date
spent int spent int
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
params: params, params: params,
httpClient: &http.Client{}, httpClient: &http.Client{},
scores: scores, scores: scores,
checking: map[netip.Prefix]bool{}, checking: map[netip.Prefix]context.CancelFunc{},
} }
} }
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
// addresses its request comes from. A client without one is checked in // addresses its request comes from. A client without one is checked in
// the background, by addr, the address its request came from, if // the background, by addr, the address its request came from, if
// offender, if it has committed an offence, unless its check is under // offender, if it has committed an offence, unless its check is under
// way, a check failed less than failureDelay ago, or the day's checks // way, a check failed less than failureDelay ago, the day's checks have
// have used up DailyBudget; Hit never waits for a check. The check that // used up DailyBudget, or StopChecking has been called; Hit never waits
// uses the budget up is logged and raised as a source_failure alert. ctx // for a check. The check that uses the budget up is logged and raised as a
// is the context of the client's request, and a check goes on after the // source_failure alert. ctx is the context of the client's request, and a
// request ends. // check goes on after the request ends, until StopChecking.
func (a *AbuseIPDB) Hit( func (a *AbuseIPDB) Hit(
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool, ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
) (int64, bool) { ) (int64, bool) {
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
a.day, a.spent = today, 0 a.day, a.spent = today, 0
} }
check := offender && !a.checking[client] && !now.Before(a.retryAt) && _, checking := a.checking[client]
a.spent < a.params.DailyBudget
check := offender && !checking && !a.stopped &&
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
if check { if check {
a.checking[client] = true checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
a.checking[client] = end
a.checks++ a.checks++
a.spent++ a.spent++
go a.check(context.WithoutCancel(ctx), client, addr) a.checkGoroutines.Go(func() {
defer end()
a.check(checkCtx, client, addr)
})
} }
usedUp := check && a.spent == a.params.DailyBudget usedUp := check && a.spent == a.params.DailyBudget
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
return 0, false return 0, false
} }
// StopChecking ends the checks under way, and returns once their goroutines
// have. A check it ends gives no score and is no failure: it is neither
// counted, logged nor raised as an alert, and checks are not left alone
// after it. The ended check stays spent from the day's budget. After
// StopChecking, Hit checks no client.
func (a *AbuseIPDB) StopChecking() {
a.mu.Lock()
a.stopped = true
for _, end := range a.checking {
end()
}
a.mu.Unlock()
a.checkGoroutines.Wait()
}
// Checked returns how many checks were made. // Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int { func (a *AbuseIPDB) Checked() int {
a.mu.Lock() a.mu.Lock()
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
// check checks client with AbuseIPDB by addr, one of its addresses, keeps // check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A // the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a // check that fails gives no score: it is counted, logged and raised as a
// source_failure alert, and no client is checked for failureDelay. // source_failure alert, and no client is checked for failureDelay. A check
// ended as ctx ends, by StopChecking, is no failure.
func (a *AbuseIPDB) check( func (a *AbuseIPDB) check(
ctx context.Context, ctx context.Context,
client netip.Prefix, client netip.Prefix,
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
) { ) {
score, err := a.ask(ctx, addr) score, err := a.ask(ctx, addr)
now := a.params.Now() now := a.params.Now()
isFailure := err != nil && ctx.Err() == nil
a.mu.Lock() a.mu.Lock()
delete(a.checking, client) delete(a.checking, client)
if err == nil { switch {
case err == nil:
a.scores.Add( a.scores.Add(
client, client,
Score{Client: client, Score: score, Fetched: now}, Score{Client: client, Score: score, Fetched: now},
) )
} else { case isFailure:
a.failures++ a.failures++
a.retryAt = now.Add(failureDelay) a.retryAt = now.Add(failureDelay)
} }
a.mu.Unlock() a.mu.Unlock()
if err != nil { if isFailure {
a.alert("checking a client with AbuseIPDB failed", err) a.alert("checking a client with AbuseIPDB failed", err)
} }
} }
+37
View File
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
}) })
} }
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
// The check is ended, not waited out, and is no failure.
began := time.Now()
checker.StopChecking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopChecking returned after %s, want at once", waited)
}
wantFailures(t, checker, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// An offender without a score is then not checked.
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) { func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel() t.Parallel()
+44 -13
View File
@@ -82,8 +82,12 @@ type DNSBL struct {
// verdicts are by query. Each is added as it is fetched and never moved // verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped. // up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict] verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way. // asking are the queries under way, each with the function that ends
asking map[query]bool // it. stopped is true once StopAsking has been called, and
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// queries and failures count, by zone, the queries made and those that // queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be // failed, and retryAt is when a zone whose last query failed may be
// asked again. // asked again.
@@ -120,7 +124,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
params: params, params: params,
resolver: resolver, resolver: resolver,
verdicts: verdicts, verdicts: verdicts,
asking: map[query]bool{}, asking: map[query]context.CancelFunc{},
queries: map[string]int{}, queries: map[string]int{},
failures: map[string]int{}, failures: map[string]int{},
retryAt: map[string]time.Time{}, retryAt: map[string]time.Time{},
@@ -138,9 +142,10 @@ func (d *DNSBL) Zones() []string {
// log, the alerts and the metrics. A verdict is used until CacheTTL has // log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr // passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the // in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, or maxQueries are under way; // zone is left alone after a failure, maxQueries are under way, or
// ListedBy never waits for a query. ctx is the context of the client's // StopAsking has been called; ListedBy never waits for a query. ctx is the
// request, and a query goes on after the request ends. // context of the client's request, and a query goes on after the request
// ends, until StopAsking.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string { func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock() d.mu.Lock()
defer d.mu.Unlock() defer d.mu.Unlock()
@@ -153,23 +158,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
q := query{zone: zone, client: addr} q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q) kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch { switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL: case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed { if kept.Listed {
listedBy = append(listedBy, config.MaskZoneKey(zone)) listedBy = append(listedBy, config.MaskZoneKey(zone))
} }
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries: case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
d.asking[q] = true len(d.asking) < maxQueries:
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
d.queries[zone]++ d.queries[zone]++
go d.ask(context.WithoutCancel(ctx), q) d.queryGoroutines.Go(func() {
defer end()
d.ask(queryCtx, q)
})
} }
} }
return listedBy return listedBy
} }
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone. // Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int { func (d *DNSBL) Queries(zone string) int {
d.mu.Lock() d.mu.Lock()
@@ -238,27 +267,29 @@ func (d *DNSBL) Load(verdicts []Verdict) {
// query as no longer under way. A query that fails gives no verdict: it // query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the // is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for // zone with its key masked, and the zone is not asked again for
// failureDelay. // failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
func (d *DNSBL) ask(ctx context.Context, q query) { func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q) listed, err := d.lookUp(ctx, q)
now := d.params.Now() now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock() d.mu.Lock()
delete(d.asking, q) delete(d.asking, q)
if err == nil { switch {
case err == nil:
d.verdicts.Add(q, Verdict{ d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now, Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
}) })
} else { case isFailure:
d.failures[q.zone]++ d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay) d.retryAt[q.zone] = now.Add(failureDelay)
} }
d.mu.Unlock() d.mu.Unlock()
if err != nil { if isFailure {
const failed = "asking a DNSBL zone failed" const failed = "asking a DNSBL zone failed"
shown := config.MaskZoneKey(q.zone) shown := config.MaskZoneKey(q.zone)
+39
View File
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
}) })
} }
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := dnsblParams(zone)
p.Alerts = queue
resolver := &resolverStandIn{hanging: true}
dnsbl := newDNSBL(resolver, p)
wantZones(t, dnsbl, listed)
synctest.Wait()
// The query is ended, not waited out, and is no failure.
began := time.Now()
dnsbl.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
wantQueries(t, dnsbl, 1, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// A client without a verdict is then asked about in no zone.
wantZones(t, dnsbl, unlisted)
synctest.Wait()
wantQueries(t, dnsbl, 1, 0)
wantAsked(t, resolver, listedName)
waitForTheResolver()
})
}
// //
//nolint:paralleltest // one at a time, as the comment at the top of this file says //nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute( func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
+4 -33
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the lines sww prints on stdout: one // Package requestlog writes the request log sww prints on stdout: JSON
// JSON object per request, two from waf and a short one from rproxy, // lines marked "type":"request", at most two from waf and a short one
// marked "type":"request", and the process's own messages as JSON lines marked // from rproxy for each request. Package processlog writes the process's
// "type":"process". // own messages.
package requestlog package requestlog
import ( import (
@@ -9,7 +9,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log/slog"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond, time.Millisecond/time.Microsecond,
) )
} }
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
-87
View File
@@ -3,8 +3,6 @@ package requestlog_test
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"log/slog"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want) t.Errorf("wrote %s, want %s", out.String(), want)
} }
} }
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+43 -12
View File
@@ -99,8 +99,12 @@ type Client struct {
// answers are by client. Each is added as it comes and never moved up, // answers are by client. Each is added as it comes and never moved up,
// so that the one that came longest ago is the first dropped. // so that the one that came longest ago is the first dropped.
answers *simplelru.LRU[netip.Prefix, Answer] answers *simplelru.LRU[netip.Prefix, Answer]
// asking are the clients whose query is under way. // asking are the clients whose query is under way, each with the
asking map[netip.Prefix]bool // function that ends it. stopped is true once StopAsking has been
// called, and queryGoroutines waits for the goroutine of each query.
asking map[netip.Prefix]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// retryAt is when routewatch may be asked again after a query failed. // retryAt is when routewatch may be asked again after a query failed.
retryAt time.Time retryAt time.Time
} }
@@ -116,7 +120,7 @@ func New(params Params) *Client {
params: params, params: params,
httpClient: &http.Client{}, httpClient: &http.Client{},
answers: answers, answers: answers,
asking: map[netip.Prefix]bool{}, asking: map[netip.Prefix]context.CancelFunc{},
} }
} }
@@ -124,8 +128,9 @@ func New(params Params) *Client {
// IPv6 group, or why it has no route. It never waits for a query. A client // IPv6 group, or why it has no route. It never waits for a query. A client
// without an answer is asked about in the background, by addr, the address its // without an answer is asked about in the background, by addr, the address its
// request came from, unless its query is under way, a query failed less than // request came from, unless its query is under way, a query failed less than
// failureKept ago, or maxQueries are under way. ctx is the context of the // failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
// client's request, and a query goes on after the request ends. // is the context of the client's request, and a query goes on after the request
// ends, until StopAsking.
func (c *Client) Route( func (c *Client) Route(
ctx context.Context, client netip.Prefix, addr netip.Addr, ctx context.Context, client netip.Prefix, addr netip.Addr,
) (Route, Missing) { ) (Route, Missing) {
@@ -147,15 +152,38 @@ func (c *Client) Route(
return Route{}, RequestFailed return Route{}, RequestFailed
} }
if !c.asking[client] && len(c.asking) < maxQueries { _, asking := c.asking[client]
c.asking[client] = true if !asking && !c.stopped && len(c.asking) < maxQueries {
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
c.asking[client] = end
go c.ask(context.WithoutCancel(ctx), client, addr) c.queryGoroutines.Go(func() {
defer end()
c.ask(queryCtx, client, addr)
})
} }
return Route{}, NoAnswerYet return Route{}, NoAnswerYet
} }
// StopAsking ends the queries under way, and returns once their goroutines
// have. A query it ends gives no answer and is no failure: it is not logged,
// and routewatch is not left alone after it. After StopAsking, Route asks
// routewatch nothing.
func (c *Client) StopAsking() {
c.mu.Lock()
c.stopped = true
for _, end := range c.asking {
end()
}
c.mu.Unlock()
c.queryGoroutines.Wait()
}
// Snapshot returns every answer still in use, sorted by client, as // Snapshot returns every answer still in use, sorted by client, as
// routewatch.json lists them. // routewatch.json lists them.
func (c *Client) Snapshot() []Answer { func (c *Client) Snapshot() []Answer {
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
// ask asks routewatch about addr, keeps the answer as client's, and notes the // ask asks routewatch about addr, keeps the answer as client's, and notes the
// query as no longer under way. A query that fails gives no answer, and // query as no longer under way. A query that fails gives no answer, and
// routewatch is not asked again for failureKept. Only the first failure of // routewatch is not asked again for failureKept. Only the first failure of
// those under way is logged. // those under way is logged. A query ended as ctx ends, by StopAsking, is no
// failure.
func (c *Client) ask( func (c *Client) ask(
ctx context.Context, ctx context.Context,
client netip.Prefix, client netip.Prefix,
@@ -213,18 +242,20 @@ func (c *Client) ask(
) { ) {
route, err := c.query(ctx, addr) route, err := c.query(ctx, addr)
now := c.params.Now() now := c.params.Now()
isFailure := err != nil && ctx.Err() == nil
c.mu.Lock() c.mu.Lock()
delete(c.asking, client) delete(c.asking, client)
firstFailure := err != nil && !now.Before(c.retryAt) firstFailure := isFailure && !now.Before(c.retryAt)
if err == nil { switch {
case err == nil:
c.answers.Add(client, Answer{ c.answers.Add(client, Answer{
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now, Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
}) })
} else { case isFailure:
c.retryAt = now.Add(failureKept) c.retryAt = now.Add(failureKept)
} }
+48
View File
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
}) })
} }
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
// says.
synctest.Test(t, func(t *testing.T) {
f := &fixture{t: t, log: &syncBuffer{}}
f.client = routewatch.New(routewatch.Params{
URL: silentURL, Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
})
f.route()
synctest.Wait()
// The query is ended, not waited out.
began := time.Now()
f.client.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
// A query started now would fail within waitLimit. None is, and the
// one ended is no failure: nothing is logged, and routewatch is not
// left alone.
for range 2 {
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
t.Errorf(
"Route gave %q, want %q",
missing,
routewatch.NoAnswerYet,
)
}
time.Sleep(waitLimit)
synctest.Wait()
}
if f.log.String() != "" {
t.Errorf("process log %q, want nothing", f.log.String())
}
})
}
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) { func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
t.Parallel() t.Parallel()
+6 -6
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
io.Discard, Output: io.Discard,
"test", InstanceName: "test",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
+6 -6
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -370,11 +370,11 @@ func startServerLogging(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
processLog, Output: processLog,
"test", InstanceName: "test",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
+19 -31
View File
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the // A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it. // write off no further, and is written with it.
first, _ := params.Ledger.Ban( first, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
second, _ := params.Ledger.Ban( second, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"), netip.MustParsePrefix("203.0.113.10/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
time.Sleep(5*time.Second - time.Nanosecond) time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run) run(t, load(t, params).Run)
// A ban, and bans.json written with it. // A ban, and bans.json written with it.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"), params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit}) midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay) time.Sleep(params.BansWriteDelay)
synctest.Wait() synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before // A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and // then, an admin saves bans.json with the first ban lifted and
// another added. // another added.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"), params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit}) midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay - time.Nanosecond) time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait() synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON) edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err) t.Fatalf("mkdir: %v", err)
} }
params.Ledger.Ban( params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
params.Limiter.Count( params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err) t.Fatalf("mkdir: %v", err)
} }
params.Ledger.Ban( params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
err = files.WriteAll() err = files.WriteAll()
@@ -2135,11 +2135,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted. // The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32") netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.Ban( params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting()) _, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned { if !banned {
@@ -2452,7 +2448,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour, AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000, MaxBans: 5000,
}), }),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000), Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
GeoJS: geojs.New(geojs.Params{Now: midnight}), GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{ Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{ BlocklistURLs: []string{
@@ -2514,15 +2510,11 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()}) params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.Ban(client, now, bans.Grounds{ params.Ledger.BanForLimit(client, now, bans.Notes{
Cause: bans.CauseLimit, ASN: asn, ASName: asName, Country: "DE", Limit: 1,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
}) })
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now, params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{ bans.Notes{RuleID: "env-file", Target: "path"})
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole) params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2708,11 +2700,7 @@ func wantLiftedBanKept(
} }
// Were the lifted ban counted, the next would last three hours. // Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
if ban.Expires.Sub(ban.Start) != time.Hour { if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf( t.Errorf(
"the next ban lasts %s, want 1h", "the next ban lasts %s, want 1h",
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
) )
@@ -23,11 +23,11 @@ import (
// setting for either. // setting for either.
func RunRProxy(ctx context.Context, params Params) int { func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := requestlog.NewProcessLogger( processLog := processlog.New(processlog.Params{
params.Stdout, Output: params.Stdout,
hostname, InstanceName: hostname,
slog.LevelInfo, Level: slog.LevelInfo,
) })
cfg, err := config.RProxyFromEnvironment(params.Viper) cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil { if err != nil {
+58 -28
View File
@@ -19,10 +19,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state" "sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int { func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's // Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through. // error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := requestlog.NewProcessLogger(params.Stdout, processLog := processlog.New(processlog.Params{
config.InstanceName(params.Viper), slog.LevelError) Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
cfg, err := config.WAFFromEnvironment(params.Viper) cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil { if err != nil {
@@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote) stdout = io.MultiWriter(params.Stdout, remote)
} }
processLog = requestlog.NewProcessLogger( processLog = processlog.New(processlog.Params{
stdout, Output: stdout,
cfg.InstanceName, InstanceName: cfg.InstanceName,
cfg.LogLevel, Level: cfg.LogLevel,
) })
if remote != nil { if remote != nil {
stopSending := startSending(ctx, remote, processLog) stopSending := startSending(ctx, remote, processLog)
@@ -118,7 +121,8 @@ func RunWAF(ctx context.Context, params Params) int {
"settings", cfg) "settings", cfg)
// The last write follows the stop of every hook, and so the end of every // The last write follows the stop of every hook, and so the end of every
// loop that changes the state or raises an alert. A hook of the state // loop that changes the state or raises an alert, and of every ask of
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
// files would stop before the loops of the parts built ahead of them. // files would stop before the loops of the parts built ahead of them.
return run(ctx, app, failed, processLog, func() error { return run(ctx, app, failed, processLog, func() error {
err := files.WriteAll() err := files.WriteAll()
@@ -238,14 +242,17 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits. // newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter { func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Limits{ return ratelimit.New(ratelimit.Params{
PerMinute: cfg.RateLimitPerMinute, Limits: ratelimit.Limits{
PerHour: cfg.RateLimitPerHour, PerMinute: cfg.RateLimitPerMinute,
PerDay: cfg.RateLimitPerDay, PerHour: cfg.RateLimitPerHour,
BytesPerMinute: cfg.BytesLimitPerMinute, PerDay: cfg.RateLimitPerDay,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerDay: cfg.BytesLimitPerDay, BytesPerHour: cfg.BytesLimitPerHour,
}, cfg.MaxTrackedClients) BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
} }
// newLedger returns the ledger of the bans, with the settings for them. // newLedger returns the ledger of the bans, with the settings for them.
@@ -297,11 +304,13 @@ func newLists(
return lists return lists
} }
// newDNSBL returns what asks the DNSBL zones, none asked yet. // newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
// ends the queries under way, as reputation.DNSBL.StopAsking says.
func newDNSBL( func newDNSBL(
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue, lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) *reputation.DNSBL { ) *reputation.DNSBL {
return reputation.NewDNSBL(reputation.DNSBLParams{ dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones, Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver, Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL, CacheTTL: cfg.IPReputationCacheTTL,
@@ -310,12 +319,17 @@ func newDNSBL(
ProcessLog: processLog, ProcessLog: processLog,
Alerts: alertQueue, Alerts: alertQueue,
}) })
lc.Append(fx.StopHook(dnsbl.StopAsking))
return dnsbl
} }
// newAbuseIPDB returns what checks clients with AbuseIPDB while // newAbuseIPDB returns what checks clients with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it // SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in // fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
// ledger, as waf.BanAbuseIPDBBlacklist says. // ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
// under way, as reputation.AbuseIPDB.StopChecking says.
func newAbuseIPDB( func newAbuseIPDB(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger, lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue, ledger *bans.Ledger, alertQueue *alerts.Queue, ledger *bans.Ledger,
@@ -341,24 +355,30 @@ func newAbuseIPDB(
}) })
lc.Append(background(abuseIPDB.RunBlacklist)) lc.Append(background(abuseIPDB.RunBlacklist))
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
return abuseIPDB return abuseIPDB
} }
// newRouteWatch returns what asks routewatch about clients, or nil while // newRouteWatch returns what asks routewatch about clients, or nil while
// SWW_WAF_ROUTEWATCH_URL is unset. // SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
// way, as routewatch.Client.StopAsking says.
func newRouteWatch( func newRouteWatch(
cfg *config.WAF, processLog *slog.Logger, lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
) *routewatch.Client { ) *routewatch.Client {
if cfg.RouteWatchURL == "" { if cfg.RouteWatchURL == "" {
return nil return nil
} }
return routewatch.New(routewatch.Params{ routeWatch := routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL, URL: cfg.RouteWatchURL,
Now: utcNow, Now: utcNow,
ProcessLog: processLog, ProcessLog: processLog,
}) })
lc.Append(fx.StopHook(routeWatch.StopAsking))
return routeWatch
} }
// newMetrics returns the metrics, with those read from the other parts as // newMetrics returns the metrics, with those read from the other parts as
@@ -396,12 +416,18 @@ func newMetrics(
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each // newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each // answer goes to the limiter and the ledger, as waf.AddLookup says, and each
// failure is handled as waf.GeoJSFailed says. // failure is handled as waf.GeoJSFailed says. At the stop it ends the request
// under way, as geojs.GeoJS.StopAsking says.
func newGeoJS( func newGeoJS(
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter, lc fx.Lifecycle,
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger, cfg *config.WAF,
m *metrics.Metrics,
limiter *ratelimit.Limiter,
ledger *bans.Ledger,
alertQueue *alerts.Queue,
processLog *slog.Logger,
) *geojs.GeoJS { ) *geojs.GeoJS {
return geojs.New(geojs.Params{ geoJS := geojs.New(geojs.Params{
URL: geojs.URL, URL: geojs.URL,
Answered: func(answer geojs.Answer) { Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer) waf.AddLookup(cfg, limiter, ledger, answer)
@@ -413,6 +439,10 @@ func newGeoJS(
}, },
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc, WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
}) })
lc.Append(fx.StopHook(geoJS.StopAsking))
return geoJS
} }
// newWAFServer returns the server waf runs, with its parts and the time // newWAFServer returns the server waf runs, with its parts and the time
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"", "",
map[string]string{metricsToken: token}, map[string]string{metricsToken: token},
) )
server.Ledger.Ban( server.Ledger.BanForLimit(
netip.MustParsePrefix(otherClient+"/32"), netip.MustParsePrefix(otherClient+"/32"),
clk.Now(), clk.Now(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
before := server.Ledger.Snapshot() before := server.Ledger.Snapshot()
+5 -3
View File
@@ -130,9 +130,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make // A ban for a clear sign of attack, which a request under it would make
// permanent. // permanent.
group := netip.MustParsePrefix(ipv6Group) group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{ attackBan, _ := server.Ledger.BanForAttack(
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"}, group,
}) start,
bans.Notes{RuleID: "probe"},
)
// The third request breaks the limit, and so does the fourth, within the // The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack. // cooldown, which raises nothing. The probe is a clear sign of attack.
+2 -2
View File
@@ -686,10 +686,10 @@ func askDirectly(
// banClient bans client at now. // banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) { func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.Ban( server.Ledger.BanForLimit(
netip.MustParsePrefix(client+"/32"), netip.MustParsePrefix(client+"/32"),
now, now,
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
} }
+53 -72
View File
@@ -103,8 +103,9 @@ func (rq *request) countedBytes() int64 {
// one whose bytes broke a byte limit, the refusal for one that broke the // one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a // refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets // rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what // the client's counters back to zero. In observe mode it makes no ban and
// happens instead. // sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (rq *request) banForLimit( func (rq *request) banForLimit(
now time.Time, now time.Time,
hit ratelimit.Hit, hit ratelimit.Hit,
@@ -122,11 +123,23 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit) rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{ notes := bans.Notes{
Kind: hit.Kind, ASN: rq.line.ASN,
Limit: hit.Limit, ASName: rq.line.ASName,
Window: hit.Window, Country: rq.line.Country,
Count: hit.Count, Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
} }
switch hit.Kind { switch hit.Kind {
@@ -136,54 +149,47 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged() notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
} }
rq.banClient( if rq.h.config.Observe {
now, ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
status, if wouldBan {
bans.Grounds{Cause: bans.CauseLimit, Notes: notes}, rq.alertBan(ban)
) }
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
}
} }
// banForAttack bans the client's netblock at now for a clear sign of // banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path // attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient // asked for. It fills in the rest of the notes. In observe mode it makes
// says what happens instead. // no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) string {
netblock := banNetblock(rq.h.config, rq.client) netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
if rq.h.config.Observe { if rq.h.config.Observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) { ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
if wouldBan { if wouldBan {
rq.alertBan(ban) rq.alertBan(ban)
} }
@@ -191,13 +197,7 @@ func (rq *request) banClient(
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock) ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
rq.line.BanExpires = banExpires(ban) rq.line.BanExpires = banExpires(ban)
if made { if made {
@@ -207,25 +207,6 @@ func (rq *request) banClient(
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause // wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would // made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per // have made is worked out only then, at most once per
+22 -18
View File
@@ -24,6 +24,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -398,11 +399,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
io.Discard, Output: io.Discard,
"rproxy", InstanceName: "rproxy",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
@@ -471,11 +472,11 @@ func newWAFWith(
} }
out := &output{} out := &output{}
processLog := requestlog.NewProcessLogger( processLog := processlog.New(processlog.Params{
out, Output: out,
cfg.InstanceName, InstanceName: cfg.InstanceName,
cfg.LogLevel, Level: cfg.LogLevel,
) })
ruleFiles, err := rules.Load(rules.Params{ ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -561,14 +562,17 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs, cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName, cfg.InstanceName,
) )
params.Limiter = ratelimit.New(ratelimit.Limits{ params.Limiter = ratelimit.New(ratelimit.Params{
PerMinute: cfg.RateLimitPerMinute, Limits: ratelimit.Limits{
PerHour: cfg.RateLimitPerHour, PerMinute: cfg.RateLimitPerMinute,
PerDay: cfg.RateLimitPerDay, PerHour: cfg.RateLimitPerHour,
BytesPerMinute: cfg.BytesLimitPerMinute, PerDay: cfg.RateLimitPerDay,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerDay: cfg.BytesLimitPerDay, BytesPerHour: cfg.BytesLimitPerHour,
}, cfg.MaxTrackedClients) BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
params.Ledger = bans.New(bans.Params{ params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration, LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow, LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,