Compare commits

..
Author SHA1 Message Date
sneak 806ff7a2c0 Merge next: resolve the conflict with the exported setting names
check / check (push) Waiting to run
next now passes ASNBytesLimitPercentSetting to env.percents, and this
branch passes reputation.ParseASN in the same call. The call takes
both.

Model: opus-5-5
2026-11-19 18:32:25 +00:00
sneak b110cfabe8 config: test the header-name check on every single-byte name
check / check (push) Waiting to run
TestHeaderNameCheckTakesTheRFC9110Characters passed string(char) for
a byte, which for 0x80 to 0xff is the two-byte UTF-8 of U+0080 to
U+00FF. Half of the names it called single-byte were not checked.
It now builds each name from the byte itself.

Model: opus-5-5
2026-11-19 18:05:20 +00:00
sneak f9e034b885 No part imports config (closes #447)
check / check (push) Waiting to run
TagList, MaskZoneKey, ParseNetblock, ParseASN and ParsePercent move
from config to reputation; config imports them from there. rules and
config check header names with httpguts.ValidHeaderFieldName, and
IsHeaderName goes. A new test pins that check to the characters
IsHeaderName took, for the empty name and every single-byte name.

Model: opus-5-5
2026-11-19 17:39:34 +00:00
40 changed files with 997 additions and 1180 deletions

No files matched your search

+3 -7
View File
@@ -906,13 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each.
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
`internal/lookup` read their files again when they change.
- `internal/state`: the state and cache files.
- `internal/requestlog`: the request log.
- `internal/processlog`: the process's own messages.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
`internal/metrics`: the state and cache files, the log lines, the syslog
sender and the metrics.
## Rationale
+2 -7
View File
@@ -93,9 +93,6 @@ never as a time.
imports no other package of this repository: `waf` gives it what it needs
to count and report failures.
- `internal/lookup`: looks clients up in the lookup database.
- `internal/dirwatch`: watches a directory, and calls a function once it has
gone a short time without a change. `internal/rules` and `internal/lookup`
read their files again this way.
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
AbuseIPDB in the background, and keeps their answers. It also fetches
AbuseIPDB's blacklist, which `internal/waf` bans.
@@ -107,10 +104,8 @@ never as a time.
its own.
- `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop.
- `internal/requestlog`: the request log, as JSON lines on stdout.
- `internal/processlog`: the process lines on stdout, which hold the
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
sending them to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its
+4 -8
View File
@@ -54,11 +54,7 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
now := midnight()
scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made)
@@ -123,7 +119,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(limited, now, bans.Notes{})
// listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
@@ -148,7 +144,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
}
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(limited, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made)
@@ -158,7 +154,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
// The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes.
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
ledger.BanForAttack(attacker, now, bans.Notes{})
want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores {
+23 -64
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one.
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(second, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
}
// And the next drops that one.
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(third, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes.
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(other, second.Start, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
kept := first
kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams())
limit, _ := ledger.Ban(
limit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
)
byteLimit, _ := ledger.Ban(
byteLimit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.3/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
)
attack, _ := ledger.Ban(
attack, _ := ledger.BanForAttack(
netip.MustParsePrefix("203.0.113.2/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
bans.Notes{RuleID: "git-dir", Target: "path"},
)
for _, tc := range []struct{ got, want string }{
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses")
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
}
// The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf(
"the next ban for an attack ends at %s, want seven days on",
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
made, _ := ledger.Ban(
made, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams())
// An hour's ban for a broken limit.
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"),
+82 -54
View File
@@ -300,50 +300,81 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest
}
// Grounds are why a request makes a ban: its cause, CauseLimit or
// CauseAttack, and its notes.
type Grounds struct {
Cause string
Notes Notes
// BanForLimit bans netblock at now for a broken limit, with notes, and
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
// made within LimitBanRepeatWindow after the netblock's ban that ended
// last, other than one for a clear sign of attack or a lifted one, lasts
// repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
}
// Ban bans netblock at now on grounds, and returns the ban, and true. If
// a ban on netblock is still active, as when two of its requests break a
// limit at once, that ban is returned with false, and no other is made.
// The ledger fills in the notes' Refused and EarlierBans itself.
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
// WouldBanForLimit returns what BanForLimit would, without making the ban:
// what observe mode would have done.
func (l *Ledger) WouldBanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, true)
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
}
// WouldBan returns what Ban would, without making the ban: what observe
// mode would have done.
func (l *Ledger) WouldBan(
netblock netip.Prefix, now time.Time, grounds Grounds,
// BanForAttack bans netblock at now for a clear sign of attack, with
// notes, and returns the ban, and whether it made it, as BanForLimit
// does. A first ban lasts AttackBanDuration; once the netblock has had
// one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, false)
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
}
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as Ban would make it. It
// works out nothing else of the ban.
// or CauseAttack, made at now would be permanent, as BanForLimit or
// BanForAttack would make it. It works out nothing else of the ban.
func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause string,
) bool {
@@ -380,9 +411,10 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
// on netblock is active, since the admin asked for this one. The ledger
// fills in the notes' EarlierBans, and counts the ban among those made.
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// ban even while another on netblock is active, since the admin asked
// for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string,
) Ban {
@@ -727,13 +759,15 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
return forgot
}
// ban bans netblock at now on grounds, as Ban describes, and returns the
// ban, and whether it made it. Unless keep is true, the ban is not made,
// only returned: it is the ban that would have been made.
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
func (l *Ledger) ban(
netblock netip.Prefix,
now time.Time,
grounds Grounds,
cause, reason string,
notes Notes,
keep bool,
) (Ban, bool) {
l.mu.Lock()
@@ -742,8 +776,6 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active.
var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock)
if found {
active := activeBan(*bans, now)
@@ -759,19 +791,15 @@ func (l *Ledger) ban(
ban := Ban{
Netblock: netblock,
Start: now,
Cause: grounds.Cause,
Cause: cause,
Reason: reason,
Notes: notes,
}
switch grounds.Cause {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
if cause == CauseAttack {
ban.Expires = l.attackExpiry(held, now)
default:
panic("bans: a request's ban has the cause " + grounds.Cause)
} else {
ban.Expires = l.limitExpiry(held, now)
}
if !keep {
@@ -779,7 +807,7 @@ func (l *Ledger) ban(
}
l.add(ban)
l.made[grounds.Cause]++
l.made[cause]++
l.markChanged()
return ban, true
+51 -121
View File
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) ||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends.
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
}
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netblock,
first.Expires.Add(tc.gap),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if second.Expires.Sub(second.Start) != tc.want ||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params)
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight()
for i := range 14 {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Expires.After(ban.Start) {
t.Fatalf(
"ban %d starts at %s and ends at %s",
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
}
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
if !made {
t.Error("the first ban was not made")
}
again, made := ledger.Ban(
again, made := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) ||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
)
}
again, made = ledger.Ban(
again, made = ledger.BanForAttack(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) {
t.Errorf(
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
for range 3 {
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
got, banned, _ := ledger.Find(
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64")
now := midnight()
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.BanForLimit(b, now, bans.Notes{})
ledger.BanForLimit(c, now, bans.Notes{})
// A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's.
ledger.Check(a.Addr(), now)
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(d, now, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(a, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes
// first.
ledger.Check(d.Addr(), first.Expires)
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(b, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" ||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit.
third, _ := ledger.Ban(
third, _ := ledger.BanForAttack(
netblock,
second.Expires.Add(30*day),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// While the first ban lasts, none would be made.
during, would := ledger.WouldBan(
during, would := ledger.WouldBanForAttack(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if would || !reflect.DeepEqual(during, first) {
t.Errorf(
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made.
limitGrounds := bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
}
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
bans.Notes{RuleID: "git-dir"})
limit, wouldLimit := ledger.WouldBanForLimit(
netblock,
first.Expires,
limitGrounds,
limitNotes,
)
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false)
// The ban made is the one that would have been.
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit)
}
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1)
}
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
now = ban.Expires
}
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
}
// Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent")
}
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit {
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
}
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
bans.Notes{Request: request},
)
cut := long[:256]
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock.
ledger.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
ledger.BanForLimit(netblock, midnight(), bans.Notes{
ASN: "AS64497", ASName: "Other Net", Country: "FR",
})
ledger.Ban(
netblock,
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
ledger.BanForLimit(other, midnight(), bans.Notes{})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+21 -36
View File
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false)
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write.
ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false)
// Two bans before the value is read leave one.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.11/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
wantChanged(t, ledger, true)
wantChanged(t, ledger, false)
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
v6,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
ledger.BanForLimit(high, midnight(), bans.Notes{})
ledger.BanForLimit(low, midnight(), bans.Notes{})
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
snapshot := ledger.Snapshot()
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
// Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse")
}
again, _ := after.Ban(
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(),
},
})
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
for client, want := range map[string]bool{
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
}
// A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
nineHours.Expires.Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
}
// The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.8/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
want := []bans.Ban{first, second, kept}
-90
View File
@@ -1,90 +0,0 @@
// Package dirwatch watches a directory, and calls a function once the
// directory has gone a short time without a change after one. The rule
// files and the lookup database are read again this way.
package dirwatch
import (
"context"
"time"
"github.com/fsnotify/fsnotify"
)
// quietTime is how long the directory must go without a change before the
// function is called, so that a file still being written, such as one
// saved in place, appended to or copied in with scp, is read only once
// whole.
const quietTime = 2 * time.Second
// Watcher watches one directory. The directory is watched, rather than a
// file in it, so that a file replaced by a rename is seen.
type Watcher struct {
watcher *fsnotify.Watcher
}
// New starts watching dir, or returns why it cannot.
func New(dir string) (*Watcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
err = watcher.Add(dir)
if err != nil {
_ = watcher.Close()
return nil, err
}
return &Watcher{watcher: watcher}, nil
}
// Close stops watching the directory.
func (w *Watcher) Close() error {
return w.watcher.Close()
}
// CallWhenQuiet calls changed each time the directory has gone quietTime
// without a change, after a change to a path for which counts is true, until
// ctx is done. It also calls failed with each error the watch reports. The
// wait starts at once, as if for a change, so that a change made before the
// watch began is taken in too.
func (w *Watcher) CallWhenQuiet(
ctx context.Context,
counts func(path string) bool,
changed func(),
failed func(error),
) {
callWhenQuiet(
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
)
}
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
// errors from errs.
func callWhenQuiet(
ctx context.Context,
events <-chan fsnotify.Event,
errs <-chan error,
counts func(path string) bool,
changed func(),
failed func(error),
) {
quiet := time.NewTimer(quietTime)
defer quiet.Stop()
for {
select {
case <-ctx.Done():
return
case event := <-events:
if counts(event.Name) {
quiet.Reset(quietTime)
}
case <-quiet.C:
changed()
case err := <-errs:
failed(err)
}
}
}
-167
View File
@@ -1,167 +0,0 @@
package dirwatch
import (
"context"
"errors"
"path/filepath"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/fsnotify/fsnotify"
)
// errWatch is an error the watch reports.
var errWatch = errors.New("too many changes")
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
t.Parallel()
_, err := New(filepath.Join(t.TempDir(), "missing"))
if err == nil {
t.Error("a missing directory is watched, want an error")
}
}
// The tests below run callWhenQuiet in a synctest bubble, where time is a
// clock of the test's own: time.Sleep moves it on at once, and
// synctest.Wait returns once callWhenQuiet waits again, so that every call
// due by then is made. The test sends the changes itself, as the watch of
// a directory cannot run in a bubble.
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
loop := start(t, everyPath)
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
time.Sleep(quietTime - time.Nanosecond)
synctest.Wait()
loop.wantCalls(t, 0)
// The second part starts the wait again.
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
time.Sleep(quietTime - time.Nanosecond)
synctest.Wait()
loop.wantCalls(t, 0)
time.Sleep(time.Nanosecond)
synctest.Wait()
loop.wantCalls(t, 1)
})
}
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
// An edit saved before the watch began sends no change, and is
// taken in by this call.
loop := start(t, everyPath)
time.Sleep(quietTime)
synctest.Wait()
loop.wantCalls(t, 1)
// Long after, it has been called only once.
time.Sleep(time.Hour)
synctest.Wait()
loop.wantCalls(t, 1)
})
}
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
loop := start(t, func(path string) bool {
return path == "ipinfo_lite.mmdb"
})
// The wait that starts with the watch ends with a call.
time.Sleep(quietTime)
synctest.Wait()
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
time.Sleep(quietTime)
synctest.Wait()
loop.wantCalls(t, 1)
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
time.Sleep(quietTime)
synctest.Wait()
loop.wantCalls(t, 2)
})
}
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
loop := start(t, everyPath)
loop.errs <- errWatch
synctest.Wait()
if !errors.Is(loop.failure, errWatch) {
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
}
})
}
// loop is a callWhenQuiet that a test runs.
type loop struct {
// changes and errs send it changes and errors.
changes chan fsnotify.Event
errs chan error
// calls counts its calls of changed, and failure is the last error it
// handed to failed.
calls atomic.Int32
failure error
}
// start runs callWhenQuiet, with counts, until the test ends.
func start(t *testing.T, counts func(path string) bool) *loop {
t.Helper()
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
ctx, stop := context.WithCancel(t.Context())
stopped := make(chan struct{})
go func() {
callWhenQuiet(ctx, l.changes, l.errs, counts,
func() { l.calls.Add(1) },
func(err error) { l.failure = err },
)
close(stopped)
}()
t.Cleanup(func() {
stop()
<-stopped
})
return l
}
// everyPath counts a change to any path.
func everyPath(string) bool {
return true
}
// wantCalls checks how many times changed has been called.
func (l *loop) wantCalls(t *testing.T, want int32) {
t.Helper()
got := l.calls.Load()
if got != want {
t.Errorf("changed called %d times, want %d", got, want)
}
}
+24 -14
View File
@@ -84,7 +84,12 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct {
params Params
url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure.
httpClient *http.Client
@@ -140,7 +145,12 @@ func New(params Params) *GeoJS {
}
return &GeoJS{
params: params,
url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
@@ -183,7 +193,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client)
if !found {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
}
w, waiting := g.waiting[client]
@@ -263,7 +273,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge()
now := g.params.Now()
now := g.now()
for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor {
@@ -300,7 +310,7 @@ func (g *GeoJS) answerOrWait(
}
if w == nil {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
return Answer{}, nil // too many clients wait already
}
@@ -312,7 +322,7 @@ func (g *GeoJS) answerOrWait(
}
if w.late {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
return Answer{}, nil
}
@@ -323,7 +333,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.params.Now()
now := g.now()
kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor {
@@ -340,7 +350,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
// been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking.
func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
if g.asking || g.stopped || g.now().Before(g.retryAt) {
return
}
@@ -372,9 +382,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
kept, answered := g.keep(clients, given, err)
if g.params.Answered != nil {
if g.answered != nil {
for _, answer := range kept {
g.params.Answered(answer)
g.answered(answer)
}
}
@@ -423,7 +433,7 @@ func (g *GeoJS) keep(
g.mu.Lock()
defer g.mu.Unlock()
now := g.params.Now()
now := g.now()
kept := make([]Answer, 0, len(clients))
leftOut := 0
@@ -467,7 +477,7 @@ func (g *GeoJS) keep(
w.late = true
}
g.params.Failed(err, g.retryDelay)
g.failed(err, g.retryDelay)
return kept, false
}
@@ -497,7 +507,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext(
ctx,
http.MethodGet,
g.params.URL,
g.url,
http.NoBody,
)
if err != nil {
@@ -506,7 +516,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.params.Asked()
g.asked()
res, err := g.httpClient.Do(req)
if err != nil {
+49 -18
View File
@@ -13,13 +13,18 @@ import (
"sync"
"time"
"github.com/fsnotify/fsnotify"
"github.com/oschwald/maxminddb-golang/v2"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
"sneak.berlin/go/smallwebwaf/internal/geojs"
)
// quietTime is how long the lookup database must go without a change
// before it is read again, so that a file still being copied in is read
// only once whole.
const quietTime = 2 * time.Second
// FileParams are what OpenFile needs.
type FileParams struct {
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
@@ -117,13 +122,20 @@ func (f *File) ReadFailures() int {
}
// Watch watches the directory of the lookup database until ctx is done,
// and reads the file again once it has gone without a change for a short
// time, after it is replaced, written or removed, and after Watch starts
// watching, as dirwatch describes. A change to another file in its
// directory does not count. If the directory cannot be watched, that is
// logged, and the database read at start stays in use.
// and reads the file again once it has gone without a change for
// quietTime, after it is replaced, written or removed, and after Watch
// starts watching. If the directory cannot be watched, that is logged, and
// the database read at start stays in use.
func (f *File) Watch(ctx context.Context) {
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
watcher, err := fsnotify.NewWatcher()
if err == nil {
defer func() {
_ = watcher.Close()
}()
err = watcher.Add(filepath.Dir(f.params.Path))
}
if err != nil {
f.params.ProcessLog.Error(
"cannot watch the lookup database for replacements",
@@ -134,22 +146,41 @@ func (f *File) Watch(ctx context.Context) {
return
}
defer func() {
_ = watcher.Close()
}()
f.params.ProcessLog.Info("watching the lookup database for replacements",
"file", f.params.Path)
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
f.params.ProcessLog.Warn("watching the lookup database failed",
"error", err.Error())
})
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
}
// isDatabase reports whether path names the lookup database.
func (f *File) isDatabase(path string) bool {
return filepath.Clean(path) == filepath.Clean(f.params.Path)
// readAfterChanges reads the lookup database again once quietTime has
// passed without a change to it from events, until ctx is done, and logs
// the errors from errs. A change to another file in its directory does not
// count. The wait starts at once, as if for a change, so that a file
// replaced after OpenFile read it, and before its directory was watched,
// is read too.
func (f *File) readAfterChanges(
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
) {
path := filepath.Clean(f.params.Path)
quiet := time.NewTimer(quietTime)
defer quiet.Stop()
for {
select {
case <-ctx.Done():
return
case event := <-events:
if filepath.Clean(event.Name) == path {
quiet.Reset(quietTime)
}
case <-quiet.C:
f.readAgain()
case err := <-errs:
f.params.ProcessLog.Warn("watching the lookup database failed",
"error", err.Error())
}
}
}
// readAgain reads the lookup database again, in place of the one in use,
+141 -35
View File
@@ -1,6 +1,7 @@
package lookup
import (
"context"
"log/slog"
"net/netip"
"net/url"
@@ -11,6 +12,7 @@ import (
"testing/synctest"
"time"
"github.com/fsnotify/fsnotify"
"github.com/maxmind/mmdbwriter/mmdbtype"
"sneak.berlin/go/smallwebwaf/internal/alerts"
@@ -131,48 +133,66 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
}
}
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
t.Parallel()
// The tests below run readAfterChanges in a synctest bubble, where time is
// a clock of the test's own: time.Sleep moves it on at once, and
// synctest.Wait returns once readAfterChanges waits again, so that every
// reading due by then is done. The test sends the changes itself, as the
// watch of a directory cannot run in a bubble.
dir := t.TempDir()
path := filepath.Join(dir, "ipinfo_lite.mmdb")
writeDatabase(t, path, "DE")
f := openFile(t, path, newQueue())
for changed, want := range map[string]bool{
path: true,
dir + "/./ipinfo_lite.mmdb": true,
filepath.Join(dir, "other.mmdb"): false,
} {
if f.isDatabase(changed) != want {
t.Errorf(
"a change to %s counts: %t, want %t",
changed,
!want,
want,
)
}
}
}
// The tests below run in a synctest bubble, where time is a clock of the
// test's own, so that the time a file is read can be checked. The wait for
// the directory to go quiet before a reading is tested in dirwatch.
func TestReplacementIsReadAgain(t *testing.T) {
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
dir := t.TempDir()
path := filepath.Join(dir, "ipinfo_lite.mmdb")
writeDatabase(t, path, "DE")
queue := newQueue()
f := openFile(t, path, queue)
changes := watch(t, f)
writeDatabase(t, path, "KP")
time.Sleep(time.Hour)
f.readAgain()
other := filepath.Join(dir, "replacement.mmdb")
writeDatabase(t, other, "KP")
//nolint:gosec // a file the test wrote
replacement, err := os.ReadFile(other)
if err != nil {
t.Fatalf("read %s: %v", other, err)
}
// The file in use is overwritten in place, and keeps giving what
// it gave. Its first part alone is not a .mmdb file.
file, err := os.Create(path) //nolint:gosec // a file the test wrote
if err != nil {
t.Fatalf("create %s: %v", path, err)
}
defer func() {
_ = file.Close()
}()
half := len(replacement) / 2
write(t, file, replacement[:half])
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
time.Sleep(quietTime - time.Nanosecond)
synctest.Wait()
wantCountry(t, f, "DE")
// The second part starts the wait again.
write(t, file, replacement[half:])
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
time.Sleep(quietTime - time.Nanosecond)
synctest.Wait()
wantCountry(t, f, "DE")
time.Sleep(time.Nanosecond)
synctest.Wait()
wantCountry(t, f, "KP")
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
@@ -199,10 +219,15 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
queue := newQueue()
f := openFile(t, path, queue)
read := f.LastRead()
changes := watch(t, f)
writeFile(t, path, "not a lookup database\n")
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
// Long after, the replacement has been read once.
time.Sleep(time.Hour)
f.readAgain()
synctest.Wait()
wantCountry(t, f, "DE")
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
@@ -215,7 +240,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
}
wantAlerts(t, queue, alerts.Alert{
Time: read.Add(time.Hour),
Time: read.Add(quietTime),
Event: alerts.EventFileError,
Reason: "the lookup database cannot be read, and the one read before stays in use",
Detail: map[string]any{
@@ -227,6 +252,55 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
})
}
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "ipinfo_lite.mmdb")
writeDatabase(t, path, "DE")
f := openFile(t, path, newQueue())
changes := watch(t, f)
// The wait that starts with the watch ends with a reading.
time.Sleep(quietTime)
synctest.Wait()
writeDatabase(t, path, "KP")
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
time.Sleep(quietTime)
synctest.Wait()
wantCountry(t, f, "DE")
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
time.Sleep(quietTime)
synctest.Wait()
wantCountry(t, f, "KP")
})
}
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
writeDatabase(t, path, "DE")
f := openFile(t, path, newQueue())
// Saved after OpenFile read the file, and before its directory was
// watched, so that no change is seen for it.
writeDatabase(t, path, "KP")
watch(t, f)
time.Sleep(quietTime)
synctest.Wait()
wantCountry(t, f, "KP")
})
}
// newQueue returns a queue of alerts for a webhook that is never sent
// them, so that they wait in it for the test to look at.
func newQueue() *alerts.Queue {
@@ -270,6 +344,28 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
return f
}
// watch runs f's readAfterChanges until the test ends, and returns the
// channel that sends it changes.
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
t.Helper()
changes := make(chan fsnotify.Event)
ctx, stop := context.WithCancel(t.Context())
stopped := make(chan struct{})
go func() {
f.readAfterChanges(ctx, changes, nil)
close(stopped)
}()
t.Cleanup(func() {
stop()
<-stopped
})
return changes
}
// wantCountry checks the country f gives testClient.
func wantCountry(t *testing.T, f *File, want string) {
t.Helper()
@@ -305,3 +401,13 @@ func writeFile(t *testing.T, path, content string) {
t.Fatalf("write %s: %v", path, err)
}
}
// write writes data to the end of file.
func write(t *testing.T, file *os.File, data []byte) {
t.Helper()
_, err := file.Write(data)
if err != nil {
t.Fatalf("write: %v", err)
}
}
-44
View File
@@ -1,44 +0,0 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
-105
View File
@@ -1,105 +0,0 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+4 -7
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight()
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for client, requests := range map[string]int{
"198.51.100.9/32": 2,
+6 -18
View File
@@ -156,28 +156,16 @@ type Request struct {
BlockedByCoreRuleSet bool
}
// Params are what New needs.
type Params struct {
// Limits are what each client's requests and bytes are counted against.
Limits Limits
// MaxClients is the most clients the table holds,
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
// New returns a Limiter for limits, with no client counted yet, whose
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
// it, the least recently seen client is dropped, with its history, and
// starts afresh if it comes back.
func New(limits Limits, maxClients int) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
if err != nil {
panic(err) // NewLRU fails only for a size below one
}
limits := params.Limits
return &Limiter{
windows: [3]window{
{
+39 -63
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
tableSize,
)
if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
off := ratelimit.New(ratelimit.Limits{}, tableSize)
off.Count(client, now, whole)
if off.OverARateLimit(client, now) {
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
limiter := ratelimit.New(tc.limits, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
quarter := tc.length / 4
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerHour: limit},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: 5,
BytesPerDay: math.MaxInt64,
},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel()
// Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
limiter := ratelimit.New(tc.limits, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it.
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight()
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerDay: limit},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
tableSize,
)
refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight()
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
now := midnight()
clients := make([]netip.Prefix, maxClients+1)
+6 -12
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64",
}
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
}
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
before := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
for range limit {
wantCount(t, before, client, start, "")
}
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no
// fresh allowance.
later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour)
}
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit)
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client {
t.Helper()
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
after := ratelimit.New(ratelimit.Limits{}, tableSize)
after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0]
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next()
}
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
limiter.Load(clients, midnight())
got := limiter.Snapshot()
+33 -4
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the request log sww prints on stdout: JSON
// lines marked "type":"request", at most two from waf and a short one
// from rproxy for each request. Package processlog writes the process's
// own messages.
// Package requestlog writes the lines sww prints on stdout: one
// JSON object per request, two from waf and a short one from rproxy,
// marked "type":"request", and the process's own messages as JSON lines marked
// "type":"process".
package requestlog
import (
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond,
)
}
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
+87
View File
@@ -3,6 +3,8 @@ package requestlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want)
}
}
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
-6
View File
@@ -2,12 +2,6 @@ package rproxy
import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct {
b breaker
+10 -19
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{
Output: io.Discard,
InstanceName: "test",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
io.Discard,
"test",
slog.LevelInfo,
),
})
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
@@ -870,14 +870,10 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
t.Parallel()
// rproxy is waiting for the rest of the body well within this limit, even on
// a busy host, and answers when it runs out, well before dial's deadline.
const limit = 5 * time.Second
standIn := startWAF(t, allow)
app := startApp(t, greet)
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
maxRequestTime: "3000",
})
conn := dial(t, addr)
@@ -1266,9 +1262,7 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
) {
t.Parallel()
// The app's answer arrives well within this limit, even on a busy host, and
// the connection closes when it runs out, well before dial's deadline.
const limit = 5 * time.Second
const limit = 2 * time.Second
standIn := startWAF(t, allow)
app := startApp(t, answerBeforeTheBody)
@@ -1433,20 +1427,17 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
const answer = "the end of a long answer"
// The request reaches the app well within this limit, even on a busy host.
const limit = 5 * time.Second
standIn := startWAF(t, allow)
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
_ = http.NewResponseController(w).Flush()
time.Sleep(limit + 100*time.Millisecond)
time.Sleep(2100 * time.Millisecond)
_, _ = io.WriteString(w, answer)
})
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
maxRequestTime: "2000",
})
got := visit(
+2 -5
View File
@@ -94,7 +94,6 @@ func New(params Params) *Server {
reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(),
breakerClock: time.Now,
}
return &Server{
@@ -134,10 +133,8 @@ type handler struct {
wafTransport http.RoundTripper
// reportURL is where the reports go.
reportURL string
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
// the time.
breaker breaker
breakerClock func() time.Time
// breaker is the circuit breaker for asks to waf.
breaker breaker
// reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
+21 -54
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -356,19 +356,6 @@ func startServerLogging(
) (string, *rproxy.Server) {
t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000",
@@ -380,20 +367,15 @@ func newServer(
t.Fatalf("settings %v: %v", settings, err)
}
return rproxy.New(rproxy.Params{
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: processlog.New(processlog.Params{
Output: processLog,
InstanceName: "test",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
processLog,
"test",
slog.LevelInfo,
),
})
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen(
t.Context(),
@@ -414,7 +396,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
server.WaitForReports()
})
return listener.Addr().String()
return listener.Addr().String(), server
}
// answered is the visitor's answer.
@@ -1452,18 +1434,14 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
greet(w, r)
})
out := &output{}
server := newServer(
t, standIn.URL, app.URL,
map[string]string{wafBackendTimeout: "200"},
out, out,
addr, out := startRProxy(
t,
standIn.URL,
app.URL,
map[string]string{
wafBackendTimeout: "200",
},
)
// The breaker's clock stands still, as in
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
// failures could take more than five seconds, and the next request could
// come more than a second later.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
// An ask can time out before it reaches the stand-in, so these
// may leave fewer than five questions, or leave some late.
@@ -1554,13 +1532,9 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r)
})
app := startApp(t, greet)
out := &output{}
server := newServer(t, standIn.URL, app.URL, nil, out, out)
// The breaker's clock stands still. On a slow host, the five failures could
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
wafBackendTimeout: "60000",
})
for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
@@ -1835,15 +1809,11 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
t.Parallel()
// The app's answer to the upgrade arrives well within these limits, even on a
// busy host.
const limit = 5 * time.Second
standIn := startWAF(t, allow)
app := startApp(t, echoAfterUpgrade)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
maxRequestTime: "2000",
maxResponseTime: "2000",
})
conn := dial(t, addr)
@@ -1868,10 +1838,7 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
}
// Past both time limits, the connection still carries what is sent.
time.Sleep(limit + 100*time.Millisecond)
// dial's deadline ran from before the upgrade and this wait.
_ = conn.SetDeadline(time.Now().Add(waitLimit))
time.Sleep(2100 * time.Millisecond)
const message = "still here\n"
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"maps"
"net/http"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -96,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context,
) (*wafAnswer, error) {
b := &rq.h.breaker
if !b.mayAsk(rq.h.breakerClock()) {
if !b.mayAsk(time.Now()) {
return nil, errBreakerOpen
}
@@ -106,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil:
b.noteAnswer()
case ctx.Err() == nil:
b.noteFailure(rq.h.breakerClock())
b.noteFailure(time.Now())
}
return answer, err
+48 -19
View File
@@ -18,11 +18,12 @@ import (
"slices"
"strings"
"sync/atomic"
"time"
"github.com/fsnotify/fsnotify"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
)
// The actions a rule takes when it matches.
@@ -39,6 +40,12 @@ const (
// extension ends the name of every rule file.
const extension = ".rules"
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
// the rule files are read again, so that a file still being written, such
// as one saved in place, appended to or copied in with scp, is read only
// once whole.
const quietTime = 2 * time.Second
// headerTarget starts the target that is one request header,
// header:<Name>.
const headerTarget = "header:"
@@ -169,18 +176,26 @@ func (f *Files) Len() int {
}
// Watch watches Dir until ctx is done, and reads the rule files again
// once Dir has had no change for a short time, after one is edited, added
// or removed, and after Watch starts watching, as dirwatch describes. If
// they then hold an error, the rules stay as they were, the error is
// logged with its file and line, and the files are read again after the
// next change. If Dir cannot be watched, that is logged, and the rules
// stay as they were loaded. While Enabled is false, Watch returns at once.
// once Dir has had no change for quietTime, after one is edited, added or
// removed, and after Watch starts watching. If they then hold an error,
// the rules stay as they were, the error is logged with its file and
// line, and the files are read again after the next change. If Dir cannot
// be watched, that is logged, and the rules stay as they were loaded.
// While Enabled is false, Watch returns at once.
func (f *Files) Watch(ctx context.Context) {
if !f.params.Enabled {
return
}
watcher, err := dirwatch.New(f.params.Dir)
watcher, err := fsnotify.NewWatcher()
if err == nil {
defer func() {
_ = watcher.Close()
}()
err = watcher.Add(f.params.Dir)
}
if err != nil {
f.params.ProcessLog.Error("cannot watch the rule files for edits",
"error", err.Error())
@@ -188,22 +203,36 @@ func (f *Files) Watch(ctx context.Context) {
return
}
defer func() {
_ = watcher.Close()
}()
f.params.ProcessLog.Info("watching the rule files for edits",
"directory", f.params.Dir)
// A change to any file in Dir counts.
watcher.CallWhenQuiet(ctx,
func(string) bool { return true },
f.readAgain,
func(err error) {
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
}
// readAfterChanges reads the rule files again once quietTime has passed
// without a change from events, until ctx is done, and logs the errors
// from errs. The wait starts at once, as if for a change, so that an edit
// saved after Load read the files, and before Dir was watched, is taken
// in too.
func (f *Files) readAfterChanges(
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
) {
quiet := time.NewTimer(quietTime)
defer quiet.Stop()
for {
select {
case <-ctx.Done():
return
case <-events:
quiet.Reset(quietTime)
case <-quiet.C:
f.readAgain()
case err := <-errs:
f.params.ProcessLog.Warn("watching the rule files failed",
"error", err.Error())
},
)
}
}
}
// readAgain reads the rule files again, in place of the rules loaded, or
+165
View File
@@ -0,0 +1,165 @@
package rules
import (
"context"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
"testing"
"testing/synctest"
"time"
"github.com/fsnotify/fsnotify"
"sneak.berlin/go/smallwebwaf/internal/alerts"
)
// The tests below run readAfterChanges in a synctest bubble, where time is
// a clock of the test's own: time.Sleep moves it on at once, and
// synctest.Wait returns once readAfterChanges waits again, so that every
// reading due by then is done. The test sends the changes itself, as the
// watch of a directory cannot run in a bubble.
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "50-app.rules")
writeFile(t, path, "first path block ^/first\n")
files := load(t, dir)
changes := run(t, files)
file, err := os.Create(path) //nolint:gosec // a file the test wrote
if err != nil {
t.Fatalf("create: %v", err)
}
defer func() {
_ = file.Close()
}()
// The first part ends in the middle of a ban rule's regex, which,
// read then, would ban every request.
write(t, file, "first path block ^/first\nprobe path ban ^/")
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
time.Sleep(quietTime - time.Nanosecond)
synctest.Wait()
wantMatched(t, files, "/anything")
// The second part starts the wait again.
write(t, file, `\.env$`+"\n")
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
time.Sleep(quietTime - time.Nanosecond)
synctest.Wait()
wantMatched(t, files, "/.env")
time.Sleep(time.Nanosecond)
synctest.Wait()
wantMatched(t, files, "/.env", "probe")
wantMatched(t, files, "/anything")
})
}
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "50-app.rules")
writeFile(t, path, "first path block ^/first\n")
files := load(t, dir)
// Saved after Load read the files, and before the directory was
// watched, so that no change is seen for it.
writeFile(t, path, "first path block ^/edited\n")
run(t, files)
time.Sleep(quietTime)
synctest.Wait()
wantMatched(t, files, "/edited", "first")
})
}
// load loads the rules in dir.
func load(t *testing.T, dir string) *Files {
t.Helper()
files, err := Load(Params{
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
Alerts: alerts.New(alerts.Params{}),
})
if err != nil {
t.Fatalf("load: %v", err)
}
return files
}
// run runs files' readAfterChanges until the test ends, and returns the
// channel that sends it changes.
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
t.Helper()
changes := make(chan fsnotify.Event)
ctx, stop := context.WithCancel(t.Context())
stopped := make(chan struct{})
go func() {
files.readAfterChanges(ctx, changes, nil)
close(stopped)
}()
t.Cleanup(func() {
stop()
<-stopped
})
return changes
}
// writeFile writes content to the file at path.
func writeFile(t *testing.T, path, content string) {
t.Helper()
err := os.WriteFile(path, []byte(content), 0o600)
if err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
// write writes text to the end of file.
func write(t *testing.T, file *os.File, text string) {
t.Helper()
_, err := file.WriteString(text)
if err != nil {
t.Fatalf("write: %v", err)
}
}
// wantMatched checks the ids of the rules that a GET request for path
// matches, in order.
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
t.Helper()
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
"http://app.example"+path, nil)
matched := files.Match(r)
got := make([]string, 0, len(matched))
for _, rule := range matched {
got = append(got, rule.ID)
}
if !slices.Equal(got, want) {
t.Errorf("%s matched %v, want %v", path, got, want)
}
}
+19 -31
View File
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it.
first, _ := params.Ledger.Ban(
first, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5 * time.Second)
second, _ := params.Ledger.Ban(
second, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run)
// A ban, and bans.json written with it.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay)
synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and
// another added.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
err = files.WriteAll()
@@ -2135,11 +2135,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned {
@@ -2452,7 +2448,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000,
}),
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{
@@ -2514,15 +2510,11 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.Ban(client, now, bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
params.Ledger.BanForLimit(client, now, bans.Notes{
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
})
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Notes{RuleID: "env-file", Target: "path"})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2708,11 +2700,7 @@ func wantLiftedBanKept(
}
// Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.Ban(
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf(
"the next ban lasts %s, want 1h",
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
)
@@ -23,11 +23,11 @@ import (
// setting for either.
func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: hostname,
Level: slog.LevelInfo,
})
processLog := requestlog.NewProcessLogger(
params.Stdout,
hostname,
slog.LevelInfo,
)
cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil {
+7 -4
View File
@@ -755,7 +755,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
}
}
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper()
@@ -768,6 +769,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close()
})
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request)
if err != nil {
t.Fatalf("send: %v", err)
@@ -832,9 +838,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err)
+16 -22
View File
@@ -19,10 +19,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,11 +43,8 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
processLog := requestlog.NewProcessLogger(params.Stdout,
config.InstanceName(params.Viper), slog.LevelError)
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
@@ -66,11 +63,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = processlog.New(processlog.Params{
Output: stdout,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
processLog = requestlog.NewProcessLogger(
stdout,
cfg.InstanceName,
cfg.LogLevel,
)
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
@@ -242,17 +239,14 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
return ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
}
// newLedger returns the ledger of the bans, with the settings for them.
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"",
map[string]string{metricsToken: token},
)
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(otherClient+"/32"),
clk.Now(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
before := server.Ledger.Snapshot()
+5 -3
View File
@@ -130,9 +130,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make
// permanent.
group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
})
attackBan, _ := server.Ledger.BanForAttack(
group,
start,
bans.Notes{RuleID: "probe"},
)
// The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack.
+2 -2
View File
@@ -686,10 +686,10 @@ func askDirectly(
// banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(client+"/32"),
now,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
}
+53 -72
View File
@@ -103,8 +103,9 @@ func (rq *request) countedBytes() int64 {
// one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what
// happens instead.
// the client's counters back to zero. In observe mode it makes no ban and
// sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (rq *request) banForLimit(
now time.Time,
hit ratelimit.Hit,
@@ -122,11 +123,23 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
}
switch hit.Kind {
@@ -136,54 +149,47 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
}
rq.banClient(
now,
status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
)
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
}
}
// banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead.
// asked for. It fills in the rest of the notes. In observe mode it makes
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) string {
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
if rq.h.config.Observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
@@ -191,13 +197,7 @@ func (rq *request) banClient(
return banAction(ban.Permanent())
}
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
rq.line.BanExpires = banExpires(ban)
if made {
@@ -207,25 +207,6 @@ func (rq *request) banClient(
return banAction(ban.Permanent())
}
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per
+5 -4
View File
@@ -548,10 +548,6 @@ func (s *sender) requestWithHeader(
// body is sent as it is after the headers, header holding its
// Content-Length or Transfer-Encoding. header may hold several lines,
// separated by "\r\n". It returns the whole answer.
//
// It waits for the answer as long as it takes, since a busy host can slow
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
// whose answer never comes.
func (s *sender) requestWithBody(
method, from, path, header, body string, status int, action string,
) (logLine, answer) {
@@ -570,6 +566,11 @@ func (s *sender) requestWithBody(
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
if err != nil {
s.t.Fatalf("set read deadline: %v", err)
}
var got answer
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
+6
View File
@@ -104,6 +104,12 @@ func onReport() []heuristic {
}
}
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
}
// runHeuristics runs list, one of those checkClient runs, on the request at now,
// and returns the first verdict, or nil when none gives one. The log line's
// heuristic and heuristic_figures name the heuristic that gave it, and give its
+24 -24
View File
@@ -24,7 +24,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -194,11 +193,11 @@ func (o *output) requestLine(t *testing.T) logLine {
// endedLines waits for count requests that went through rproxy, or to waf's
// own endpoints, to have ended, and returns the last log line of each, in the
// order waf took them. A question's last line is the one its report writes.
// It waits as long as it takes, as requestWithBody does.
func (o *output) endedLines(t *testing.T, count int) []logLine {
t.Helper()
for {
deadline := time.Now().Add(waitLimit)
for time.Now().Before(deadline) {
found := o.lastLines(t)
ended := len(found) >= count
@@ -214,6 +213,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
time.Sleep(pollInterval)
}
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
return nil
}
// lastLines returns the last request log line written so far of each request,
@@ -395,11 +398,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{
Output: io.Discard,
InstanceName: "rproxy",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
io.Discard,
"rproxy",
slog.LevelInfo,
),
})
listener, err := (&net.ListenConfig{}).Listen(
@@ -468,11 +471,11 @@ func newWAFWith(
}
out := &output{}
processLog := processlog.New(processlog.Params{
Output: out,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
processLog := requestlog.NewProcessLogger(
out,
cfg.InstanceName,
cfg.LogLevel,
)
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -558,17 +561,14 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName,
)
params.Limiter = ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
params.Limiter = ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
-10
View File
@@ -345,16 +345,6 @@ func timing(start, end time.Time) *float64 {
return new(requestlog.Milliseconds(end.Sub(start)))
}
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
// It then counts the request for the anomaly thresholds and adds it to its
// client's history. list runs first, so that a refusal limit it finds broken
// is in that history. ctx is the request's own context.
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
rq.runBanningHeuristics(ctx, list)
rq.countAnomalies()
rq.addToHistory()
}
// addToHistory adds the request, which has ended, to its client's
// history, and counts its offences in the metrics, and then the lookup's
// answer about the client, as answerAtTheEnd gives it, to that history and
+28 -21
View File
@@ -177,13 +177,11 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
peer.Unmap(),
true,
)
// These run even when a check panics. finish runs first, so that what is
// held has the finished log line. Go's server sends the answer only once
// this function has returned, so the report cannot come before it is held.
defer func() {
rq.finish()
rq.holdForReport()
}()
// The last deferred runs first, so that what is held has the finished log
// line. Go's server sends the answer only once this function has
// returned, so the report cannot come before it is held.
defer rq.holdForReport()
defer rq.finish()
rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision
@@ -192,28 +190,35 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
if refused == nil && !underOwnPrefix {
rq.allow()
// In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), afterAnswer())
defer rq.runAfterAnswer(r.Context())
rq.allow()
return
}
// The last deferred runs first: runAfterAnswer before addToHistory, so that
// a broken refusal limit is in the client's history.
defer rq.addToHistory()
defer rq.countAnomalies()
defer rq.runAfterAnswer(r.Context())
if refused != nil {
rq.deny(*refused)
} else {
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
return
}
rq.banAndCount(r.Context(), afterAnswer())
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
}
// readQuestion returns the visitor's address and the body a question carries,
@@ -486,7 +491,9 @@ func (h *handler) finishWith(
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, onReport())
rq.runBanningHeuristics(ctx, onReport())
rq.countAnomalies()
rq.addToHistory()
rq.h.metrics.RequestReported(
&rq.line,