Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a0277879d7 | ||
|
|
2083584ee0 | ||
|
|
fe61d6b066 | ||
|
|
9a8ed745bd | ||
|
|
0a230c86ce | ||
|
|
40386dce6e | ||
|
|
433d5cc0a9 | ||
|
|
419ee1c441 | ||
|
|
aaa28f541b | ||
|
|
98d7432077 | ||
|
|
11ac8aee50 |
No files matched your search
@@ -906,6 +906,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
|
- `internal/dirwatch`: watches a directory, so that `internal/rules` and
|
||||||
|
`internal/lookup` read their files again when they change.
|
||||||
- `internal/state`: the state and cache files.
|
- `internal/state`: the state and cache files.
|
||||||
- `internal/requestlog`: the request log.
|
- `internal/requestlog`: the request log.
|
||||||
- `internal/processlog`: the process's own messages.
|
- `internal/processlog`: the process's own messages.
|
||||||
|
|||||||
@@ -93,6 +93,9 @@ never as a time.
|
|||||||
imports no other package of this repository: `waf` gives it what it needs
|
imports no other package of this repository: `waf` gives it what it needs
|
||||||
to count and report failures.
|
to count and report failures.
|
||||||
- `internal/lookup`: looks clients up in the lookup database.
|
- `internal/lookup`: looks clients up in the lookup database.
|
||||||
|
- `internal/dirwatch`: watches a directory, and calls a function once it has
|
||||||
|
gone a short time without a change. `internal/rules` and `internal/lookup`
|
||||||
|
read their files again this way.
|
||||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
AbuseIPDB in the background, and keeps their answers. It also fetches
|
||||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
AbuseIPDB's blacklist, which `internal/waf` bans.
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ require (
|
|||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
|
golang.org/x/net v0.60.0
|
||||||
golang.org/x/sys v0.48.0
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -50,7 +51,6 @@ require (
|
|||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||||
golang.org/x/net v0.60.0 // indirect
|
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
|
|||||||
@@ -54,7 +54,11 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
|
|||||||
now := midnight()
|
now := midnight()
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
scores := map[netip.Prefix]int64{netblock: 100}
|
||||||
|
|
||||||
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
limitBan, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
||||||
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
||||||
@@ -119,7 +123,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
|
|||||||
expires := now.Add(30 * day)
|
expires := now.Add(30 * day)
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
||||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
// listed is off the blacklist when it is fetched after the ban's end.
|
// listed is off the blacklist when it is fetched after the ban's end.
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
||||||
@@ -144,7 +148,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
||||||
}
|
}
|
||||||
|
|
||||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
||||||
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
||||||
@@ -154,7 +158,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
|
|
||||||
// The ban for an attack takes the place of the one for a limit, and no
|
// The ban for an attack takes the place of the one for a limit, and no
|
||||||
// blacklist ban goes.
|
// blacklist ban goes.
|
||||||
ledger.BanForAttack(attacker, now, bans.Notes{})
|
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
|
||||||
|
|
||||||
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
||||||
for netblock := range scores {
|
for netblock := range scores {
|
||||||
|
|||||||
+64
-23
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
||||||
|
|
||||||
// A new ban drops the ban sww made, and only that one.
|
// A new ban drops the ban sww made, and only that one.
|
||||||
ledger.BanForLimit(second, midnight(), bans.Notes{})
|
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// And the next drops that one.
|
// And the next drops that one.
|
||||||
ledger.BanForLimit(third, midnight(), bans.Notes{})
|
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -98,20 +98,28 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
|
|||||||
// twice, each time after its last ban has ended.
|
// twice, each time after its last ban has ended.
|
||||||
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
||||||
"probes for logins")
|
"probes for logins")
|
||||||
first, _ := ledger.BanForLimit(
|
first, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
|
||||||
|
|
||||||
// A ban on another netblock fills the ledger, and the netblock's first
|
// A ban on another netblock fills the ledger, and the netblock's first
|
||||||
// ban for a limit, its earliest that is not an admin's, goes.
|
// ban for a limit, its earliest that is not an admin's, goes.
|
||||||
ledger.BanForLimit(other, second.Start, bans.Notes{})
|
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
||||||
|
|
||||||
// The netblock's next ban still counts it.
|
// The netblock's next ban still counts it.
|
||||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
third, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
second.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -127,15 +135,27 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
|
|||||||
// The netblock breaks a limit twice, the second time after its first
|
// The netblock breaks a limit twice, the second time after its first
|
||||||
// ban has ended, and the second ban's notes count the first. An admin
|
// ban has ended, and the second ban's notes count the first. An admin
|
||||||
// then keeps the first ban.
|
// then keeps the first ban.
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
kept := first
|
kept := first
|
||||||
kept.Cause = bans.CauseAdmin
|
kept.Cause = bans.CauseAdmin
|
||||||
ledger.LoadEdit([]bans.Ban{kept, second})
|
ledger.LoadEdit([]bans.Ban{kept, second})
|
||||||
|
|
||||||
// The netblock's next ban counts the kept ban once, as an admin's.
|
// The netblock's next ban counts the kept ban once, as an admin's.
|
||||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
third, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
second.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -147,20 +167,37 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
limit, _ := ledger.BanForLimit(
|
limit, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{
|
||||||
|
Kind: "requests",
|
||||||
|
Limit: 1000,
|
||||||
|
Window: "minute",
|
||||||
|
},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
byteLimit, _ := ledger.BanForLimit(
|
byteLimit, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.3/32"),
|
netip.MustParsePrefix("203.0.113.3/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{
|
||||||
|
Kind: "bytes",
|
||||||
|
Limit: 10 << 30,
|
||||||
|
Window: "hour",
|
||||||
|
},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
attack, _ := ledger.BanForAttack(
|
attack, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.2/32"),
|
netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{RuleID: "git-dir", Target: "path"},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseAttack,
|
||||||
|
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
for _, tc := range []struct{ got, want string }{
|
for _, tc := range []struct{ got, want string }{
|
||||||
@@ -200,7 +237,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
|
|||||||
t.Error("the lifted ban refuses")
|
t.Error("the lifted ban refuses")
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -251,7 +288,11 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The next clear sign of attack bans for seven days, as a first does.
|
// The next clear sign of attack bans for seven days, as a first does.
|
||||||
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != 7*day {
|
if ban.Expires.Sub(ban.Start) != 7*day {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban for an attack ends at %s, want seven days on",
|
"the next ban for an attack ends at %s, want seven days on",
|
||||||
@@ -264,10 +305,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
made, _ := ledger.BanForLimit(
|
made, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
atStart := bans.Ban{
|
atStart := bans.Ban{
|
||||||
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
@@ -311,7 +352,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// An hour's ban for a broken limit.
|
// An hour's ban for a broken limit.
|
||||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A minute later an admin bans the netblock for good, named by an
|
// A minute later an admin bans the netblock for good, named by an
|
||||||
@@ -431,10 +472,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// The temporary ban is made first, so that its netblock is looked at first.
|
// The temporary ban is made first, so that its netblock is looked at first.
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
ledger.BanForAdmin(
|
ledger.BanForAdmin(
|
||||||
netip.MustParsePrefix("203.0.113.0/24"),
|
netip.MustParsePrefix("203.0.113.0/24"),
|
||||||
|
|||||||
+54
-82
@@ -300,81 +300,50 @@ func activeBan(bans []Ban, now time.Time) *Ban {
|
|||||||
return newest
|
return newest
|
||||||
}
|
}
|
||||||
|
|
||||||
// BanForLimit bans netblock at now for a broken limit, with notes, and
|
// Grounds are why a request makes a ban: its cause, CauseLimit or
|
||||||
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
|
// CauseAttack, and its notes.
|
||||||
// made within LimitBanRepeatWindow after the netblock's ban that ended
|
type Grounds struct {
|
||||||
// last, other than one for a clear sign of attack or a lifted one, lasts
|
Cause string
|
||||||
// repeatFactor times as long as that one. A ban that would be longer
|
Notes Notes
|
||||||
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
|
|
||||||
// active, as when two of its requests break a limit at once, that ban is
|
|
||||||
// returned with false, and no other is made. The ledger fills in the
|
|
||||||
// notes' Refused and EarlierBans itself, and gives the ban the reason
|
|
||||||
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
|
|
||||||
// as "requests per minute over the limit of 1000".
|
|
||||||
func (l *Ledger) BanForLimit(
|
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
|
||||||
) (Ban, bool) {
|
|
||||||
return l.ban(
|
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseLimit,
|
|
||||||
limitReason(notes),
|
|
||||||
notes,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBanForLimit returns what BanForLimit would, without making the ban:
|
// Ban bans netblock at now on grounds, and returns the ban, and true. If
|
||||||
// what observe mode would have done.
|
// a ban on netblock is still active, as when two of its requests break a
|
||||||
func (l *Ledger) WouldBanForLimit(
|
// limit at once, that ban is returned with false, and no other is made.
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
// The ledger fills in the notes' Refused and EarlierBans itself.
|
||||||
|
//
|
||||||
|
// A ban for a broken limit:
|
||||||
|
// - lasts LimitBanDuration the first time;
|
||||||
|
// - lasts repeatFactor times as long as the netblock's ban that ended
|
||||||
|
// last, when made within LimitBanRepeatWindow after it, unless that
|
||||||
|
// ban was for a clear sign of attack or was lifted;
|
||||||
|
// - is permanent instead when it would be longer than
|
||||||
|
// LimitBanPermanentAbove;
|
||||||
|
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
|
||||||
|
// the notes, such as "requests per minute over the limit of 1000".
|
||||||
|
//
|
||||||
|
// A ban for a clear sign of attack:
|
||||||
|
// - lasts AttackBanDuration the first time;
|
||||||
|
// - is permanent once the netblock has had one that was not lifted;
|
||||||
|
// - has the reason "matched the rule <RuleID>", or "asked for the trap
|
||||||
|
// path <TrapPath>".
|
||||||
|
func (l *Ledger) Ban(
|
||||||
|
netblock netip.Prefix, now time.Time, grounds Grounds,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(
|
return l.ban(netblock, now, grounds, true)
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseLimit,
|
|
||||||
limitReason(notes),
|
|
||||||
notes,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BanForAttack bans netblock at now for a clear sign of attack, with
|
// WouldBan returns what Ban would, without making the ban: what observe
|
||||||
// notes, and returns the ban, and whether it made it, as BanForLimit
|
// mode would have done.
|
||||||
// does. A first ban lasts AttackBanDuration; once the netblock has had
|
func (l *Ledger) WouldBan(
|
||||||
// one that was not lifted, the next is permanent. Its reason is "matched
|
netblock netip.Prefix, now time.Time, grounds Grounds,
|
||||||
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
|
|
||||||
func (l *Ledger) BanForAttack(
|
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(
|
return l.ban(netblock, now, grounds, false)
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseAttack,
|
|
||||||
attackReason(notes),
|
|
||||||
notes,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WouldBanForAttack returns what BanForAttack would, without making the
|
|
||||||
// ban: what observe mode would have done.
|
|
||||||
func (l *Ledger) WouldBanForAttack(
|
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
|
||||||
) (Ban, bool) {
|
|
||||||
return l.ban(
|
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseAttack,
|
|
||||||
attackReason(notes),
|
|
||||||
notes,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
||||||
// or CauseAttack, made at now would be permanent, as BanForLimit or
|
// or CauseAttack, made at now would be permanent, as Ban would make it. It
|
||||||
// BanForAttack would make it. It works out nothing else of the ban.
|
// works out nothing else of the ban.
|
||||||
func (l *Ledger) WouldBePermanent(
|
func (l *Ledger) WouldBePermanent(
|
||||||
netblock netip.Prefix, now time.Time, cause string,
|
netblock netip.Prefix, now time.Time, cause string,
|
||||||
) bool {
|
) bool {
|
||||||
@@ -411,10 +380,9 @@ func attackReason(notes Notes) string {
|
|||||||
|
|
||||||
// BanForAdmin bans netblock at now for an admin, with reason, until
|
// BanForAdmin bans netblock at now for an admin, with reason, until
|
||||||
// expires, or for good when expires is zero, and returns the ban, whose
|
// expires, or for good when expires is zero, and returns the ban, whose
|
||||||
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
|
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
|
||||||
// ban even while another on netblock is active, since the admin asked
|
// on netblock is active, since the admin asked for this one. The ledger
|
||||||
// for this one. The ledger fills in the notes' EarlierBans, and counts
|
// fills in the notes' EarlierBans, and counts the ban among those made.
|
||||||
// the ban among those made.
|
|
||||||
func (l *Ledger) BanForAdmin(
|
func (l *Ledger) BanForAdmin(
|
||||||
netblock netip.Prefix, now, expires time.Time, reason string,
|
netblock netip.Prefix, now, expires time.Time, reason string,
|
||||||
) Ban {
|
) Ban {
|
||||||
@@ -759,15 +727,13 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
|
|||||||
return forgot
|
return forgot
|
||||||
}
|
}
|
||||||
|
|
||||||
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
|
// ban bans netblock at now on grounds, as Ban describes, and returns the
|
||||||
// reason and notes, as BanForLimit and BanForAttack describe, and returns
|
// ban, and whether it made it. Unless keep is true, the ban is not made,
|
||||||
// the ban, and whether it made it. Unless keep is true, the ban is not
|
// only returned: it is the ban that would have been made.
|
||||||
// made, only returned: it is the ban that would have been made.
|
|
||||||
func (l *Ledger) ban(
|
func (l *Ledger) ban(
|
||||||
netblock netip.Prefix,
|
netblock netip.Prefix,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
cause, reason string,
|
grounds Grounds,
|
||||||
notes Notes,
|
|
||||||
keep bool,
|
keep bool,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
@@ -776,6 +742,8 @@ func (l *Ledger) ban(
|
|||||||
// held are the netblock's bans, none of them active.
|
// held are the netblock's bans, none of them active.
|
||||||
var held []Ban
|
var held []Ban
|
||||||
|
|
||||||
|
notes := grounds.Notes
|
||||||
|
|
||||||
bans, found := l.netblocks.Get(netblock)
|
bans, found := l.netblocks.Get(netblock)
|
||||||
if found {
|
if found {
|
||||||
active := activeBan(*bans, now)
|
active := activeBan(*bans, now)
|
||||||
@@ -791,15 +759,19 @@ func (l *Ledger) ban(
|
|||||||
ban := Ban{
|
ban := Ban{
|
||||||
Netblock: netblock,
|
Netblock: netblock,
|
||||||
Start: now,
|
Start: now,
|
||||||
Cause: cause,
|
Cause: grounds.Cause,
|
||||||
Reason: reason,
|
|
||||||
Notes: notes,
|
Notes: notes,
|
||||||
}
|
}
|
||||||
|
|
||||||
if cause == CauseAttack {
|
switch grounds.Cause {
|
||||||
ban.Expires = l.attackExpiry(held, now)
|
case CauseLimit:
|
||||||
} else {
|
ban.Reason = limitReason(notes)
|
||||||
ban.Expires = l.limitExpiry(held, now)
|
ban.Expires = l.limitExpiry(held, now)
|
||||||
|
case CauseAttack:
|
||||||
|
ban.Reason = attackReason(notes)
|
||||||
|
ban.Expires = l.attackExpiry(held, now)
|
||||||
|
default:
|
||||||
|
panic("bans: a request's ban has the cause " + grounds.Cause)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !keep {
|
if !keep {
|
||||||
@@ -807,7 +779,7 @@ func (l *Ledger) ban(
|
|||||||
}
|
}
|
||||||
|
|
||||||
l.add(ban)
|
l.add(ban)
|
||||||
l.made[cause]++
|
l.made[grounds.Cause]++
|
||||||
l.markChanged()
|
l.markChanged()
|
||||||
|
|
||||||
return ban, true
|
return ban, true
|
||||||
|
|||||||
+121
-51
@@ -22,7 +22,11 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
||||||
// 81 hours.
|
// 81 hours.
|
||||||
for i, hours := range []int{1, 3, 9, 27, 81} {
|
for i, hours := range []int{1, 3, 9, 27, 81} {
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
length := time.Duration(hours) * time.Hour
|
length := time.Duration(hours) * time.Hour
|
||||||
if !ban.Expires.Equal(now.Add(length)) ||
|
if !ban.Expires.Equal(now.Add(length)) ||
|
||||||
@@ -42,7 +46,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
|
|
||||||
// The sixth would last 243 hours, more than seven days: it is
|
// The sixth would last 243 hours, more than seven days: it is
|
||||||
// permanent, and never ends.
|
// permanent, and never ends.
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -71,15 +75,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(
|
first, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
second, _ := ledger.BanForLimit(
|
second, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires.Add(tc.gap),
|
first.Expires.Add(tc.gap),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != tc.want ||
|
if second.Expires.Sub(second.Start) != tc.want ||
|
||||||
@@ -104,10 +108,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
|
|||||||
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
||||||
@@ -127,7 +131,11 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
for i := range 14 {
|
for i := range 14 {
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if !ban.Expires.After(ban.Start) {
|
if !ban.Expires.After(ban.Start) {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"ban %d starts at %s and ends at %s",
|
"ban %d starts at %s and ends at %s",
|
||||||
@@ -140,7 +148,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -152,15 +160,19 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, made := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if !made {
|
if !made {
|
||||||
t.Error("the first ban was not made")
|
t.Error("the first ban was not made")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made := ledger.BanForLimit(
|
again, made := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
if made || !reflect.DeepEqual(again, first) ||
|
if made || !reflect.DeepEqual(again, first) ||
|
||||||
@@ -175,10 +187,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made = ledger.BanForAttack(
|
again, made = ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
)
|
)
|
||||||
if made || !reflect.DeepEqual(again, first) {
|
if made || !reflect.DeepEqual(again, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -195,10 +207,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Requests: 5},
|
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
||||||
)
|
)
|
||||||
|
|
||||||
for range 3 {
|
for range 3 {
|
||||||
@@ -237,10 +249,10 @@ func TestFindCountsNothing(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Requests: 5},
|
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
||||||
)
|
)
|
||||||
|
|
||||||
got, banned, _ := ledger.Find(
|
got, banned, _ := ledger.Find(
|
||||||
@@ -287,24 +299,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
|
|||||||
d := netip.MustParsePrefix("2001:db8::/64")
|
d := netip.MustParsePrefix("2001:db8::/64")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
|
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
ledger.BanForLimit(b, now, bans.Notes{})
|
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
ledger.BanForLimit(c, now, bans.Notes{})
|
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
// A request from a makes b the netblock seen longest ago, and its ban
|
// A request from a makes b the netblock seen longest ago, and its ban
|
||||||
// goes to make room for d's.
|
// goes to make room for d's.
|
||||||
ledger.Check(a.Addr(), now)
|
ledger.Check(a.Addr(), now)
|
||||||
ledger.BanForLimit(d, now, bans.Notes{})
|
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
||||||
|
|
||||||
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
||||||
ledger.BanForLimit(a, first.Expires, bans.Notes{})
|
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
||||||
|
|
||||||
// With d seen since, a is seen longest ago, and its earlier ban goes
|
// With d seen since, a is seen longest ago, and its earlier ban goes
|
||||||
// first.
|
// first.
|
||||||
ledger.Check(d.Addr(), first.Expires)
|
ledger.Check(d.Addr(), first.Expires)
|
||||||
ledger.BanForLimit(b, first.Expires, bans.Notes{})
|
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
||||||
|
|
||||||
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
||||||
@@ -325,8 +337,16 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
|
|||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
held := ledger.Bans(netblock)
|
held := ledger.Bans(netblock)
|
||||||
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
||||||
@@ -343,7 +363,11 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
||||||
|
|
||||||
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
||||||
|
)
|
||||||
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
||||||
ban.Cause != bans.CauseAttack ||
|
ban.Cause != bans.CauseAttack ||
|
||||||
ban.Notes.RuleID != "env-file" ||
|
ban.Notes.RuleID != "env-file" ||
|
||||||
@@ -418,8 +442,16 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// A ban for a broken limit before does not count.
|
// A ban for a broken limit before does not count.
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != 7*day {
|
if second.Expires.Sub(second.Start) != 7*day {
|
||||||
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
||||||
@@ -435,10 +467,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
|
|
||||||
// Its notes show the earlier ban for an attack that makes it permanent,
|
// Its notes show the earlier ban for an attack that makes it permanent,
|
||||||
// beside the one for a limit.
|
// beside the one for a limit.
|
||||||
third, _ := ledger.BanForAttack(
|
third, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
second.Expires.Add(30*day),
|
second.Expires.Add(30*day),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
)
|
)
|
||||||
if !third.Permanent() ||
|
if !third.Permanent() ||
|
||||||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
||||||
@@ -452,14 +484,18 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// While the first ban lasts, none would be made.
|
// While the first ban lasts, none would be made.
|
||||||
during, would := ledger.WouldBanForAttack(
|
during, would := ledger.WouldBan(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
)
|
)
|
||||||
if would || !reflect.DeepEqual(during, first) {
|
if would || !reflect.DeepEqual(during, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -472,13 +508,19 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
// As it ends, a clear sign of attack would ban for seven days, and a
|
// As it ends, a clear sign of attack would ban for seven days, and a
|
||||||
// limit broken again for three hours, but neither is made.
|
// limit broken again for three hours, but neither is made.
|
||||||
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
|
limitGrounds := bans.Grounds{
|
||||||
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
|
Cause: bans.CauseLimit,
|
||||||
bans.Notes{RuleID: "git-dir"})
|
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
|
||||||
limit, wouldLimit := ledger.WouldBanForLimit(
|
}
|
||||||
|
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
|
||||||
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseAttack,
|
||||||
|
Notes: bans.Notes{RuleID: "git-dir"},
|
||||||
|
})
|
||||||
|
limit, wouldLimit := ledger.WouldBan(
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires,
|
first.Expires,
|
||||||
limitNotes,
|
limitGrounds,
|
||||||
)
|
)
|
||||||
|
|
||||||
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
||||||
@@ -505,7 +547,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// The ban made is the one that would have been.
|
// The ban made is the one that would have been.
|
||||||
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
|
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
|
||||||
if !reflect.DeepEqual(made, limit) {
|
if !reflect.DeepEqual(made, limit) {
|
||||||
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
||||||
}
|
}
|
||||||
@@ -526,7 +568,11 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,7 +585,11 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Once a first ban for an attack has ended, the next would be permanent.
|
// Once a first ban for an attack has ended, the next would be permanent.
|
||||||
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
attack, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
||||||
t.Error("a second ban for an attack would not be permanent")
|
t.Error("a second ban for an attack would not be permanent")
|
||||||
}
|
}
|
||||||
@@ -553,8 +603,16 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
|
|||||||
|
|
||||||
// Three times the seven days would be permanent; a limit broken as the
|
// Three times the seven days would be permanent; a limit broken as the
|
||||||
// ban for an attack ends bans for an hour, as a first broken limit does.
|
// ban for an attack ends bans for an hour, as a first broken limit does.
|
||||||
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
|
attack, _ := ledger.Ban(
|
||||||
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
|
limit, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
attack.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
||||||
limit.Cause != bans.CauseLimit {
|
limit.Cause != bans.CauseLimit {
|
||||||
@@ -578,10 +636,13 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
|
|||||||
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Request: request},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{Request: request},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
cut := long[:256]
|
cut := long[:256]
|
||||||
@@ -607,11 +668,20 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
|
|||||||
|
|
||||||
// A ban made with the client's lookup, one made before it came, after
|
// A ban made with the client's lookup, one made before it came, after
|
||||||
// the first ended, and one on another netblock.
|
// the first ended, and one on another netblock.
|
||||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{
|
ledger.Ban(netblock, midnight(), bans.Grounds{
|
||||||
ASN: "AS64497", ASName: "Other Net", Country: "FR",
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{
|
||||||
|
ASN: "AS64497",
|
||||||
|
ASName: "Other Net",
|
||||||
|
Country: "FR",
|
||||||
|
},
|
||||||
})
|
})
|
||||||
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
|
ledger.Ban(
|
||||||
ledger.BanForLimit(other, midnight(), bans.Notes{})
|
netblock,
|
||||||
|
midnight().Add(time.Hour),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
||||||
|
|
||||||
|
|||||||
@@ -19,25 +19,29 @@ func TestChangedAfterABanIsMade(t *testing.T) {
|
|||||||
|
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A limit broken during the ban makes no other, and a refusal changes
|
// A limit broken during the ban makes no other, and a refusal changes
|
||||||
// only the counts in the notes, which wait for the interval's write.
|
// only the counts in the notes, which wait for the interval's write.
|
||||||
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
|
ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight().Add(time.Minute),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// Two bans before the value is read leave one.
|
// Two bans before the value is read leave one.
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.11/32"),
|
netip.MustParsePrefix("203.0.113.11/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
@@ -51,10 +55,14 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
|
|||||||
high := netip.MustParsePrefix("203.0.113.10/32")
|
high := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
low := netip.MustParsePrefix("203.0.113.9/32")
|
low := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
ledger.BanForLimit(high, midnight(), bans.Notes{})
|
v6,
|
||||||
ledger.BanForLimit(low, midnight(), bans.Notes{})
|
midnight(),
|
||||||
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
snapshot := ledger.Snapshot()
|
snapshot := ledger.Snapshot()
|
||||||
|
|
||||||
@@ -80,7 +88,10 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
|
|
||||||
before := bans.New(defaultParams())
|
before := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
|
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{Limit: 1},
|
||||||
|
})
|
||||||
|
|
||||||
// Loaded into a new ledger, as across a restart, the ban still refuses
|
// Loaded into a new ledger, as across a restart, the ban still refuses
|
||||||
// while it lasts, and once it has ended a broken limit bans for three
|
// while it lasts, and once it has ended a broken limit bans for three
|
||||||
@@ -96,7 +107,11 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
t.Error("the loaded ban does not refuse")
|
t.Error("the loaded ban does not refuse")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
|
again, _ := after.Ban(
|
||||||
|
netblock,
|
||||||
|
ban.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
||||||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -124,10 +139,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
|
|||||||
Start: midnight(),
|
Start: midnight(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
for client, want := range map[string]bool{
|
for client, want := range map[string]bool{
|
||||||
@@ -200,7 +215,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A limit broken now makes no shorter ban over the permanent one.
|
// A limit broken now makes no shorter ban over the permanent one.
|
||||||
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"a broken limit returned %+v and left the netblock %d bans, "+
|
"a broken limit returned %+v and left the netblock %d bans, "+
|
||||||
@@ -236,10 +251,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
|
|||||||
// Once both have ended, a limit broken within the repeat window bans
|
// Once both have ended, a limit broken within the repeat window bans
|
||||||
// for three times the 9 hours, and the notes count the two bans
|
// for three times the 9 hours, and the notes count the two bans
|
||||||
// before the 9-hour one and it, for a limit, and the admin's.
|
// before the 9-hour one and it, for a limit, and the admin's.
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
nineHours.Expires.Add(time.Hour),
|
nineHours.Expires.Add(time.Hour),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
||||||
@@ -313,15 +328,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The ledger holds one ban, so it makes two more without dropping any.
|
// The ledger holds one ban, so it makes two more without dropping any.
|
||||||
first, _ := ledger.BanForLimit(
|
first, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
second, _ := ledger.BanForLimit(
|
second, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("198.51.100.8/32"),
|
netip.MustParsePrefix("198.51.100.8/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
want := []bans.Ban{first, second, kept}
|
want := []bans.Ban{first, second, kept}
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// off is the value that switches a timeout, a size limit or a rate limit
|
// off is the value that switches a timeout, a size limit or a rate limit
|
||||||
@@ -41,8 +43,6 @@ var (
|
|||||||
errNotMilliseconds = errors.New(
|
errNotMilliseconds = errors.New(
|
||||||
"is not a whole number of milliseconds above zero, such as 2000")
|
"is not a whole number of milliseconds above zero, such as 2000")
|
||||||
errEmptyItem = errors.New("has an empty item in its list")
|
errEmptyItem = errors.New("has an empty item in its list")
|
||||||
errNotNetblock = errors.New(
|
|
||||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
|
||||||
errNotListenAddr = errors.New(
|
errNotListenAddr = errors.New(
|
||||||
"is not an address to listen on, such as :8080")
|
"is not an address to listen on, such as :8080")
|
||||||
errNotCommand = errors.New("is not rproxy or waf")
|
errNotCommand = errors.New("is not rproxy or waf")
|
||||||
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
netblocks := make([]netip.Prefix, 0, len(items))
|
netblocks := make([]netip.Prefix, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
netblock, err := ParseNetblock(item)
|
netblock, err := reputation.ParseNetblock(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -264,27 +264,6 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
return netblocks, nil
|
return netblocks, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
|
||||||
// address is a netblock of that address alone, a /32 or a /128. A
|
|
||||||
// blocklist's lines are read with it too.
|
|
||||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
|
||||||
if strings.Contains(value, "/") {
|
|
||||||
netblock, err := netip.ParsePrefix(value)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netblock.Masked(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(value)
|
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseListenAddr checks an address to listen on: an optional host and a
|
// parseListenAddr checks an address to listen on: an optional host and a
|
||||||
// port number.
|
// port number.
|
||||||
func parseListenAddr(value string) (string, error) {
|
func parseListenAddr(value string) (string, error) {
|
||||||
|
|||||||
+77
-121
@@ -20,11 +20,26 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The names of the settings that waf writes into log lines and ban notes,
|
||||||
|
// as the setting that refused a request or lowered a client's limits.
|
||||||
|
const (
|
||||||
|
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
|
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||||
|
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
||||||
|
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
||||||
|
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
||||||
|
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
||||||
|
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
||||||
|
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||||
@@ -100,7 +115,7 @@ type WAF struct {
|
|||||||
// with a tag that percentage of every rate limit and byte limit, from 1
|
// with a tag that percentage of every rate limit and byte limit, from 1
|
||||||
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
||||||
// lower case, a country code in capitals.
|
// lower case, a country code in capitals.
|
||||||
TagLists []TagList
|
TagLists []reputation.TagList
|
||||||
TagDeny []string
|
TagDeny []string
|
||||||
TagLimitPercent map[string]int64
|
TagLimitPercent map[string]int64
|
||||||
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
||||||
@@ -497,12 +512,8 @@ var (
|
|||||||
"is not a whole number above zero, such as 60, or off",
|
"is not a whole number above zero, such as 60, or off",
|
||||||
)
|
)
|
||||||
errNotUTF8 = errors.New("is not valid UTF-8")
|
errNotUTF8 = errors.New("is not valid UTF-8")
|
||||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
|
||||||
errNotPercentItem = errors.New(
|
errNotPercentItem = errors.New(
|
||||||
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
||||||
errNotPercent = errors.New(
|
|
||||||
"is not a percentage, a whole number from 0 to 100",
|
|
||||||
)
|
|
||||||
errNotAbuseIPDBThreshold = errors.New(
|
errNotAbuseIPDBThreshold = errors.New(
|
||||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
||||||
"blacklist can be cut at",
|
"blacklist can be cut at",
|
||||||
@@ -618,24 +629,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"800",
|
"800",
|
||||||
),
|
),
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
ExclusivelyAllowedCountriesSetting, ""),
|
||||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||||
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
||||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||||
ASNBytesLimitPercent: env.percents(
|
ASNBytesLimitPercent: env.percents(
|
||||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
ASNBytesLimitPercentSetting,
|
||||||
ParseASN,
|
reputation.ParseASN,
|
||||||
),
|
),
|
||||||
CountryBytesLimitPercent: env.percents(
|
CountryBytesLimitPercent: env.percents(
|
||||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
CountryBytesLimitPercentSetting,
|
||||||
parseCountry,
|
parseCountry,
|
||||||
),
|
),
|
||||||
UnknownCountryLimitPercent: env.percent(
|
UnknownCountryLimitPercent: env.percent(
|
||||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
UnknownCountryLimitPercentSetting,
|
||||||
"100",
|
"100",
|
||||||
),
|
),
|
||||||
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
||||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||||
@@ -809,9 +820,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
BlocklistActionSetting, string(ReputationActionDeny))
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
DNSBLAbuseIPDBActionSetting, "limit:25")
|
||||||
|
|
||||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||||
env.checkLookupDBPath(cfg)
|
env.checkLookupDBPath(cfg)
|
||||||
@@ -1011,7 +1022,7 @@ func (e *environment) percents(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
||||||
func (e *environment) tagLists(name string) []TagList {
|
func (e *environment) tagLists(name string) []reputation.TagList {
|
||||||
lists, err := parseTagLists(e.value(name, ""))
|
lists, err := parseTagLists(e.value(name, ""))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
@@ -1037,7 +1048,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
|
|||||||
|
|
||||||
// percent reads a setting that is a percentage, from 0 to 100.
|
// percent reads a setting that is a percentage, from 0 to 100.
|
||||||
func (e *environment) percent(name, defaultValue string) int64 {
|
func (e *environment) percent(name, defaultValue string) int64 {
|
||||||
percent, err := ParsePercent(e.value(name, defaultValue))
|
percent, err := reputation.ParsePercent(e.value(name, defaultValue))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
return percent
|
return percent
|
||||||
@@ -1122,7 +1133,7 @@ func (e *environment) action(
|
|||||||
string(value), string(ReputationActionLimit)+":",
|
string(value), string(ReputationActionLimit)+":",
|
||||||
)
|
)
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := reputation.ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
@@ -1131,8 +1142,8 @@ func (e *environment) action(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
// default. The log shows each zone with its key masked, as MaskZoneKey
|
// default. The log shows each zone with its key masked, as
|
||||||
// masks it.
|
// reputation.MaskZoneKey masks it.
|
||||||
func (e *environment) zones(name string) []string {
|
func (e *environment) zones(name string) []string {
|
||||||
value, _ := e.lookup(name)
|
value, _ := e.lookup(name)
|
||||||
zones, err := parseZones(value)
|
zones, err := parseZones(value)
|
||||||
@@ -1140,7 +1151,7 @@ func (e *environment) zones(name string) []string {
|
|||||||
|
|
||||||
logged := make([]string, len(zones))
|
logged := make([]string, len(zones))
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
logged[i] = MaskZoneKey(zone)
|
logged[i] = reputation.MaskZoneKey(zone)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.settings = append(
|
e.settings = append(
|
||||||
@@ -1220,7 +1231,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
|||||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||||
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
e.check(ExclusivelyAllowedCountriesSetting,
|
||||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1239,15 +1250,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
}{
|
}{
|
||||||
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||||
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
{TagLimitPercentSetting, slices.ContainsFunc(
|
||||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||||
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
||||||
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
||||||
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
||||||
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||||
@@ -1267,7 +1278,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||||
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
e.check(ASNLimitPercentURLSetting,
|
||||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1294,14 +1305,18 @@ func (e *environment) checkTags(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for tag := range cfg.TagLimitPercent {
|
for tag := range cfg.TagLimitPercent {
|
||||||
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkTag returns an error when tag would never apply, as checkTags says.
|
// checkTag returns an error when tag would never apply, as checkTags says.
|
||||||
func checkTag(cfg *WAF, tag string) error {
|
func checkTag(cfg *WAF, tag string) error {
|
||||||
if !requestCanGetTag(cfg, tag) {
|
if !requestCanGetTag(cfg, tag) {
|
||||||
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
|
return fmt.Errorf(
|
||||||
|
"%q %w",
|
||||||
|
reputation.MaskZoneKey(tag),
|
||||||
|
errNoRequestGetsTag,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -1309,7 +1324,7 @@ func checkTag(cfg *WAF, tag string) error {
|
|||||||
|
|
||||||
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
||||||
func requestCanGetTag(cfg *WAF, tag string) bool {
|
func requestCanGetTag(cfg *WAF, tag string) bool {
|
||||||
isListName := func(list TagList) bool { return list.Name == tag }
|
isListName := func(list reputation.TagList) bool { return list.Name == tag }
|
||||||
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -1333,7 +1348,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
|
|||||||
|
|
||||||
return isZoneTag &&
|
return isZoneTag &&
|
||||||
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
||||||
return strings.ToLower(MaskZoneKey(each)) == zone
|
return strings.ToLower(reputation.MaskZoneKey(each)) == zone
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1831,7 +1846,9 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
|
netblock, err := reputation.ParseNetblock(
|
||||||
|
strings.TrimSpace(netblockText),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2037,21 +2054,6 @@ func parseCountry(value string) (string, error) {
|
|||||||
return country, nil
|
return country, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
|
||||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
|
||||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
|
||||||
// read with it too.
|
|
||||||
func ParseASN(value string) (string, error) {
|
|
||||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
|
||||||
|
|
||||||
number, err := strconv.ParseUint(digits, 10, 32)
|
|
||||||
if !hasAS || err != nil {
|
|
||||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
return "AS" + strconv.FormatUint(number, 10), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tags a request can get that the settings can name, as waf
|
// The tags a request can get that the settings can name, as waf
|
||||||
// writes them. The prefixes are followed by a value.
|
// writes them. The prefixes are followed by a value.
|
||||||
const (
|
const (
|
||||||
@@ -2065,24 +2067,17 @@ const (
|
|||||||
ExploitSignatureTag = "exploitsignature"
|
ExploitSignatureTag = "exploitsignature"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
|
||||||
// requests of the clients it holds, and the URL it is fetched from.
|
|
||||||
type TagList struct {
|
|
||||||
Name string
|
|
||||||
URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
||||||
// torexit=https://example.com/tor.txt, and returns each list with its name
|
// torexit=https://example.com/tor.txt, and returns each list with its name
|
||||||
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
||||||
// = before http. A name or a URL listed twice is an error.
|
// = before http. A name or a URL listed twice is an error.
|
||||||
func parseTagLists(value string) ([]TagList, error) {
|
func parseTagLists(value string) ([]reputation.TagList, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
lists := make([]TagList, 0, len(items))
|
lists := make([]reputation.TagList, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
end := strings.Index(item, "=http")
|
end := strings.Index(item, "=http")
|
||||||
@@ -2090,9 +2085,12 @@ func parseTagLists(value string) ([]TagList, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
||||||
}
|
}
|
||||||
|
|
||||||
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
|
list := reputation.TagList{
|
||||||
|
Name: strings.ToLower(item[:end]),
|
||||||
|
URL: item[end+1:],
|
||||||
|
}
|
||||||
|
|
||||||
if slices.ContainsFunc(lists, func(earlier TagList) bool {
|
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool {
|
||||||
return earlier.Name == list.Name || earlier.URL == list.URL
|
return earlier.Name == list.Name || earlier.URL == list.URL
|
||||||
}) {
|
}) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||||
@@ -2105,7 +2103,8 @@ func parseTagLists(value string) ([]TagList, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
||||||
// tag listed twice is an error, which shows the tag as MaskZoneKey does.
|
// tag listed twice is an error, which shows the tag as
|
||||||
|
// reputation.MaskZoneKey does.
|
||||||
func parseTags(value string) ([]string, error) {
|
func parseTags(value string) ([]string, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2125,7 +2124,7 @@ func parseTags(value string) ([]string, error) {
|
|||||||
if slices.Contains(tags, tag) {
|
if slices.Contains(tags, tag) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item),
|
reputation.MaskZoneKey(item),
|
||||||
errListedTwice,
|
errListedTwice,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2194,7 +2193,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if colon <= 0 {
|
if colon <= 0 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item),
|
reputation.MaskZoneKey(item),
|
||||||
errNotTagPercentItem,
|
errNotTagPercentItem,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2208,7 +2207,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if err != nil || percent < 1 || percent > 99 {
|
if err != nil || percent < 1 || percent > 99 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item[colon+1:]),
|
reputation.MaskZoneKey(item[colon+1:]),
|
||||||
errNotTagPercent,
|
errNotTagPercent,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2220,7 +2219,11 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if _, listed := percents[tag]; listed {
|
if _, listed := percents[tag]; listed {
|
||||||
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
||||||
// would not mask.
|
// would not mask.
|
||||||
return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
|
return nil, fmt.Errorf(
|
||||||
|
"%q %w",
|
||||||
|
reputation.MaskZoneKey(tag),
|
||||||
|
errListedTwice,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
percents[tag] = percent
|
percents[tag] = percent
|
||||||
@@ -2254,7 +2257,7 @@ func parsePercents(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := reputation.ParsePercent(percentText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2269,38 +2272,6 @@ func parsePercents(
|
|||||||
return percents, nil
|
return percents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
|
||||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
|
||||||
func ParsePercent(value string) (int64, error) {
|
|
||||||
percent, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || percent < 0 || percent > 100 {
|
|
||||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
|
||||||
}
|
|
||||||
|
|
||||||
return percent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// headerNameChars are the characters RFC 9110 allows in a header name:
|
|
||||||
// letters, digits and these marks.
|
|
||||||
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
|
|
||||||
"0123456789!#$%&'*+-.^_`|~"
|
|
||||||
|
|
||||||
// IsHeaderName reports whether name can be a header name: one or more of
|
|
||||||
// the characters RFC 9110 allows in one.
|
|
||||||
func IsHeaderName(name string) bool {
|
|
||||||
if name == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, char := range name {
|
|
||||||
if !strings.ContainsRune(headerNameChars, char) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseHeaderNames reads a comma-separated list of header names in either
|
// parseHeaderNames reads a comma-separated list of header names in either
|
||||||
// case, and returns them in lower case. Host and Transfer-Encoding are
|
// case, and returns them in lower case. Host and Transfer-Encoding are
|
||||||
// refused: Go's HTTP server takes them out of the request's headers.
|
// refused: Go's HTTP server takes them out of the request's headers.
|
||||||
@@ -2313,7 +2284,7 @@ func parseHeaderNames(value string) ([]string, error) {
|
|||||||
headers := make([]string, 0, len(items))
|
headers := make([]string, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if !IsHeaderName(item) {
|
if !httpguts.ValidHeaderFieldName(item) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2469,8 +2440,8 @@ const (
|
|||||||
// other name for one that does not exist, so that the zone would list no
|
// other name for one that does not exist, so that the zone would list no
|
||||||
// client. A zone listed twice is an error, whatever the case of its
|
// client. A zone listed twice is an error, whatever the case of its
|
||||||
// letters, which DNS names ignore, and whatever its key, since
|
// letters, which DNS names ignore, and whatever its key, since
|
||||||
// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
|
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a
|
||||||
// MaskZoneKey does.
|
// zone as it does.
|
||||||
func parseZones(value string) ([]string, error) {
|
func parseZones(value string) ([]string, error) {
|
||||||
zones, err := parseList(value)
|
zones, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2480,11 +2451,14 @@ func parseZones(value string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
shown := MaskZoneKey(zone)
|
shown := reputation.MaskZoneKey(zone)
|
||||||
listedBefore := slices.ContainsFunc(
|
listedBefore := slices.ContainsFunc(
|
||||||
zones[:i],
|
zones[:i],
|
||||||
func(earlier string) bool {
|
func(earlier string) bool {
|
||||||
return strings.EqualFold(MaskZoneKey(earlier), shown)
|
return strings.EqualFold(
|
||||||
|
reputation.MaskZoneKey(earlier),
|
||||||
|
shown,
|
||||||
|
)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2501,24 +2475,6 @@ func parseZones(value string) ([]string, error) {
|
|||||||
return zones, nil
|
return zones, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
|
||||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
|
||||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
|
||||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
|
||||||
// the process: in the log, the alerts and the metrics.
|
|
||||||
func MaskZoneKey(zone string) string {
|
|
||||||
// DNS names ignore case, and a name may be written with a dot at its
|
|
||||||
// end.
|
|
||||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
|
||||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
|
||||||
return zone
|
|
||||||
}
|
|
||||||
|
|
||||||
_, rest, _ := strings.Cut(zone, ".")
|
|
||||||
|
|
||||||
return masked + "." + rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// isZone reports whether each label of zone is as parseZones takes it. It
|
// isZone reports whether each label of zone is as parseZones takes it. It
|
||||||
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
||||||
func isZone(zone string) bool {
|
func isZone(zone string) bool {
|
||||||
@@ -2574,7 +2530,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
|
|||||||
name, headerValue, found := strings.Cut(item, ":")
|
name, headerValue, found := strings.Cut(item, ":")
|
||||||
name = strings.TrimSpace(name)
|
name = strings.TrimSpace(name)
|
||||||
|
|
||||||
if !found || !IsHeaderName(name) ||
|
if !found || !httpguts.ValidHeaderFieldName(name) ||
|
||||||
strings.ContainsAny(headerValue, "\r\n\x00") {
|
strings.ContainsAny(headerValue, "\r\n\x00") {
|
||||||
return nil, "", fmt.Errorf(
|
return nil, "", fmt.Errorf(
|
||||||
"item %d %w",
|
"item %d %w",
|
||||||
|
|||||||
+29
-20
@@ -18,10 +18,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The settings, by name.
|
// The settings, by name.
|
||||||
@@ -864,6 +866,32 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// config and rules check header names with httpguts.ValidHeaderFieldName.
|
||||||
|
// It takes a name of one or more of the characters RFC 9110 allows in one,
|
||||||
|
// as the check it replaced did: letters, digits and these marks.
|
||||||
|
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const marks = "!#$%&'*+-.^_`|~"
|
||||||
|
|
||||||
|
if httpguts.ValidHeaderFieldName("") {
|
||||||
|
t.Error("the empty name is taken")
|
||||||
|
}
|
||||||
|
|
||||||
|
for b := range 256 {
|
||||||
|
char := byte(b)
|
||||||
|
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
|
||||||
|
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
|
||||||
|
|
||||||
|
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
|
||||||
|
// for a byte above 0x7f.
|
||||||
|
name := string([]byte{char})
|
||||||
|
if got := httpguts.ValidHeaderFieldName(name); got != want {
|
||||||
|
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1694,7 +1722,7 @@ func TestTagSettingsAsSet(t *testing.T) {
|
|||||||
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantLists := []config.TagList{
|
wantLists := []reputation.TagList{
|
||||||
{Name: torExitTag, URL: torURL},
|
{Name: torExitTag, URL: torURL},
|
||||||
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||||
}
|
}
|
||||||
@@ -2288,25 +2316,6 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for zone, want := range map[string]string{
|
|
||||||
spamhaus: spamhausMasked,
|
|
||||||
spamhaus + ".": spamhausMasked + ".",
|
|
||||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
|
||||||
dronebl: dronebl,
|
|
||||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
|
||||||
spamhaus + ".example": spamhaus + ".example",
|
|
||||||
} {
|
|
||||||
if got := config.MaskZoneKey(zone); got != want {
|
|
||||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
// Package dirwatch watches a directory, and calls a function once the
|
||||||
|
// directory has gone a short time without a change after one. The rule
|
||||||
|
// files and the lookup database are read again this way.
|
||||||
|
package dirwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
)
|
||||||
|
|
||||||
|
// quietTime is how long the directory must go without a change before the
|
||||||
|
// function is called, so that a file still being written, such as one
|
||||||
|
// saved in place, appended to or copied in with scp, is read only once
|
||||||
|
// whole.
|
||||||
|
const quietTime = 2 * time.Second
|
||||||
|
|
||||||
|
// Watcher watches one directory. The directory is watched, rather than a
|
||||||
|
// file in it, so that a file replaced by a rename is seen.
|
||||||
|
type Watcher struct {
|
||||||
|
watcher *fsnotify.Watcher
|
||||||
|
}
|
||||||
|
|
||||||
|
// New starts watching dir, or returns why it cannot.
|
||||||
|
func New(dir string) (*Watcher, error) {
|
||||||
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = watcher.Add(dir)
|
||||||
|
if err != nil {
|
||||||
|
_ = watcher.Close()
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Watcher{watcher: watcher}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close stops watching the directory.
|
||||||
|
func (w *Watcher) Close() error {
|
||||||
|
return w.watcher.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallWhenQuiet calls changed each time the directory has gone quietTime
|
||||||
|
// without a change, after a change to a path for which counts is true, until
|
||||||
|
// ctx is done. It also calls failed with each error the watch reports. The
|
||||||
|
// wait starts at once, as if for a change, so that a change made before the
|
||||||
|
// watch began is taken in too.
|
||||||
|
func (w *Watcher) CallWhenQuiet(
|
||||||
|
ctx context.Context,
|
||||||
|
counts func(path string) bool,
|
||||||
|
changed func(),
|
||||||
|
failed func(error),
|
||||||
|
) {
|
||||||
|
callWhenQuiet(
|
||||||
|
ctx, w.watcher.Events, w.watcher.Errors, counts, changed, failed,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// callWhenQuiet is CallWhenQuiet, for the changes from events and the
|
||||||
|
// errors from errs.
|
||||||
|
func callWhenQuiet(
|
||||||
|
ctx context.Context,
|
||||||
|
events <-chan fsnotify.Event,
|
||||||
|
errs <-chan error,
|
||||||
|
counts func(path string) bool,
|
||||||
|
changed func(),
|
||||||
|
failed func(error),
|
||||||
|
) {
|
||||||
|
quiet := time.NewTimer(quietTime)
|
||||||
|
defer quiet.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case event := <-events:
|
||||||
|
if counts(event.Name) {
|
||||||
|
quiet.Reset(quietTime)
|
||||||
|
}
|
||||||
|
case <-quiet.C:
|
||||||
|
changed()
|
||||||
|
case err := <-errs:
|
||||||
|
failed(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package dirwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errWatch is an error the watch reports.
|
||||||
|
var errWatch = errors.New("too many changes")
|
||||||
|
|
||||||
|
func TestDirectoryThatCannotBeWatchedIsAnError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, err := New(filepath.Join(t.TempDir(), "missing"))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("a missing directory is watched, want an error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tests below run callWhenQuiet in a synctest bubble, where time is a
|
||||||
|
// clock of the test's own: time.Sleep moves it on at once, and
|
||||||
|
// synctest.Wait returns once callWhenQuiet waits again, so that every call
|
||||||
|
// due by then is made. The test sends the changes itself, as the watch of
|
||||||
|
// a directory cannot run in a bubble.
|
||||||
|
|
||||||
|
func TestSecondChangeStartsTheWaitAgain(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
loop := start(t, everyPath)
|
||||||
|
|
||||||
|
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 0)
|
||||||
|
|
||||||
|
// The second part starts the wait again.
|
||||||
|
loop.changes <- fsnotify.Event{Name: "50-app.rules", Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 0)
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCalledOnceQuietAfterTheStart(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
// An edit saved before the watch began sends no change, and is
|
||||||
|
// taken in by this call.
|
||||||
|
loop := start(t, everyPath)
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
|
||||||
|
// Long after, it has been called only once.
|
||||||
|
time.Sleep(time.Hour)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChangeThatDoesNotCountIsNoChange(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
loop := start(t, func(path string) bool {
|
||||||
|
return path == "ipinfo_lite.mmdb"
|
||||||
|
})
|
||||||
|
|
||||||
|
// The wait that starts with the watch ends with a call.
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
loop.changes <- fsnotify.Event{Name: "other.mmdb", Op: fsnotify.Create}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 1)
|
||||||
|
|
||||||
|
loop.changes <- fsnotify.Event{Name: "ipinfo_lite.mmdb", Op: fsnotify.Write}
|
||||||
|
|
||||||
|
time.Sleep(quietTime)
|
||||||
|
synctest.Wait()
|
||||||
|
loop.wantCalls(t, 2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestErrorsOfTheWatchHandedOn(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
loop := start(t, everyPath)
|
||||||
|
|
||||||
|
loop.errs <- errWatch
|
||||||
|
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
if !errors.Is(loop.failure, errWatch) {
|
||||||
|
t.Errorf("failed with %v, want %v", loop.failure, errWatch)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// loop is a callWhenQuiet that a test runs.
|
||||||
|
type loop struct {
|
||||||
|
// changes and errs send it changes and errors.
|
||||||
|
changes chan fsnotify.Event
|
||||||
|
errs chan error
|
||||||
|
// calls counts its calls of changed, and failure is the last error it
|
||||||
|
// handed to failed.
|
||||||
|
calls atomic.Int32
|
||||||
|
failure error
|
||||||
|
}
|
||||||
|
|
||||||
|
// start runs callWhenQuiet, with counts, until the test ends.
|
||||||
|
func start(t *testing.T, counts func(path string) bool) *loop {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
l := &loop{changes: make(chan fsnotify.Event), errs: make(chan error)}
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
callWhenQuiet(ctx, l.changes, l.errs, counts,
|
||||||
|
func() { l.calls.Add(1) },
|
||||||
|
func(err error) { l.failure = err },
|
||||||
|
)
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
<-stopped
|
||||||
|
})
|
||||||
|
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
// everyPath counts a change to any path.
|
||||||
|
func everyPath(string) bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantCalls checks how many times changed has been called.
|
||||||
|
func (l *loop) wantCalls(t *testing.T, want int32) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
got := l.calls.Load()
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("changed called %d times, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
+17
-48
@@ -13,18 +13,13 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
"github.com/oschwald/maxminddb-golang/v2"
|
"github.com/oschwald/maxminddb-golang/v2"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// quietTime is how long the lookup database must go without a change
|
|
||||||
// before it is read again, so that a file still being copied in is read
|
|
||||||
// only once whole.
|
|
||||||
const quietTime = 2 * time.Second
|
|
||||||
|
|
||||||
// FileParams are what OpenFile needs.
|
// FileParams are what OpenFile needs.
|
||||||
type FileParams struct {
|
type FileParams struct {
|
||||||
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
// Path is the lookup database, the IPinfo Lite file in its .mmdb form
|
||||||
@@ -122,20 +117,13 @@ func (f *File) ReadFailures() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches the directory of the lookup database until ctx is done,
|
// Watch watches the directory of the lookup database until ctx is done,
|
||||||
// and reads the file again once it has gone without a change for
|
// and reads the file again once it has gone without a change for a short
|
||||||
// quietTime, after it is replaced, written or removed, and after Watch
|
// time, after it is replaced, written or removed, and after Watch starts
|
||||||
// starts watching. If the directory cannot be watched, that is logged, and
|
// watching, as dirwatch describes. A change to another file in its
|
||||||
// the database read at start stays in use.
|
// directory does not count. If the directory cannot be watched, that is
|
||||||
|
// logged, and the database read at start stays in use.
|
||||||
func (f *File) Watch(ctx context.Context) {
|
func (f *File) Watch(ctx context.Context) {
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := dirwatch.New(filepath.Dir(f.params.Path))
|
||||||
if err == nil {
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = watcher.Add(filepath.Dir(f.params.Path))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error(
|
f.params.ProcessLog.Error(
|
||||||
"cannot watch the lookup database for replacements",
|
"cannot watch the lookup database for replacements",
|
||||||
@@ -146,41 +134,22 @@ func (f *File) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
f.params.ProcessLog.Info("watching the lookup database for replacements",
|
||||||
"file", f.params.Path)
|
"file", f.params.Path)
|
||||||
|
|
||||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
watcher.CallWhenQuiet(ctx, f.isDatabase, f.readAgain, func(err error) {
|
||||||
}
|
|
||||||
|
|
||||||
// readAfterChanges reads the lookup database again once quietTime has
|
|
||||||
// passed without a change to it from events, until ctx is done, and logs
|
|
||||||
// the errors from errs. A change to another file in its directory does not
|
|
||||||
// count. The wait starts at once, as if for a change, so that a file
|
|
||||||
// replaced after OpenFile read it, and before its directory was watched,
|
|
||||||
// is read too.
|
|
||||||
func (f *File) readAfterChanges(
|
|
||||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
|
||||||
) {
|
|
||||||
path := filepath.Clean(f.params.Path)
|
|
||||||
|
|
||||||
quiet := time.NewTimer(quietTime)
|
|
||||||
defer quiet.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case event := <-events:
|
|
||||||
if filepath.Clean(event.Name) == path {
|
|
||||||
quiet.Reset(quietTime)
|
|
||||||
}
|
|
||||||
case <-quiet.C:
|
|
||||||
f.readAgain()
|
|
||||||
case err := <-errs:
|
|
||||||
f.params.ProcessLog.Warn("watching the lookup database failed",
|
f.params.ProcessLog.Warn("watching the lookup database failed",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
// isDatabase reports whether path names the lookup database.
|
||||||
|
func (f *File) isDatabase(path string) bool {
|
||||||
|
return filepath.Clean(path) == filepath.Clean(f.params.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the lookup database again, in place of the one in use,
|
// readAgain reads the lookup database again, in place of the one in use,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package lookup
|
package lookup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -12,7 +11,6 @@ import (
|
|||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
@@ -133,66 +131,48 @@ func TestFileThatCannotBeReadIsAnError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
||||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
|
||||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
|
||||||
// reading due by then is done. The test sends the changes itself, as the
|
|
||||||
// watch of a directory cannot run in a bubble.
|
|
||||||
|
|
||||||
func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
||||||
writeDatabase(t, path, "DE")
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
|
f := openFile(t, path, newQueue())
|
||||||
|
|
||||||
|
for changed, want := range map[string]bool{
|
||||||
|
path: true,
|
||||||
|
dir + "/./ipinfo_lite.mmdb": true,
|
||||||
|
filepath.Join(dir, "other.mmdb"): false,
|
||||||
|
} {
|
||||||
|
if f.isDatabase(changed) != want {
|
||||||
|
t.Errorf(
|
||||||
|
"a change to %s counts: %t, want %t",
|
||||||
|
changed,
|
||||||
|
!want,
|
||||||
|
want,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tests below run in a synctest bubble, where time is a clock of the
|
||||||
|
// test's own, so that the time a file is read can be checked. The wait for
|
||||||
|
// the directory to go quiet before a reading is tested in dirwatch.
|
||||||
|
|
||||||
|
func TestReplacementIsReadAgain(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||||
|
writeDatabase(t, path, "DE")
|
||||||
|
|
||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
changes := watch(t, f)
|
|
||||||
|
|
||||||
other := filepath.Join(dir, "replacement.mmdb")
|
writeDatabase(t, path, "KP")
|
||||||
writeDatabase(t, other, "KP")
|
time.Sleep(time.Hour)
|
||||||
|
f.readAgain()
|
||||||
//nolint:gosec // a file the test wrote
|
|
||||||
replacement, err := os.ReadFile(other)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read %s: %v", other, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The file in use is overwritten in place, and keeps giving what
|
|
||||||
// it gave. Its first part alone is not a .mmdb file.
|
|
||||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create %s: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = file.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
half := len(replacement) / 2
|
|
||||||
write(t, file, replacement[:half])
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "DE")
|
|
||||||
|
|
||||||
// The second part starts the wait again.
|
|
||||||
write(t, file, replacement[half:])
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "DE")
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "KP")
|
wantCountry(t, f, "KP")
|
||||||
|
|
||||||
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 {
|
||||||
@@ -219,15 +199,10 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
queue := newQueue()
|
queue := newQueue()
|
||||||
f := openFile(t, path, queue)
|
f := openFile(t, path, queue)
|
||||||
read := f.LastRead()
|
read := f.LastRead()
|
||||||
changes := watch(t, f)
|
|
||||||
|
|
||||||
writeFile(t, path, "not a lookup database\n")
|
writeFile(t, path, "not a lookup database\n")
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
// Long after, the replacement has been read once.
|
|
||||||
time.Sleep(time.Hour)
|
time.Sleep(time.Hour)
|
||||||
synctest.Wait()
|
f.readAgain()
|
||||||
wantCountry(t, f, "DE")
|
wantCountry(t, f, "DE")
|
||||||
|
|
||||||
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
if !f.LastRead().Equal(read) || f.ReadFailures() != 1 {
|
||||||
@@ -240,7 +215,7 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
}
|
}
|
||||||
|
|
||||||
wantAlerts(t, queue, alerts.Alert{
|
wantAlerts(t, queue, alerts.Alert{
|
||||||
Time: read.Add(quietTime),
|
Time: read.Add(time.Hour),
|
||||||
Event: alerts.EventFileError,
|
Event: alerts.EventFileError,
|
||||||
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
Reason: "the lookup database cannot be read, and the one read before stays in use",
|
||||||
Detail: map[string]any{
|
Detail: map[string]any{
|
||||||
@@ -252,55 +227,6 @@ func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "ipinfo_lite.mmdb")
|
|
||||||
writeDatabase(t, path, "DE")
|
|
||||||
|
|
||||||
f := openFile(t, path, newQueue())
|
|
||||||
changes := watch(t, f)
|
|
||||||
|
|
||||||
// The wait that starts with the watch ends with a reading.
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
writeDatabase(t, path, "KP")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "DE")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "KP")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
|
||||||
writeDatabase(t, path, "DE")
|
|
||||||
|
|
||||||
f := openFile(t, path, newQueue())
|
|
||||||
|
|
||||||
// Saved after OpenFile read the file, and before its directory was
|
|
||||||
// watched, so that no change is seen for it.
|
|
||||||
writeDatabase(t, path, "KP")
|
|
||||||
watch(t, f)
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantCountry(t, f, "KP")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newQueue returns a queue of alerts for a webhook that is never sent
|
// newQueue returns a queue of alerts for a webhook that is never sent
|
||||||
// them, so that they wait in it for the test to look at.
|
// them, so that they wait in it for the test to look at.
|
||||||
func newQueue() *alerts.Queue {
|
func newQueue() *alerts.Queue {
|
||||||
@@ -344,28 +270,6 @@ func openFile(t *testing.T, path string, queue *alerts.Queue) *File {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
// watch runs f's readAfterChanges until the test ends, and returns the
|
|
||||||
// channel that sends it changes.
|
|
||||||
func watch(t *testing.T, f *File) chan<- fsnotify.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes := make(chan fsnotify.Event)
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
f.readAfterChanges(ctx, changes, nil)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
return changes
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantCountry checks the country f gives testClient.
|
// wantCountry checks the country f gives testClient.
|
||||||
func wantCountry(t *testing.T, f *File, want string) {
|
func wantCountry(t *testing.T, f *File, want string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -401,13 +305,3 @@ func writeFile(t *testing.T, path, content string) {
|
|||||||
t.Fatalf("write %s: %v", path, err)
|
t.Fatalf("write %s: %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// write writes data to the end of file.
|
|
||||||
func write(t *testing.T, file *os.File, data []byte) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := file.Write(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
@@ -301,7 +300,7 @@ const sourceLabel = "source"
|
|||||||
|
|
||||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
|
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||||
// the fetches that failed and when the copy in use was fetched, and for a
|
// the fetches that failed and when the copy in use was fetched, and for a
|
||||||
@@ -320,7 +319,9 @@ func (m *Metrics) AddReputation(
|
|||||||
m.registry.MustRegister(m.reputationHits)
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
for _, zone := range dnsbl.Zones() {
|
||||||
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
|
source := prometheus.Labels{
|
||||||
|
sourceLabel: reputation.MaskZoneKey(zone),
|
||||||
|
}
|
||||||
|
|
||||||
m.addReputationQueries(
|
m.addReputationQueries(
|
||||||
source,
|
source,
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
|
|
||||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -32,6 +31,8 @@ const (
|
|||||||
// it fails, and no client is checked with AbuseIPDB after a check
|
// it fails, and no client is checked with AbuseIPDB after a check
|
||||||
// fails, so that a source refusing them is not asked on every request.
|
// fails, so that a source refusing them is not asked on every request.
|
||||||
failureDelay = time.Minute
|
failureDelay = time.Minute
|
||||||
|
// maskedKey is what MaskZoneKey shows in place of a zone's key.
|
||||||
|
maskedKey = "********"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -131,6 +132,24 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
||||||
|
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
||||||
|
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
||||||
|
// other zone it returns as it is. A zone is shown so wherever it leaves
|
||||||
|
// the process: in the log, the alerts and the metrics.
|
||||||
|
func MaskZoneKey(zone string) string {
|
||||||
|
// DNS names ignore case, and a name may be written with a dot at its
|
||||||
|
// end.
|
||||||
|
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||||
|
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
||||||
|
return zone
|
||||||
|
}
|
||||||
|
|
||||||
|
_, rest, _ := strings.Cut(zone, ".")
|
||||||
|
|
||||||
|
return maskedKey + "." + rest
|
||||||
|
}
|
||||||
|
|
||||||
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
||||||
func (d *DNSBL) Zones() []string {
|
func (d *DNSBL) Zones() []string {
|
||||||
return slices.Clone(d.params.Zones)
|
return slices.Clone(d.params.Zones)
|
||||||
@@ -138,7 +157,7 @@ func (d *DNSBL) Zones() []string {
|
|||||||
|
|
||||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||||
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
||||||
// masked, as config.MaskZoneKey masks it, since they go to the request
|
// masked, as MaskZoneKey masks it, since they go to the request
|
||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
@@ -163,7 +182,7 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
listedBy = append(listedBy, MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||||
len(d.asking) < maxQueries:
|
len(d.asking) < maxQueries:
|
||||||
@@ -292,7 +311,7 @@ func (d *DNSBL) ask(ctx context.Context, q query) {
|
|||||||
if isFailure {
|
if isFailure {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := config.MaskZoneKey(q.zone)
|
shown := MaskZoneKey(q.zone)
|
||||||
|
|
||||||
// Raised before it is logged, so that the alert is there once the
|
// Raised before it is logged, so that the alert is there once the
|
||||||
// log line is.
|
// log line is.
|
||||||
|
|||||||
@@ -860,3 +860,28 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
|||||||
t.Errorf("asked about %v, want %v", got, want)
|
t.Errorf("asked about %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
dronebl = "dnsbl.dronebl.org"
|
||||||
|
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
||||||
|
spamhausMasked = "********.xbl.dq.spamhaus.net"
|
||||||
|
)
|
||||||
|
|
||||||
|
for zone, want := range map[string]string{
|
||||||
|
spamhaus: spamhausMasked,
|
||||||
|
spamhaus + ".": spamhausMasked + ".",
|
||||||
|
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
||||||
|
dronebl: dronebl,
|
||||||
|
"dq.spamhaus.net": "dq.spamhaus.net",
|
||||||
|
spamhaus + ".example": spamhaus + ".example",
|
||||||
|
} {
|
||||||
|
if got := reputation.MaskZoneKey(zone); got != want {
|
||||||
|
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,8 +7,10 @@
|
|||||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||||
// today. The state package writes all of these to files and reads them
|
// today. The state package writes all of these to files and reads them
|
||||||
// from there, so that a restart keeps them too. Last, it fetches
|
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
|
||||||
// AbuseIPDB's blacklist, which it does not keep.
|
// blacklist, which it does not keep. Last, it reads the netblocks, AS
|
||||||
|
// numbers and percentages that both the settings and the lists hold, and
|
||||||
|
// masks the keys of DNSBL zones.
|
||||||
package reputation
|
package reputation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -20,12 +22,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -42,12 +44,25 @@ const (
|
|||||||
var (
|
var (
|
||||||
errStatus = errors.New("the server answered")
|
errStatus = errors.New("the server answered")
|
||||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||||
errNotNetblock = errors.New(
|
errNotNetblockLine = errors.New(
|
||||||
"is not an address or a netblock, such as 192.0.2.0/24",
|
"is not an address or a netblock, such as 192.0.2.0/24",
|
||||||
)
|
)
|
||||||
errNotASNPercent = errors.New(
|
errNotASNPercent = errors.New(
|
||||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||||
|
errNotNetblock = errors.New(
|
||||||
|
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||||
|
errNotASN = errors.New("is not an AS number such as AS64496")
|
||||||
|
errNotPercent = errors.New(
|
||||||
|
"is not a percentage, a whole number from 0 to 100",
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
||||||
|
// requests of the clients it holds, and the URL it is fetched from.
|
||||||
|
type TagList struct {
|
||||||
|
Name string
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
// List is a list as lists.json holds it: the URL it is fetched from,
|
// List is a list as lists.json holds it: the URL it is fetched from,
|
||||||
// when it was last tried, the fetch failed or not, and its last good copy:
|
// when it was last tried, the fetch failed or not, and its last good copy:
|
||||||
@@ -68,7 +83,7 @@ type Params struct {
|
|||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
ASNLimitPercentURL string
|
ASNLimitPercentURL string
|
||||||
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
||||||
TagLists []config.TagList
|
TagLists []TagList
|
||||||
// Refresh is how long after a list was last fetched or tried it is
|
// Refresh is how long after a list was last fetched or tried it is
|
||||||
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
||||||
Refresh time.Duration
|
Refresh time.Duration
|
||||||
@@ -465,7 +480,11 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
|
|
||||||
netblock, ok := parseNetblock(text)
|
netblock, ok := parseNetblock(text)
|
||||||
if !ok {
|
if !ok {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
|
return entries{}, fmt.Errorf(
|
||||||
|
"line %d %w",
|
||||||
|
i+1,
|
||||||
|
errNotNetblockLine,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
found.netblocks[netblock] = true
|
found.netblocks[netblock] = true
|
||||||
@@ -485,7 +504,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
||||||
// stands for none, is refused.
|
// stands for none, is refused.
|
||||||
func parseNetblock(text string) (netip.Prefix, bool) {
|
func parseNetblock(text string) (netip.Prefix, bool) {
|
||||||
netblock, err := config.ParseNetblock(text)
|
netblock, err := ParseNetblock(text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netip.Prefix{}, false
|
return netip.Prefix{}, false
|
||||||
}
|
}
|
||||||
@@ -520,9 +539,9 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
asnText, percentText, _ := strings.Cut(text, ":")
|
asnText, percentText, _ := strings.Cut(text, ":")
|
||||||
asn, asnErr := config.ParseASN(asnText)
|
asn, asnErr := ParseASN(asnText)
|
||||||
|
|
||||||
percent, percentErr := config.ParsePercent(percentText)
|
percent, percentErr := ParsePercent(percentText)
|
||||||
if asnErr != nil || percentErr != nil {
|
if asnErr != nil || percentErr != nil {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
||||||
}
|
}
|
||||||
@@ -536,6 +555,53 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
return found, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||||
|
// address is a netblock of that address alone, a /32 or a /128. A
|
||||||
|
// blocklist's lines are read with it too.
|
||||||
|
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||||
|
if strings.Contains(value, "/") {
|
||||||
|
netblock, err := netip.ParsePrefix(value)
|
||||||
|
if err != nil {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netblock.Masked(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(value)
|
||||||
|
if err != nil || addr.Zone() != "" {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseASN reads an AS number such as AS64496, in either case, and
|
||||||
|
// returns it as the lookup gives it: AS and the number, in capitals and
|
||||||
|
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
||||||
|
// read with it too.
|
||||||
|
func ParseASN(value string) (string, error) {
|
||||||
|
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
||||||
|
|
||||||
|
number, err := strconv.ParseUint(digits, 10, 32)
|
||||||
|
if !hasAS || err != nil {
|
||||||
|
return "", fmt.Errorf("%q %w", value, errNotASN)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "AS" + strconv.FormatUint(number, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
||||||
|
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
||||||
|
func ParsePercent(value string) (int64, error) {
|
||||||
|
percent, err := strconv.ParseInt(value, 10, 64)
|
||||||
|
if err != nil || percent < 0 || percent > 100 {
|
||||||
|
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return percent, nil
|
||||||
|
}
|
||||||
|
|
||||||
// withoutComment returns line without anything after a ; or a #, and
|
// withoutComment returns line without anything after a ; or a #, and
|
||||||
// without the spaces around what is left.
|
// without the spaces around what is left.
|
||||||
func withoutComment(line string) string {
|
func withoutComment(line string) string {
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ package rproxy
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
|
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
||||||
|
// long requests take cannot matter to it. It is called before s serves.
|
||||||
|
func (s *Server) StopBreakerClock(at time.Time) {
|
||||||
|
s.h.breakerClock = func() time.Time { return at }
|
||||||
|
}
|
||||||
|
|
||||||
// Breaker is the circuit breaker, for tests that give it the time.
|
// Breaker is the circuit breaker, for tests that give it the time.
|
||||||
type Breaker struct {
|
type Breaker struct {
|
||||||
b breaker
|
b breaker
|
||||||
|
|||||||
@@ -870,10 +870,14 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
|
|||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// rproxy is waiting for the rest of the body well within this limit, even on
|
||||||
|
// a busy host, and answers when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "3000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1262,7 +1266,9 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const limit = 2 * time.Second
|
// The app's answer arrives well within this limit, even on a busy host, and
|
||||||
|
// the connection closes when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, answerBeforeTheBody)
|
app := startApp(t, answerBeforeTheBody)
|
||||||
@@ -1427,17 +1433,20 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
|
|||||||
|
|
||||||
const answer = "the end of a long answer"
|
const answer = "the end of a long answer"
|
||||||
|
|
||||||
|
// The request reaches the app well within this limit, even on a busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
_ = http.NewResponseController(w).Flush()
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
_, _ = io.WriteString(w, answer)
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
got := visit(
|
got := visit(
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ func New(params Params) *Server {
|
|||||||
reportURL: (&url.URL{
|
reportURL: (&url.URL{
|
||||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||||
}).String(),
|
}).String(),
|
||||||
|
breakerClock: time.Now,
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Server{
|
return &Server{
|
||||||
@@ -133,8 +134,10 @@ type handler struct {
|
|||||||
wafTransport http.RoundTripper
|
wafTransport http.RoundTripper
|
||||||
// reportURL is where the reports go.
|
// reportURL is where the reports go.
|
||||||
reportURL string
|
reportURL string
|
||||||
// breaker is the circuit breaker for asks to waf.
|
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
||||||
|
// the time.
|
||||||
breaker breaker
|
breaker breaker
|
||||||
|
breakerClock func() time.Time
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
// reports waits for the reports being sent. reportsMu guards starting one
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
// against reportsStopped, which is set once nothing is to wait for another.
|
||||||
reports sync.WaitGroup
|
reports sync.WaitGroup
|
||||||
|
|||||||
@@ -356,6 +356,19 @@ func startServerLogging(
|
|||||||
) (string, *rproxy.Server) {
|
) (string, *rproxy.Server) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
||||||
|
|
||||||
|
return serve(t, server), server
|
||||||
|
}
|
||||||
|
|
||||||
|
// newServer returns rproxy, not yet serving, with its request lines going to
|
||||||
|
// requestLog and its process lines to processLog.
|
||||||
|
func newServer(
|
||||||
|
t *testing.T, backend, appURL string, env map[string]string,
|
||||||
|
requestLog, processLog io.Writer,
|
||||||
|
) *rproxy.Server {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
wafBackendTimeout: "60000",
|
wafBackendTimeout: "60000",
|
||||||
@@ -367,7 +380,7 @@ func startServerLogging(
|
|||||||
t.Fatalf("settings %v: %v", settings, err)
|
t.Fatalf("settings %v: %v", settings, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
return rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
@@ -376,6 +389,11 @@ func startServerLogging(
|
|||||||
Level: slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve has server serve until the test ends, and returns its address.
|
||||||
|
func serve(t *testing.T, server *rproxy.Server) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
t.Context(),
|
t.Context(),
|
||||||
@@ -396,7 +414,7 @@ func startServerLogging(
|
|||||||
server.WaitForReports()
|
server.WaitForReports()
|
||||||
})
|
})
|
||||||
|
|
||||||
return listener.Addr().String(), server
|
return listener.Addr().String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// answered is the visitor's answer.
|
// answered is the visitor's answer.
|
||||||
@@ -1434,14 +1452,18 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|||||||
|
|
||||||
greet(w, r)
|
greet(w, r)
|
||||||
})
|
})
|
||||||
addr, out := startRProxy(
|
out := &output{}
|
||||||
t,
|
server := newServer(
|
||||||
standIn.URL,
|
t, standIn.URL, app.URL,
|
||||||
app.URL,
|
map[string]string{wafBackendTimeout: "200"},
|
||||||
map[string]string{
|
out, out,
|
||||||
wafBackendTimeout: "200",
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
// The breaker's clock stands still, as in
|
||||||
|
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
|
||||||
|
// failures could take more than five seconds, and the next request could
|
||||||
|
// come more than a second later.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
// An ask can time out before it reaches the stand-in, so these
|
||||||
// may leave fewer than five questions, or leave some late.
|
// may leave fewer than five questions, or leave some late.
|
||||||
@@ -1532,9 +1554,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
|||||||
tooLarge(w, r)
|
tooLarge(w, r)
|
||||||
})
|
})
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
out := &output{}
|
||||||
wafBackendTimeout: "60000",
|
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
||||||
})
|
// The breaker's clock stands still. On a slow host, the five failures could
|
||||||
|
// otherwise take more than the five seconds that open the breaker. The next
|
||||||
|
// request could come more than a second later, when it may ask waf again.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
for range 5 {
|
for range 5 {
|
||||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||||
@@ -1809,11 +1835,15 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// The app's answer to the upgrade arrives well within these limits, even on a
|
||||||
|
// busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, echoAfterUpgrade)
|
app := startApp(t, echoAfterUpgrade)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
maxResponseTime: "2000",
|
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1838,7 +1868,10 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Past both time limits, the connection still carries what is sent.
|
// Past both time limits, the connection still carries what is sent.
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
|
// dial's deadline ran from before the upgrade and this wait.
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||||
|
|
||||||
const message = "still here\n"
|
const message = "still here\n"
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"maps"
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
) (*wafAnswer, error) {
|
) (*wafAnswer, error) {
|
||||||
b := &rq.h.breaker
|
b := &rq.h.breaker
|
||||||
if !b.mayAsk(time.Now()) {
|
if !b.mayAsk(rq.h.breakerClock()) {
|
||||||
return nil, errBreakerOpen
|
return nil, errBreakerOpen
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
case err == nil:
|
case err == nil:
|
||||||
b.noteAnswer()
|
b.noteAnswer()
|
||||||
case ctx.Err() == nil:
|
case ctx.Err() == nil:
|
||||||
b.noteFailure(time.Now())
|
b.noteFailure(rq.h.breakerClock())
|
||||||
}
|
}
|
||||||
|
|
||||||
return answer, err
|
return answer, err
|
||||||
|
|||||||
+21
-50
@@ -18,12 +18,11 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/dirwatch"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -40,12 +39,6 @@ const (
|
|||||||
// extension ends the name of every rule file.
|
// extension ends the name of every rule file.
|
||||||
const extension = ".rules"
|
const extension = ".rules"
|
||||||
|
|
||||||
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
|
||||||
// the rule files are read again, so that a file still being written, such
|
|
||||||
// as one saved in place, appended to or copied in with scp, is read only
|
|
||||||
// once whole.
|
|
||||||
const quietTime = 2 * time.Second
|
|
||||||
|
|
||||||
// headerTarget starts the target that is one request header,
|
// headerTarget starts the target that is one request header,
|
||||||
// header:<Name>.
|
// header:<Name>.
|
||||||
const headerTarget = "header:"
|
const headerTarget = "header:"
|
||||||
@@ -176,26 +169,18 @@ func (f *Files) Len() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Watch watches Dir until ctx is done, and reads the rule files again
|
// Watch watches Dir until ctx is done, and reads the rule files again
|
||||||
// once Dir has had no change for quietTime, after one is edited, added or
|
// once Dir has had no change for a short time, after one is edited, added
|
||||||
// removed, and after Watch starts watching. If they then hold an error,
|
// or removed, and after Watch starts watching, as dirwatch describes. If
|
||||||
// the rules stay as they were, the error is logged with its file and
|
// they then hold an error, the rules stay as they were, the error is
|
||||||
// line, and the files are read again after the next change. If Dir cannot
|
// logged with its file and line, and the files are read again after the
|
||||||
// be watched, that is logged, and the rules stay as they were loaded.
|
// next change. If Dir cannot be watched, that is logged, and the rules
|
||||||
// While Enabled is false, Watch returns at once.
|
// stay as they were loaded. While Enabled is false, Watch returns at once.
|
||||||
func (f *Files) Watch(ctx context.Context) {
|
func (f *Files) Watch(ctx context.Context) {
|
||||||
if !f.params.Enabled {
|
if !f.params.Enabled {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := dirwatch.New(f.params.Dir)
|
||||||
if err == nil {
|
|
||||||
defer func() {
|
|
||||||
_ = watcher.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = watcher.Add(f.params.Dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
f.params.ProcessLog.Error("cannot watch the rule files for edits",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
@@ -203,36 +188,22 @@ func (f *Files) Watch(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = watcher.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
f.params.ProcessLog.Info("watching the rule files for edits",
|
f.params.ProcessLog.Info("watching the rule files for edits",
|
||||||
"directory", f.params.Dir)
|
"directory", f.params.Dir)
|
||||||
|
|
||||||
f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
|
// A change to any file in Dir counts.
|
||||||
}
|
watcher.CallWhenQuiet(ctx,
|
||||||
|
func(string) bool { return true },
|
||||||
// readAfterChanges reads the rule files again once quietTime has passed
|
f.readAgain,
|
||||||
// without a change from events, until ctx is done, and logs the errors
|
func(err error) {
|
||||||
// from errs. The wait starts at once, as if for a change, so that an edit
|
|
||||||
// saved after Load read the files, and before Dir was watched, is taken
|
|
||||||
// in too.
|
|
||||||
func (f *Files) readAfterChanges(
|
|
||||||
ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
|
|
||||||
) {
|
|
||||||
quiet := time.NewTimer(quietTime)
|
|
||||||
defer quiet.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-events:
|
|
||||||
quiet.Reset(quietTime)
|
|
||||||
case <-quiet.C:
|
|
||||||
f.readAgain()
|
|
||||||
case err := <-errs:
|
|
||||||
f.params.ProcessLog.Warn("watching the rule files failed",
|
f.params.ProcessLog.Warn("watching the rule files failed",
|
||||||
"error", err.Error())
|
"error", err.Error())
|
||||||
}
|
},
|
||||||
}
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAgain reads the rule files again, in place of the rules loaded, or
|
// readAgain reads the rule files again, in place of the rules loaded, or
|
||||||
@@ -378,7 +349,7 @@ func parse(line string) (Rule, bool, error) {
|
|||||||
rule.Target,
|
rule.Target,
|
||||||
errNotTarget,
|
errNotTarget,
|
||||||
)
|
)
|
||||||
case isHeader && !config.IsHeaderName(headerName):
|
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
|
||||||
return Rule{}, false, fmt.Errorf(
|
return Rule{}, false, fmt.Errorf(
|
||||||
"the target %q %w",
|
"the target %q %w",
|
||||||
rule.Target,
|
rule.Target,
|
||||||
|
|||||||
@@ -1,165 +0,0 @@
|
|||||||
package rules
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The tests below run readAfterChanges in a synctest bubble, where time is
|
|
||||||
// a clock of the test's own: time.Sleep moves it on at once, and
|
|
||||||
// synctest.Wait returns once readAfterChanges waits again, so that every
|
|
||||||
// reading due by then is done. The test sends the changes itself, as the
|
|
||||||
// watch of a directory cannot run in a bubble.
|
|
||||||
|
|
||||||
func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "50-app.rules")
|
|
||||||
writeFile(t, path, "first path block ^/first\n")
|
|
||||||
files := load(t, dir)
|
|
||||||
changes := run(t, files)
|
|
||||||
|
|
||||||
file, err := os.Create(path) //nolint:gosec // a file the test wrote
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = file.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// The first part ends in the middle of a ban rule's regex, which,
|
|
||||||
// read then, would ban every request.
|
|
||||||
write(t, file, "first path block ^/first\nprobe path ban ^/")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/anything")
|
|
||||||
|
|
||||||
// The second part starts the wait again.
|
|
||||||
write(t, file, `\.env$`+"\n")
|
|
||||||
|
|
||||||
changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
|
|
||||||
|
|
||||||
time.Sleep(quietTime - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/.env")
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/.env", "probe")
|
|
||||||
wantMatched(t, files, "/anything")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "50-app.rules")
|
|
||||||
writeFile(t, path, "first path block ^/first\n")
|
|
||||||
files := load(t, dir)
|
|
||||||
|
|
||||||
// Saved after Load read the files, and before the directory was
|
|
||||||
// watched, so that no change is seen for it.
|
|
||||||
writeFile(t, path, "first path block ^/edited\n")
|
|
||||||
run(t, files)
|
|
||||||
time.Sleep(quietTime)
|
|
||||||
synctest.Wait()
|
|
||||||
wantMatched(t, files, "/edited", "first")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// load loads the rules in dir.
|
|
||||||
func load(t *testing.T, dir string) *Files {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
files, err := Load(Params{
|
|
||||||
Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
|
|
||||||
Alerts: alerts.New(alerts.Params{}),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("load: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
// run runs files' readAfterChanges until the test ends, and returns the
|
|
||||||
// channel that sends it changes.
|
|
||||||
func run(t *testing.T, files *Files) chan<- fsnotify.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes := make(chan fsnotify.Event)
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
files.readAfterChanges(ctx, changes, nil)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
return changes
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeFile writes content to the file at path.
|
|
||||||
func writeFile(t *testing.T, path, content string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
err := os.WriteFile(path, []byte(content), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write %s: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// write writes text to the end of file.
|
|
||||||
func write(t *testing.T, file *os.File, text string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := file.WriteString(text)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantMatched checks the ids of the rules that a GET request for path
|
|
||||||
// matches, in order.
|
|
||||||
func wantMatched(t *testing.T, files *Files, path string, want ...string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
|
||||||
"http://app.example"+path, nil)
|
|
||||||
|
|
||||||
matched := files.Match(r)
|
|
||||||
|
|
||||||
got := make([]string, 0, len(matched))
|
|
||||||
for _, rule := range matched {
|
|
||||||
got = append(got, rule.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("%s matched %v, want %v", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
|
|||||||
|
|
||||||
// A second ban, made while the first waits to be written, puts the
|
// A second ban, made while the first waits to be written, puts the
|
||||||
// write off no further, and is written with it.
|
// write off no further, and is written with it.
|
||||||
first, _ := params.Ledger.BanForLimit(
|
first, _ := params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
second, _ := params.Ledger.BanForLimit(
|
second, _ := params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5*time.Second - time.Nanosecond)
|
time.Sleep(5*time.Second - time.Nanosecond)
|
||||||
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
|
|||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// A ban, and bans.json written with it.
|
// A ban, and bans.json written with it.
|
||||||
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
|
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(), bans.Notes{})
|
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
time.Sleep(params.BansWriteDelay)
|
time.Sleep(params.BansWriteDelay)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
|
|
||||||
// A second ban is to be written BansWriteDelay later. Just before
|
// A second ban is to be written BansWriteDelay later. Just before
|
||||||
// then, an admin saves bans.json with the first ban lifted and
|
// then, an admin saves bans.json with the first ban lifted and
|
||||||
// another added.
|
// another added.
|
||||||
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
|
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(), bans.Notes{})
|
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
edit(t, dir, bansJSON, permanentBansJSON)
|
edit(t, dir, bansJSON, permanentBansJSON)
|
||||||
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.BanForLimit(
|
params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
params.Limiter.Count(
|
params.Limiter.Count(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.BanForLimit(
|
params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
err = files.WriteAll()
|
err = files.WriteAll()
|
||||||
@@ -2135,7 +2135,11 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
|||||||
|
|
||||||
// The ban that liftedBansJSON lifts, before it is lifted.
|
// The ban that liftedBansJSON lifts, before it is lifted.
|
||||||
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
||||||
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
params.Ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
||||||
if !banned {
|
if !banned {
|
||||||
@@ -2510,11 +2514,15 @@ func fill(params state.Params) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
params.Ledger.Load([]bans.Ban{permanentBan()})
|
params.Ledger.Load([]bans.Ban{permanentBan()})
|
||||||
params.Ledger.BanForLimit(client, now, bans.Notes{
|
params.Ledger.Ban(client, now, bans.Grounds{
|
||||||
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
|
||||||
|
})
|
||||||
|
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
|
||||||
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseAttack,
|
||||||
|
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
|
||||||
})
|
})
|
||||||
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
|
|
||||||
bans.Notes{RuleID: "env-file", Target: "path"})
|
|
||||||
|
|
||||||
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
||||||
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
||||||
@@ -2700,7 +2708,11 @@ func wantLiftedBanKept(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Were the lifted ban counted, the next would last three hours.
|
// Were the lifted ban counted, the next would last three hours.
|
||||||
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
afterLifting(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour {
|
if ban.Expires.Sub(ban.Start) != time.Hour {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban lasts %s, want 1h",
|
"the next ban lasts %s, want 1h",
|
||||||
|
|||||||
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
||||||
// read from it ends with an error once waitLimit is up.
|
|
||||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
})
|
})
|
||||||
|
|
||||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set the deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
_, err = io.WriteString(conn, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send: %v", err)
|
t.Fatalf("send: %v", err)
|
||||||
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
|
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
||||||
|
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
text, err := io.ReadAll(conn)
|
||||||
if err != nil || len(text) != 0 {
|
if err != nil || len(text) != 0 {
|
||||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||||
|
|||||||
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
|||||||
"",
|
"",
|
||||||
map[string]string{metricsToken: token},
|
map[string]string{metricsToken: token},
|
||||||
)
|
)
|
||||||
server.Ledger.BanForLimit(
|
server.Ledger.Ban(
|
||||||
netip.MustParsePrefix(otherClient+"/32"),
|
netip.MustParsePrefix(otherClient+"/32"),
|
||||||
clk.Now(),
|
clk.Now(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
before := server.Ledger.Snapshot()
|
before := server.Ledger.Snapshot()
|
||||||
|
|
||||||
|
|||||||
@@ -130,11 +130,9 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
|
|||||||
// A ban for a clear sign of attack, which a request under it would make
|
// A ban for a clear sign of attack, which a request under it would make
|
||||||
// permanent.
|
// permanent.
|
||||||
group := netip.MustParsePrefix(ipv6Group)
|
group := netip.MustParsePrefix(ipv6Group)
|
||||||
attackBan, _ := server.Ledger.BanForAttack(
|
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
|
||||||
group,
|
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
|
||||||
start,
|
})
|
||||||
bans.Notes{RuleID: "probe"},
|
|
||||||
)
|
|
||||||
|
|
||||||
// The third request breaks the limit, and so does the fourth, within the
|
// The third request breaks the limit, and so does the fourth, within the
|
||||||
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
||||||
|
|||||||
@@ -686,10 +686,10 @@ func askDirectly(
|
|||||||
|
|
||||||
// banClient bans client at now.
|
// banClient bans client at now.
|
||||||
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||||
server.Ledger.BanForLimit(
|
server.Ledger.Ban(
|
||||||
netip.MustParsePrefix(client+"/32"),
|
netip.MustParsePrefix(client+"/32"),
|
||||||
now,
|
now,
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+66
-47
@@ -103,9 +103,8 @@ func (rq *request) countedBytes() int64 {
|
|||||||
// one whose bytes broke a byte limit, the refusal for one that broke the
|
// one whose bytes broke a byte limit, the refusal for one that broke the
|
||||||
// refusal limit. The ban's notes give the client's limit percentage for a
|
// refusal limit. The ban's notes give the client's limit percentage for a
|
||||||
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
||||||
// the client's counters back to zero. In observe mode it makes no ban and
|
// the client's counters back to zero. In observe mode, banClient says what
|
||||||
// sets nothing back, and raises the alert for the ban it would have made,
|
// happens instead.
|
||||||
// if that alert would be sent.
|
|
||||||
func (rq *request) banForLimit(
|
func (rq *request) banForLimit(
|
||||||
now time.Time,
|
now time.Time,
|
||||||
hit ratelimit.Hit,
|
hit ratelimit.Hit,
|
||||||
@@ -123,23 +122,11 @@ func (rq *request) banForLimit(
|
|||||||
rq.line.Offence = requestlog.OffenceLimit
|
rq.line.Offence = requestlog.OffenceLimit
|
||||||
rq.h.metrics.LimitHit(hit)
|
rq.h.metrics.LimitHit(hit)
|
||||||
|
|
||||||
netblock := banNetblock(rq.h.config, rq.client)
|
|
||||||
if rq.h.config.Observe &&
|
|
||||||
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
notes := bans.Notes{
|
notes := bans.Notes{
|
||||||
ASN: rq.line.ASN,
|
|
||||||
ASName: rq.line.ASName,
|
|
||||||
Country: rq.line.Country,
|
|
||||||
Kind: hit.Kind,
|
Kind: hit.Kind,
|
||||||
Limit: hit.Limit,
|
Limit: hit.Limit,
|
||||||
Window: hit.Window,
|
Window: hit.Window,
|
||||||
Count: hit.Count,
|
Count: hit.Count,
|
||||||
Reputation: rq.reputation,
|
|
||||||
Request: rq.noted(now, status),
|
|
||||||
Requests: rq.netblockRequests(netblock),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
switch hit.Kind {
|
switch hit.Kind {
|
||||||
@@ -149,47 +136,54 @@ func (rq *request) banForLimit(
|
|||||||
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.h.config.Observe {
|
rq.banClient(
|
||||||
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
|
now,
|
||||||
if wouldBan {
|
status,
|
||||||
rq.alertBan(ban)
|
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
|
||||||
}
|
)
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
|
|
||||||
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
|
||||||
rq.line.BanExpires = banExpires(ban)
|
|
||||||
|
|
||||||
if made {
|
|
||||||
rq.alertBan(ban)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForAttack bans the client's netblock at now for a clear sign of
|
// banForAttack bans the client's netblock at now for a clear sign of
|
||||||
// attack, which notes name: the ban rule that matched, or the trap path
|
// attack, which notes name: the ban rule that matched, or the trap path
|
||||||
// asked for. It fills in the rest of the notes. In observe mode it makes
|
// asked for. It returns the action for that ban. In observe mode, banClient
|
||||||
// no ban, and raises the alert for the ban it would have made, if that
|
// says what happens instead.
|
||||||
// alert would be sent. It returns the action for that ban.
|
|
||||||
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
||||||
netblock := banNetblock(rq.h.config, rq.client)
|
return rq.banClient(
|
||||||
if rq.h.config.Observe &&
|
now,
|
||||||
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
rq.h.config.BanResponse,
|
||||||
return banAction(
|
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
||||||
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
notes.ASN = rq.line.ASN
|
// banClient bans the client's netblock at now on grounds, and returns the
|
||||||
notes.ASName = rq.line.ASName
|
// action for that ban. It adds to the notes what the request tells of the
|
||||||
notes.Country = rq.line.Country
|
// client, with status, what the client was sent. A ban for a broken limit
|
||||||
notes.Reputation = rq.reputation
|
// sets the client's counters back to zero.
|
||||||
notes.Request = rq.noted(now, rq.h.config.BanResponse)
|
//
|
||||||
notes.Requests = rq.netblockRequests(netblock)
|
// In observe mode it makes no ban and sets nothing back. It raises the
|
||||||
|
// alert for the ban it would have made, if that alert would be sent.
|
||||||
|
func (rq *request) banClient(
|
||||||
|
now time.Time,
|
||||||
|
status int,
|
||||||
|
grounds bans.Grounds,
|
||||||
|
) string {
|
||||||
|
netblock := banNetblock(rq.h.config, rq.client)
|
||||||
|
|
||||||
if rq.h.config.Observe {
|
if rq.h.config.Observe {
|
||||||
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
|
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
|
||||||
|
return banAction(
|
||||||
|
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
grounds.Notes = rq.withClientNotes(
|
||||||
|
grounds.Notes,
|
||||||
|
now,
|
||||||
|
status,
|
||||||
|
netblock,
|
||||||
|
)
|
||||||
|
|
||||||
|
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
|
||||||
if wouldBan {
|
if wouldBan {
|
||||||
rq.alertBan(ban)
|
rq.alertBan(ban)
|
||||||
}
|
}
|
||||||
@@ -197,7 +191,13 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
|||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
|
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
|
||||||
|
|
||||||
|
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
|
||||||
|
if grounds.Cause == bans.CauseLimit {
|
||||||
|
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
||||||
|
}
|
||||||
|
|
||||||
rq.line.BanExpires = banExpires(ban)
|
rq.line.BanExpires = banExpires(ban)
|
||||||
|
|
||||||
if made {
|
if made {
|
||||||
@@ -207,6 +207,25 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
|||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withClientNotes is notes with what the request tells of the client
|
||||||
|
// added: its AS, country and reputation, the request at now with status,
|
||||||
|
// and how many requests netblock, the client's, has sent.
|
||||||
|
func (rq *request) withClientNotes(
|
||||||
|
notes bans.Notes,
|
||||||
|
now time.Time,
|
||||||
|
status int,
|
||||||
|
netblock netip.Prefix,
|
||||||
|
) bans.Notes {
|
||||||
|
notes.ASN = rq.line.ASN
|
||||||
|
notes.ASName = rq.line.ASName
|
||||||
|
notes.Country = rq.line.Country
|
||||||
|
notes.Reputation = rq.reputation
|
||||||
|
notes.Request = rq.noted(now, status)
|
||||||
|
notes.Requests = rq.netblockRequests(netblock)
|
||||||
|
|
||||||
|
return notes
|
||||||
|
}
|
||||||
|
|
||||||
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
||||||
// made at now would be sent. In observe mode the ban the request would
|
// made at now would be sent. In observe mode the ban the request would
|
||||||
// have made is worked out only then, at most once per
|
// have made is worked out only then, at most once per
|
||||||
|
|||||||
@@ -548,6 +548,10 @@ func (s *sender) requestWithHeader(
|
|||||||
// body is sent as it is after the headers, header holding its
|
// body is sent as it is after the headers, header holding its
|
||||||
// Content-Length or Transfer-Encoding. header may hold several lines,
|
// Content-Length or Transfer-Encoding. header may hold several lines,
|
||||||
// separated by "\r\n". It returns the whole answer.
|
// separated by "\r\n". It returns the whole answer.
|
||||||
|
//
|
||||||
|
// It waits for the answer as long as it takes, since a busy host can slow
|
||||||
|
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
|
||||||
|
// whose answer never comes.
|
||||||
func (s *sender) requestWithBody(
|
func (s *sender) requestWithBody(
|
||||||
method, from, path, header, body string, status int, action string,
|
method, from, path, header, body string, status int, action string,
|
||||||
) (logLine, answer) {
|
) (logLine, answer) {
|
||||||
@@ -566,11 +570,6 @@ func (s *sender) requestWithBody(
|
|||||||
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
||||||
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
||||||
|
|
||||||
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
s.t.Fatalf("set read deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got answer
|
var got answer
|
||||||
|
|
||||||
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
||||||
|
|||||||
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
unknown = percentage{
|
unknown = percentage{
|
||||||
cfg.UnknownCountryLimitPercent,
|
cfg.UnknownCountryLimitPercent,
|
||||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
config.UnknownCountryLimitPercentSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched := percentage{percent: whole}
|
fetched := percentage{percent: whole}
|
||||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||||
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
cfg.BlocklistAction == config.ReputationActionLimit {
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
"SWW_WAF_BLOCKLIST_ACTION",
|
config.BlocklistActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
config.DNSBLAbuseIPDBActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
const tagSetting = config.TagLimitPercentSetting
|
||||||
|
|
||||||
asnRequests := lowest(
|
asnRequests := lowest(
|
||||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||||
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
asnBytes = given(
|
asnBytes = given(
|
||||||
cfg.ASNBytesLimitPercent,
|
cfg.ASNBytesLimitPercent,
|
||||||
asn,
|
asn,
|
||||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
config.ASNBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
countryBytes = given(
|
countryBytes = given(
|
||||||
cfg.CountryBytesLimitPercent,
|
cfg.CountryBytesLimitPercent,
|
||||||
country,
|
country,
|
||||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
config.CountryBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -33,7 +34,8 @@ func (countries) check(
|
|||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionCountryDenied,
|
action: requestlog.ActionCountryDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
"country": country,
|
||||||
|
"setting": config.ExclusivelyAllowedCountriesSetting,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -104,12 +104,6 @@ func onReport() []heuristic {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
|
||||||
// it is called.
|
|
||||||
func (rq *request) runAfterAnswer(ctx context.Context) {
|
|
||||||
rq.runBanningHeuristics(ctx, afterAnswer())
|
|
||||||
}
|
|
||||||
|
|
||||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||||
// and returns the first verdict, or nil when none gives one. The log line's
|
// and returns the first verdict, or nil when none gives one. The log line's
|
||||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||||
|
|||||||
@@ -194,11 +194,11 @@ func (o *output) requestLine(t *testing.T) logLine {
|
|||||||
// endedLines waits for count requests that went through rproxy, or to waf's
|
// endedLines waits for count requests that went through rproxy, or to waf's
|
||||||
// own endpoints, to have ended, and returns the last log line of each, in the
|
// own endpoints, to have ended, and returns the last log line of each, in the
|
||||||
// order waf took them. A question's last line is the one its report writes.
|
// order waf took them. A question's last line is the one its report writes.
|
||||||
|
// It waits as long as it takes, as requestWithBody does.
|
||||||
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
deadline := time.Now().Add(waitLimit)
|
for {
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
found := o.lastLines(t)
|
found := o.lastLines(t)
|
||||||
|
|
||||||
ended := len(found) >= count
|
ended := len(found) >= count
|
||||||
@@ -214,10 +214,6 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
|
|||||||
|
|
||||||
time.Sleep(pollInterval)
|
time.Sleep(pollInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastLines returns the last request log line written so far of each request,
|
// lastLines returns the last request log line written so far of each request,
|
||||||
|
|||||||
@@ -345,6 +345,16 @@ func timing(start, end time.Time) *float64 {
|
|||||||
return new(requestlog.Milliseconds(end.Sub(start)))
|
return new(requestlog.Milliseconds(end.Sub(start)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
|
||||||
|
// It then counts the request for the anomaly thresholds and adds it to its
|
||||||
|
// client's history. list runs first, so that a refusal limit it finds broken
|
||||||
|
// is in that history. ctx is the request's own context.
|
||||||
|
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
|
||||||
|
rq.runBanningHeuristics(ctx, list)
|
||||||
|
rq.countAnomalies()
|
||||||
|
rq.addToHistory()
|
||||||
|
}
|
||||||
|
|
||||||
// addToHistory adds the request, which has ended, to its client's
|
// addToHistory adds the request, which has ended, to its client's
|
||||||
// history, and counts its offences in the metrics, and then the lookup's
|
// history, and counts its offences in the metrics, and then the lookup's
|
||||||
// answer about the client, as answerAtTheEnd gives it, to that history and
|
// answer about the client, as answerAtTheEnd gives it, to that history and
|
||||||
|
|||||||
+14
-21
@@ -177,11 +177,13 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
peer.Unmap(),
|
peer.Unmap(),
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
// The last deferred runs first, so that what is held has the finished log
|
// These run even when a check panics. finish runs first, so that what is
|
||||||
// line. Go's server sends the answer only once this function has
|
// held has the finished log line. Go's server sends the answer only once
|
||||||
// returned, so the report cannot come before it is held.
|
// this function has returned, so the report cannot come before it is held.
|
||||||
defer rq.holdForReport()
|
defer func() {
|
||||||
defer rq.finish()
|
rq.finish()
|
||||||
|
rq.holdForReport()
|
||||||
|
}()
|
||||||
|
|
||||||
rq.site = r.Host
|
rq.site = r.Host
|
||||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||||
@@ -190,26 +192,16 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
||||||
|
|
||||||
if refused == nil && !underOwnPrefix {
|
if refused == nil && !underOwnPrefix {
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
|
||||||
defer rq.runAfterAnswer(r.Context())
|
|
||||||
|
|
||||||
rq.allow()
|
rq.allow()
|
||||||
|
// In observe mode, what enforce mode would have refused counts here.
|
||||||
|
rq.runBanningHeuristics(r.Context(), afterAnswer())
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
|
||||||
// a broken refusal limit is in the client's history.
|
|
||||||
defer rq.addToHistory()
|
|
||||||
defer rq.countAnomalies()
|
|
||||||
defer rq.runAfterAnswer(r.Context())
|
|
||||||
|
|
||||||
if refused != nil {
|
if refused != nil {
|
||||||
rq.deny(*refused)
|
rq.deny(*refused)
|
||||||
|
} else {
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The app never sees this prefix. waf's own endpoints are at its
|
// The app never sees this prefix. waf's own endpoints are at its
|
||||||
// own address, so this answers as a path with no endpoint does.
|
// own address, so this answers as a path with no endpoint does.
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
@@ -221,6 +213,9 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rq.banAndCount(r.Context(), afterAnswer())
|
||||||
|
}
|
||||||
|
|
||||||
// readQuestion returns the visitor's address and the body a question carries,
|
// readQuestion returns the visitor's address and the body a question carries,
|
||||||
// and whether waf takes the question. It answers a question it does not
|
// and whether waf takes the question. It answers a question it does not
|
||||||
// take with 400 and no disposition. rproxy passes that 400 to the visitor,
|
// take with 400 and no disposition. rproxy passes that 400 to the visitor,
|
||||||
@@ -491,9 +486,7 @@ func (h *handler) finishWith(
|
|||||||
if rq.allowed {
|
if rq.allowed {
|
||||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||||
|
|
||||||
rq.runBanningHeuristics(ctx, onReport())
|
rq.banAndCount(ctx, onReport())
|
||||||
rq.countAnomalies()
|
|
||||||
rq.addToHistory()
|
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(
|
rq.h.metrics.RequestReported(
|
||||||
&rq.line,
|
&rq.line,
|
||||||
|
|||||||
Reference in new issue
Block a user