Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c85c1fdddb |
No files matched your search
@@ -906,11 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`: the state and cache files.
|
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
||||||
- `internal/requestlog`: the request log.
|
`internal/metrics`: the state and cache files, the log lines, the syslog
|
||||||
- `internal/processlog`: the process's own messages.
|
sender and the metrics.
|
||||||
- `internal/remotelog`: the syslog sender.
|
|
||||||
- `internal/metrics`: the metrics.
|
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -104,10 +104,8 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
||||||
- `internal/processlog`: the process lines on stdout, which hold the
|
sending them to the syslog server.
|
||||||
process's own messages.
|
|
||||||
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ require (
|
|||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
golang.org/x/net v0.60.0
|
|
||||||
golang.org/x/sys v0.48.0
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -51,6 +50,7 @@ require (
|
|||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||||
|
golang.org/x/net v0.60.0 // indirect
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
|
|||||||
@@ -54,11 +54,7 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
|
|||||||
now := midnight()
|
now := midnight()
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
scores := map[netip.Prefix]int64{netblock: 100}
|
||||||
|
|
||||||
limitBan, _ := ledger.Ban(
|
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
||||||
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
||||||
@@ -123,7 +119,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
|
|||||||
expires := now.Add(30 * day)
|
expires := now.Add(30 * day)
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
||||||
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(limited, now, bans.Notes{})
|
||||||
|
|
||||||
// listed is off the blacklist when it is fetched after the ban's end.
|
// listed is off the blacklist when it is fetched after the ban's end.
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
||||||
@@ -148,7 +144,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
||||||
}
|
}
|
||||||
|
|
||||||
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(limited, now, bans.Notes{})
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
||||||
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
||||||
@@ -158,7 +154,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
|
|
||||||
// The ban for an attack takes the place of the one for a limit, and no
|
// The ban for an attack takes the place of the one for a limit, and no
|
||||||
// blacklist ban goes.
|
// blacklist ban goes.
|
||||||
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
|
ledger.BanForAttack(attacker, now, bans.Notes{})
|
||||||
|
|
||||||
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
||||||
for netblock := range scores {
|
for netblock := range scores {
|
||||||
|
|||||||
+23
-64
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
||||||
|
|
||||||
// A new ban drops the ban sww made, and only that one.
|
// A new ban drops the ban sww made, and only that one.
|
||||||
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(second, midnight(), bans.Notes{})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// And the next drops that one.
|
// And the next drops that one.
|
||||||
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(third, midnight(), bans.Notes{})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
|
|||||||
// twice, each time after its last ban has ended.
|
// twice, each time after its last ban has ended.
|
||||||
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
||||||
"probes for logins")
|
"probes for logins")
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
)
|
||||||
|
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||||
|
|
||||||
// A ban on another netblock fills the ledger, and the netblock's first
|
// A ban on another netblock fills the ledger, and the netblock's first
|
||||||
// ban for a limit, its earliest that is not an admin's, goes.
|
// ban for a limit, its earliest that is not an admin's, goes.
|
||||||
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(other, second.Start, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
||||||
|
|
||||||
// The netblock's next ban still counts it.
|
// The netblock's next ban still counts it.
|
||||||
third, _ := ledger.Ban(
|
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
||||||
netblock,
|
|
||||||
second.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
|
|||||||
// The netblock breaks a limit twice, the second time after its first
|
// The netblock breaks a limit twice, the second time after its first
|
||||||
// ban has ended, and the second ban's notes count the first. An admin
|
// ban has ended, and the second ban's notes count the first. An admin
|
||||||
// then keeps the first ban.
|
// then keeps the first ban.
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
kept := first
|
kept := first
|
||||||
kept.Cause = bans.CauseAdmin
|
kept.Cause = bans.CauseAdmin
|
||||||
ledger.LoadEdit([]bans.Ban{kept, second})
|
ledger.LoadEdit([]bans.Ban{kept, second})
|
||||||
|
|
||||||
// The netblock's next ban counts the kept ban once, as an admin's.
|
// The netblock's next ban counts the kept ban once, as an admin's.
|
||||||
third, _ := ledger.Ban(
|
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
||||||
netblock,
|
|
||||||
second.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
limit, _ := ledger.Ban(
|
limit, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{
|
|
||||||
Kind: "requests",
|
|
||||||
Limit: 1000,
|
|
||||||
Window: "minute",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
byteLimit, _ := ledger.Ban(
|
byteLimit, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.3/32"),
|
netip.MustParsePrefix("203.0.113.3/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{
|
|
||||||
Kind: "bytes",
|
|
||||||
Limit: 10 << 30,
|
|
||||||
Window: "hour",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
attack, _ := ledger.Ban(
|
attack, _ := ledger.BanForAttack(
|
||||||
netip.MustParsePrefix("203.0.113.2/32"),
|
netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{RuleID: "git-dir", Target: "path"},
|
||||||
Cause: bans.CauseAttack,
|
|
||||||
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
for _, tc := range []struct{ got, want string }{
|
for _, tc := range []struct{ got, want string }{
|
||||||
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
|
|||||||
t.Error("the lifted ban refuses")
|
t.Error("the lifted ban refuses")
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The next clear sign of attack bans for seven days, as a first does.
|
// The next clear sign of attack bans for seven days, as a first does.
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
if ban.Expires.Sub(ban.Start) != 7*day {
|
if ban.Expires.Sub(ban.Start) != 7*day {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban for an attack ends at %s, want seven days on",
|
"the next ban for an attack ends at %s, want seven days on",
|
||||||
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
made, _ := ledger.Ban(
|
made, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
atStart := bans.Ban{
|
atStart := bans.Ban{
|
||||||
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// An hour's ban for a broken limit.
|
// An hour's ban for a broken limit.
|
||||||
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A minute later an admin bans the netblock for good, named by an
|
// A minute later an admin bans the netblock for good, named by an
|
||||||
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// The temporary ban is made first, so that its netblock is looked at first.
|
// The temporary ban is made first, so that its netblock is looked at first.
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
ledger.BanForAdmin(
|
ledger.BanForAdmin(
|
||||||
netip.MustParsePrefix("203.0.113.0/24"),
|
netip.MustParsePrefix("203.0.113.0/24"),
|
||||||
|
|||||||
+82
-54
@@ -300,50 +300,81 @@ func activeBan(bans []Ban, now time.Time) *Ban {
|
|||||||
return newest
|
return newest
|
||||||
}
|
}
|
||||||
|
|
||||||
// Grounds are why a request makes a ban: its cause, CauseLimit or
|
// BanForLimit bans netblock at now for a broken limit, with notes, and
|
||||||
// CauseAttack, and its notes.
|
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
|
||||||
type Grounds struct {
|
// made within LimitBanRepeatWindow after the netblock's ban that ended
|
||||||
Cause string
|
// last, other than one for a clear sign of attack or a lifted one, lasts
|
||||||
Notes Notes
|
// repeatFactor times as long as that one. A ban that would be longer
|
||||||
|
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
|
||||||
|
// active, as when two of its requests break a limit at once, that ban is
|
||||||
|
// returned with false, and no other is made. The ledger fills in the
|
||||||
|
// notes' Refused and EarlierBans itself, and gives the ban the reason
|
||||||
|
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
|
||||||
|
// as "requests per minute over the limit of 1000".
|
||||||
|
func (l *Ledger) BanForLimit(
|
||||||
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
|
) (Ban, bool) {
|
||||||
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
CauseLimit,
|
||||||
|
limitReason(notes),
|
||||||
|
notes,
|
||||||
|
true,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ban bans netblock at now on grounds, and returns the ban, and true. If
|
// WouldBanForLimit returns what BanForLimit would, without making the ban:
|
||||||
// a ban on netblock is still active, as when two of its requests break a
|
// what observe mode would have done.
|
||||||
// limit at once, that ban is returned with false, and no other is made.
|
func (l *Ledger) WouldBanForLimit(
|
||||||
// The ledger fills in the notes' Refused and EarlierBans itself.
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
//
|
|
||||||
// A ban for a broken limit:
|
|
||||||
// - lasts LimitBanDuration the first time;
|
|
||||||
// - lasts repeatFactor times as long as the netblock's ban that ended
|
|
||||||
// last, when made within LimitBanRepeatWindow after it, unless that
|
|
||||||
// ban was for a clear sign of attack or was lifted;
|
|
||||||
// - is permanent instead when it would be longer than
|
|
||||||
// LimitBanPermanentAbove;
|
|
||||||
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
|
|
||||||
// the notes, such as "requests per minute over the limit of 1000".
|
|
||||||
//
|
|
||||||
// A ban for a clear sign of attack:
|
|
||||||
// - lasts AttackBanDuration the first time;
|
|
||||||
// - is permanent once the netblock has had one that was not lifted;
|
|
||||||
// - has the reason "matched the rule <RuleID>", or "asked for the trap
|
|
||||||
// path <TrapPath>".
|
|
||||||
func (l *Ledger) Ban(
|
|
||||||
netblock netip.Prefix, now time.Time, grounds Grounds,
|
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(netblock, now, grounds, true)
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
CauseLimit,
|
||||||
|
limitReason(notes),
|
||||||
|
notes,
|
||||||
|
false,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBan returns what Ban would, without making the ban: what observe
|
// BanForAttack bans netblock at now for a clear sign of attack, with
|
||||||
// mode would have done.
|
// notes, and returns the ban, and whether it made it, as BanForLimit
|
||||||
func (l *Ledger) WouldBan(
|
// does. A first ban lasts AttackBanDuration; once the netblock has had
|
||||||
netblock netip.Prefix, now time.Time, grounds Grounds,
|
// one that was not lifted, the next is permanent. Its reason is "matched
|
||||||
|
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
|
||||||
|
func (l *Ledger) BanForAttack(
|
||||||
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(netblock, now, grounds, false)
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
CauseAttack,
|
||||||
|
attackReason(notes),
|
||||||
|
notes,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WouldBanForAttack returns what BanForAttack would, without making the
|
||||||
|
// ban: what observe mode would have done.
|
||||||
|
func (l *Ledger) WouldBanForAttack(
|
||||||
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
|
) (Ban, bool) {
|
||||||
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
CauseAttack,
|
||||||
|
attackReason(notes),
|
||||||
|
notes,
|
||||||
|
false,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
||||||
// or CauseAttack, made at now would be permanent, as Ban would make it. It
|
// or CauseAttack, made at now would be permanent, as BanForLimit or
|
||||||
// works out nothing else of the ban.
|
// BanForAttack would make it. It works out nothing else of the ban.
|
||||||
func (l *Ledger) WouldBePermanent(
|
func (l *Ledger) WouldBePermanent(
|
||||||
netblock netip.Prefix, now time.Time, cause string,
|
netblock netip.Prefix, now time.Time, cause string,
|
||||||
) bool {
|
) bool {
|
||||||
@@ -380,9 +411,10 @@ func attackReason(notes Notes) string {
|
|||||||
|
|
||||||
// BanForAdmin bans netblock at now for an admin, with reason, until
|
// BanForAdmin bans netblock at now for an admin, with reason, until
|
||||||
// expires, or for good when expires is zero, and returns the ban, whose
|
// expires, or for good when expires is zero, and returns the ban, whose
|
||||||
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
|
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
|
||||||
// on netblock is active, since the admin asked for this one. The ledger
|
// ban even while another on netblock is active, since the admin asked
|
||||||
// fills in the notes' EarlierBans, and counts the ban among those made.
|
// for this one. The ledger fills in the notes' EarlierBans, and counts
|
||||||
|
// the ban among those made.
|
||||||
func (l *Ledger) BanForAdmin(
|
func (l *Ledger) BanForAdmin(
|
||||||
netblock netip.Prefix, now, expires time.Time, reason string,
|
netblock netip.Prefix, now, expires time.Time, reason string,
|
||||||
) Ban {
|
) Ban {
|
||||||
@@ -727,13 +759,15 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
|
|||||||
return forgot
|
return forgot
|
||||||
}
|
}
|
||||||
|
|
||||||
// ban bans netblock at now on grounds, as Ban describes, and returns the
|
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
|
||||||
// ban, and whether it made it. Unless keep is true, the ban is not made,
|
// reason and notes, as BanForLimit and BanForAttack describe, and returns
|
||||||
// only returned: it is the ban that would have been made.
|
// the ban, and whether it made it. Unless keep is true, the ban is not
|
||||||
|
// made, only returned: it is the ban that would have been made.
|
||||||
func (l *Ledger) ban(
|
func (l *Ledger) ban(
|
||||||
netblock netip.Prefix,
|
netblock netip.Prefix,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
grounds Grounds,
|
cause, reason string,
|
||||||
|
notes Notes,
|
||||||
keep bool,
|
keep bool,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
@@ -742,8 +776,6 @@ func (l *Ledger) ban(
|
|||||||
// held are the netblock's bans, none of them active.
|
// held are the netblock's bans, none of them active.
|
||||||
var held []Ban
|
var held []Ban
|
||||||
|
|
||||||
notes := grounds.Notes
|
|
||||||
|
|
||||||
bans, found := l.netblocks.Get(netblock)
|
bans, found := l.netblocks.Get(netblock)
|
||||||
if found {
|
if found {
|
||||||
active := activeBan(*bans, now)
|
active := activeBan(*bans, now)
|
||||||
@@ -759,19 +791,15 @@ func (l *Ledger) ban(
|
|||||||
ban := Ban{
|
ban := Ban{
|
||||||
Netblock: netblock,
|
Netblock: netblock,
|
||||||
Start: now,
|
Start: now,
|
||||||
Cause: grounds.Cause,
|
Cause: cause,
|
||||||
|
Reason: reason,
|
||||||
Notes: notes,
|
Notes: notes,
|
||||||
}
|
}
|
||||||
|
|
||||||
switch grounds.Cause {
|
if cause == CauseAttack {
|
||||||
case CauseLimit:
|
|
||||||
ban.Reason = limitReason(notes)
|
|
||||||
ban.Expires = l.limitExpiry(held, now)
|
|
||||||
case CauseAttack:
|
|
||||||
ban.Reason = attackReason(notes)
|
|
||||||
ban.Expires = l.attackExpiry(held, now)
|
ban.Expires = l.attackExpiry(held, now)
|
||||||
default:
|
} else {
|
||||||
panic("bans: a request's ban has the cause " + grounds.Cause)
|
ban.Expires = l.limitExpiry(held, now)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !keep {
|
if !keep {
|
||||||
@@ -779,7 +807,7 @@ func (l *Ledger) ban(
|
|||||||
}
|
}
|
||||||
|
|
||||||
l.add(ban)
|
l.add(ban)
|
||||||
l.made[grounds.Cause]++
|
l.made[cause]++
|
||||||
l.markChanged()
|
l.markChanged()
|
||||||
|
|
||||||
return ban, true
|
return ban, true
|
||||||
|
|||||||
+51
-121
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
||||||
// 81 hours.
|
// 81 hours.
|
||||||
for i, hours := range []int{1, 3, 9, 27, 81} {
|
for i, hours := range []int{1, 3, 9, 27, 81} {
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
length := time.Duration(hours) * time.Hour
|
length := time.Duration(hours) * time.Hour
|
||||||
if !ban.Expires.Equal(now.Add(length)) ||
|
if !ban.Expires.Equal(now.Add(length)) ||
|
||||||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
|
|
||||||
// The sixth would last 243 hours, more than seven days: it is
|
// The sixth would last 243 hours, more than seven days: it is
|
||||||
// permanent, and never ends.
|
// permanent, and never ends.
|
||||||
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
second, _ := ledger.Ban(
|
second, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires.Add(tc.gap),
|
first.Expires.Add(tc.gap),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != tc.want ||
|
if second.Expires.Sub(second.Start) != tc.want ||
|
||||||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
|
|||||||
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
||||||
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
for i := range 14 {
|
for i := range 14 {
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if !ban.Expires.After(ban.Start) {
|
if !ban.Expires.After(ban.Start) {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"ban %d starts at %s and ends at %s",
|
"ban %d starts at %s and ends at %s",
|
||||||
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, made := ledger.Ban(
|
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if !made {
|
if !made {
|
||||||
t.Error("the first ban was not made")
|
t.Error("the first ban was not made")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made := ledger.Ban(
|
again, made := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
if made || !reflect.DeepEqual(again, first) ||
|
if made || !reflect.DeepEqual(again, first) ||
|
||||||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made = ledger.Ban(
|
again, made = ledger.BanForAttack(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if made || !reflect.DeepEqual(again, first) {
|
if made || !reflect.DeepEqual(again, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
bans.Notes{Requests: 5},
|
||||||
)
|
)
|
||||||
|
|
||||||
for range 3 {
|
for range 3 {
|
||||||
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
bans.Notes{Requests: 5},
|
||||||
)
|
)
|
||||||
|
|
||||||
got, banned, _ := ledger.Find(
|
got, banned, _ := ledger.Find(
|
||||||
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
|
|||||||
d := netip.MustParsePrefix("2001:db8::/64")
|
d := netip.MustParsePrefix("2001:db8::/64")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
|
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
|
||||||
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(b, now, bans.Notes{})
|
||||||
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(c, now, bans.Notes{})
|
||||||
|
|
||||||
// A request from a makes b the netblock seen longest ago, and its ban
|
// A request from a makes b the netblock seen longest ago, and its ban
|
||||||
// goes to make room for d's.
|
// goes to make room for d's.
|
||||||
ledger.Check(a.Addr(), now)
|
ledger.Check(a.Addr(), now)
|
||||||
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(d, now, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
||||||
|
|
||||||
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
||||||
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(a, first.Expires, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
||||||
|
|
||||||
// With d seen since, a is seen longest ago, and its earlier ban goes
|
// With d seen since, a is seen longest ago, and its earlier ban goes
|
||||||
// first.
|
// first.
|
||||||
ledger.Check(d.Addr(), first.Expires)
|
ledger.Check(d.Addr(), first.Expires)
|
||||||
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(b, first.Expires, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
||||||
|
|
||||||
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
||||||
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
|
|||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
held := ledger.Bans(netblock)
|
held := ledger.Bans(netblock)
|
||||||
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
||||||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
|
||||||
)
|
|
||||||
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
||||||
ban.Cause != bans.CauseAttack ||
|
ban.Cause != bans.CauseAttack ||
|
||||||
ban.Notes.RuleID != "env-file" ||
|
ban.Notes.RuleID != "env-file" ||
|
||||||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// A ban for a broken limit before does not count.
|
// A ban for a broken limit before does not count.
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != 7*day {
|
if second.Expires.Sub(second.Start) != 7*day {
|
||||||
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
||||||
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
|
|
||||||
// Its notes show the earlier ban for an attack that makes it permanent,
|
// Its notes show the earlier ban for an attack that makes it permanent,
|
||||||
// beside the one for a limit.
|
// beside the one for a limit.
|
||||||
third, _ := ledger.Ban(
|
third, _ := ledger.BanForAttack(
|
||||||
netblock,
|
netblock,
|
||||||
second.Expires.Add(30*day),
|
second.Expires.Add(30*day),
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if !third.Permanent() ||
|
if !third.Permanent() ||
|
||||||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
||||||
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// While the first ban lasts, none would be made.
|
// While the first ban lasts, none would be made.
|
||||||
during, would := ledger.WouldBan(
|
during, would := ledger.WouldBanForAttack(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if would || !reflect.DeepEqual(during, first) {
|
if would || !reflect.DeepEqual(during, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
// As it ends, a clear sign of attack would ban for seven days, and a
|
// As it ends, a clear sign of attack would ban for seven days, and a
|
||||||
// limit broken again for three hours, but neither is made.
|
// limit broken again for three hours, but neither is made.
|
||||||
limitGrounds := bans.Grounds{
|
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
|
||||||
Cause: bans.CauseLimit,
|
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
|
||||||
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
|
bans.Notes{RuleID: "git-dir"})
|
||||||
}
|
limit, wouldLimit := ledger.WouldBanForLimit(
|
||||||
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
|
|
||||||
bans.Grounds{
|
|
||||||
Cause: bans.CauseAttack,
|
|
||||||
Notes: bans.Notes{RuleID: "git-dir"},
|
|
||||||
})
|
|
||||||
limit, wouldLimit := ledger.WouldBan(
|
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires,
|
first.Expires,
|
||||||
limitGrounds,
|
limitNotes,
|
||||||
)
|
)
|
||||||
|
|
||||||
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
||||||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// The ban made is the one that would have been.
|
// The ban made is the one that would have been.
|
||||||
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
|
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
|
||||||
if !reflect.DeepEqual(made, limit) {
|
if !reflect.DeepEqual(made, limit) {
|
||||||
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
||||||
}
|
}
|
||||||
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Once a first ban for an attack has ended, the next would be permanent.
|
// Once a first ban for an attack has ended, the next would be permanent.
|
||||||
attack, _ := ledger.Ban(
|
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
||||||
t.Error("a second ban for an attack would not be permanent")
|
t.Error("a second ban for an attack would not be permanent")
|
||||||
}
|
}
|
||||||
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
|
|||||||
|
|
||||||
// Three times the seven days would be permanent; a limit broken as the
|
// Three times the seven days would be permanent; a limit broken as the
|
||||||
// ban for an attack ends bans for an hour, as a first broken limit does.
|
// ban for an attack ends bans for an hour, as a first broken limit does.
|
||||||
attack, _ := ledger.Ban(
|
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
limit, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
attack.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
||||||
limit.Cause != bans.CauseLimit {
|
limit.Cause != bans.CauseLimit {
|
||||||
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
|
|||||||
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{Request: request},
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{Request: request},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cut := long[:256]
|
cut := long[:256]
|
||||||
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
|
|||||||
|
|
||||||
// A ban made with the client's lookup, one made before it came, after
|
// A ban made with the client's lookup, one made before it came, after
|
||||||
// the first ended, and one on another netblock.
|
// the first ended, and one on another netblock.
|
||||||
ledger.Ban(netblock, midnight(), bans.Grounds{
|
ledger.BanForLimit(netblock, midnight(), bans.Notes{
|
||||||
Cause: bans.CauseLimit,
|
ASN: "AS64497", ASName: "Other Net", Country: "FR",
|
||||||
Notes: bans.Notes{
|
|
||||||
ASN: "AS64497",
|
|
||||||
ASName: "Other Net",
|
|
||||||
Country: "FR",
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
ledger.Ban(
|
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
|
||||||
netblock,
|
ledger.BanForLimit(other, midnight(), bans.Notes{})
|
||||||
midnight().Add(time.Hour),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
|
|
||||||
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
||||||
|
|
||||||
|
|||||||
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
|
|||||||
|
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A limit broken during the ban makes no other, and a refusal changes
|
// A limit broken during the ban makes no other, and a refusal changes
|
||||||
// only the counts in the notes, which wait for the interval's write.
|
// only the counts in the notes, which wait for the interval's write.
|
||||||
ledger.Ban(
|
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight().Add(time.Minute),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// Two bans before the value is read leave one.
|
// Two bans before the value is read leave one.
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.11/32"),
|
netip.MustParsePrefix("203.0.113.11/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
|
|||||||
high := netip.MustParsePrefix("203.0.113.10/32")
|
high := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
low := netip.MustParsePrefix("203.0.113.9/32")
|
low := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
|
||||||
v6,
|
ledger.BanForLimit(high, midnight(), bans.Notes{})
|
||||||
midnight(),
|
ledger.BanForLimit(low, midnight(), bans.Notes{})
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
|
||||||
)
|
|
||||||
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
|
|
||||||
snapshot := ledger.Snapshot()
|
snapshot := ledger.Snapshot()
|
||||||
|
|
||||||
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
|
|
||||||
before := bans.New(defaultParams())
|
before := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
|
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{Limit: 1},
|
|
||||||
})
|
|
||||||
|
|
||||||
// Loaded into a new ledger, as across a restart, the ban still refuses
|
// Loaded into a new ledger, as across a restart, the ban still refuses
|
||||||
// while it lasts, and once it has ended a broken limit bans for three
|
// while it lasts, and once it has ended a broken limit bans for three
|
||||||
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
t.Error("the loaded ban does not refuse")
|
t.Error("the loaded ban does not refuse")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, _ := after.Ban(
|
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
|
||||||
netblock,
|
|
||||||
ban.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
||||||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
|
|||||||
Start: midnight(),
|
Start: midnight(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
for client, want := range map[string]bool{
|
for client, want := range map[string]bool{
|
||||||
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A limit broken now makes no shorter ban over the permanent one.
|
// A limit broken now makes no shorter ban over the permanent one.
|
||||||
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"a broken limit returned %+v and left the netblock %d bans, "+
|
"a broken limit returned %+v and left the netblock %d bans, "+
|
||||||
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
|
|||||||
// Once both have ended, a limit broken within the repeat window bans
|
// Once both have ended, a limit broken within the repeat window bans
|
||||||
// for three times the 9 hours, and the notes count the two bans
|
// for three times the 9 hours, and the notes count the two bans
|
||||||
// before the 9-hour one and it, for a limit, and the admin's.
|
// before the 9-hour one and it, for a limit, and the admin's.
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
nineHours.Expires.Add(time.Hour),
|
nineHours.Expires.Add(time.Hour),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
||||||
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The ledger holds one ban, so it makes two more without dropping any.
|
// The ledger holds one ban, so it makes two more without dropping any.
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
second, _ := ledger.Ban(
|
second, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("198.51.100.8/32"),
|
netip.MustParsePrefix("198.51.100.8/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
want := []bans.Ban{first, second, kept}
|
want := []bans.Ban{first, second, kept}
|
||||||
|
|||||||
@@ -19,8 +19,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// off is the value that switches a timeout, a size limit or a rate limit
|
// off is the value that switches a timeout, a size limit or a rate limit
|
||||||
@@ -42,7 +40,9 @@ var (
|
|||||||
errNotPositive = errors.New("must be more than zero, or off")
|
errNotPositive = errors.New("must be more than zero, or off")
|
||||||
errNotMilliseconds = errors.New(
|
errNotMilliseconds = errors.New(
|
||||||
"is not a whole number of milliseconds above zero, such as 2000")
|
"is not a whole number of milliseconds above zero, such as 2000")
|
||||||
errEmptyItem = errors.New("has an empty item in its list")
|
errEmptyItem = errors.New("has an empty item in its list")
|
||||||
|
errNotNetblock = errors.New(
|
||||||
|
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||||
errNotListenAddr = errors.New(
|
errNotListenAddr = errors.New(
|
||||||
"is not an address to listen on, such as :8080")
|
"is not an address to listen on, such as :8080")
|
||||||
errNotCommand = errors.New("is not rproxy or waf")
|
errNotCommand = errors.New("is not rproxy or waf")
|
||||||
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
netblocks := make([]netip.Prefix, 0, len(items))
|
netblocks := make([]netip.Prefix, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
netblock, err := reputation.ParseNetblock(item)
|
netblock, err := ParseNetblock(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -264,6 +264,27 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
return netblocks, nil
|
return netblocks, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||||
|
// address is a netblock of that address alone, a /32 or a /128. A
|
||||||
|
// blocklist's lines are read with it too.
|
||||||
|
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||||
|
if strings.Contains(value, "/") {
|
||||||
|
netblock, err := netip.ParsePrefix(value)
|
||||||
|
if err != nil {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netblock.Masked(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(value)
|
||||||
|
if err != nil || addr.Zone() != "" {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseListenAddr checks an address to listen on: an optional host and a
|
// parseListenAddr checks an address to listen on: an optional host and a
|
||||||
// port number.
|
// port number.
|
||||||
func parseListenAddr(value string) (string, error) {
|
func parseListenAddr(value string) (string, error) {
|
||||||
|
|||||||
+152
-169
@@ -20,26 +20,11 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"golang.org/x/net/http/httpguts"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The names of the settings that waf writes into log lines and ban notes,
|
|
||||||
// as the setting that refused a request or lowered a client's limits.
|
|
||||||
const (
|
|
||||||
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
|
||||||
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
|
||||||
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
|
||||||
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
|
||||||
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
|
||||||
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
|
||||||
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
|
||||||
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||||
@@ -90,7 +75,7 @@ type WAF struct {
|
|||||||
BytesLimitPerMinute int64
|
BytesLimitPerMinute int64
|
||||||
BytesLimitPerHour int64
|
BytesLimitPerHour int64
|
||||||
BytesLimitPerDay int64
|
BytesLimitPerDay int64
|
||||||
BytesCounted BytesCounted
|
BytesCounted string
|
||||||
// LookupSource is where each client's AS number and country are looked
|
// LookupSource is where each client's AS number and country are looked
|
||||||
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
||||||
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
||||||
@@ -101,7 +86,7 @@ type WAF struct {
|
|||||||
// AddLookupHeaders is true when the app is passed the client's AS number
|
// AddLookupHeaders is true when the app is passed the client's AS number
|
||||||
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
||||||
// LookupSource is off.
|
// LookupSource is off.
|
||||||
LookupSource LookupSource
|
LookupSource string
|
||||||
LookupDBPath string
|
LookupDBPath string
|
||||||
ASNAndCountryLookupTimeout time.Duration
|
ASNAndCountryLookupTimeout time.Duration
|
||||||
AddLookupHeaders bool
|
AddLookupHeaders bool
|
||||||
@@ -115,7 +100,7 @@ type WAF struct {
|
|||||||
// with a tag that percentage of every rate limit and byte limit, from 1
|
// with a tag that percentage of every rate limit and byte limit, from 1
|
||||||
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
||||||
// lower case, a country code in capitals.
|
// lower case, a country code in capitals.
|
||||||
TagLists []reputation.TagList
|
TagLists []TagList
|
||||||
TagDeny []string
|
TagDeny []string
|
||||||
TagLimitPercent map[string]int64
|
TagLimitPercent map[string]int64
|
||||||
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
||||||
@@ -146,7 +131,7 @@ type WAF struct {
|
|||||||
// BlocklistLimitPercent is the percentage of every limit it gets.
|
// BlocklistLimitPercent is the percentage of every limit it gets.
|
||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
BlocklistRefresh time.Duration
|
BlocklistRefresh time.Duration
|
||||||
BlocklistAction ReputationAction
|
BlocklistAction string
|
||||||
BlocklistLimitPercent int64
|
BlocklistLimitPercent int64
|
||||||
// DNSBLZones are the DNSBL zones clients are asked about
|
// DNSBLZones are the DNSBL zones clients are asked about
|
||||||
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
||||||
@@ -169,7 +154,7 @@ type WAF struct {
|
|||||||
AbuseIPDBKey string
|
AbuseIPDBKey string
|
||||||
AbuseIPDBMinScore int64
|
AbuseIPDBMinScore int64
|
||||||
AbuseIPDBDailyBudget int
|
AbuseIPDBDailyBudget int
|
||||||
DNSBLAbuseIPDBAction ReputationAction
|
DNSBLAbuseIPDBAction string
|
||||||
DNSBLAbuseIPDBLimitPercent int64
|
DNSBLAbuseIPDBLimitPercent int64
|
||||||
IPReputationCacheTTL time.Duration
|
IPReputationCacheTTL time.Duration
|
||||||
DNSBLAbuseIPDBTimeout time.Duration
|
DNSBLAbuseIPDBTimeout time.Duration
|
||||||
@@ -252,7 +237,7 @@ type WAF struct {
|
|||||||
// reads none.
|
// reads none.
|
||||||
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
||||||
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
||||||
CoreRuleSetMode CoreRuleSetMode
|
CoreRuleSetMode string
|
||||||
CoreRuleSetParanoiaLevel int
|
CoreRuleSetParanoiaLevel int
|
||||||
CoreRuleSetScoreThreshold int
|
CoreRuleSetScoreThreshold int
|
||||||
CoreRuleSetDisabledRules []int
|
CoreRuleSetDisabledRules []int
|
||||||
@@ -331,66 +316,20 @@ type WAF struct {
|
|||||||
settings []slog.Attr
|
settings []slog.Attr
|
||||||
}
|
}
|
||||||
|
|
||||||
// BytesCounted is which body bytes count toward the byte limits and the
|
|
||||||
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
|
|
||||||
type BytesCounted string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_BYTES_COUNTED.
|
|
||||||
const (
|
|
||||||
// BytesCountedResponse counts the response's body bytes.
|
|
||||||
BytesCountedResponse BytesCounted = "response"
|
|
||||||
// BytesCountedRequest counts the request's body bytes.
|
|
||||||
BytesCountedRequest BytesCounted = "request"
|
|
||||||
// BytesCountedBoth counts the body bytes of both.
|
|
||||||
BytesCountedBoth BytesCounted = "both"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LookupSource is where each client's AS number and country are looked up,
|
|
||||||
// a value of SWW_WAF_LOOKUP_SOURCE.
|
|
||||||
type LookupSource string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_LOOKUP_SOURCE.
|
|
||||||
const (
|
|
||||||
// LookupSourceGeoJS asks GeoJS.
|
|
||||||
LookupSourceGeoJS LookupSource = "geojs"
|
|
||||||
// LookupSourceFile looks clients up in the lookup database, the file
|
|
||||||
// SWW_WAF_LOOKUP_DB_PATH names.
|
|
||||||
LookupSourceFile LookupSource = "file"
|
|
||||||
// LookupSourceOff looks no client up.
|
|
||||||
LookupSourceOff LookupSource = "off"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ReputationAction is what is done with a client a reputation source
|
|
||||||
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
|
|
||||||
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
|
||||||
type ReputationAction string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
|
||||||
const (
|
|
||||||
// ReputationActionDeny refuses the client's requests.
|
|
||||||
ReputationActionDeny ReputationAction = "deny"
|
|
||||||
// ReputationActionLimit gives the client a percentage of every limit,
|
|
||||||
// written limit:<percent> in the setting.
|
|
||||||
ReputationActionLimit ReputationAction = "limit"
|
|
||||||
// ReputationActionLog only notes the hit in the log line and raises an
|
|
||||||
// alert.
|
|
||||||
ReputationActionLog ReputationAction = "log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CoreRuleSetMode is what the Core Rule Set does, a value of
|
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE.
|
|
||||||
type CoreRuleSetMode string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
||||||
const (
|
const (
|
||||||
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
||||||
CoreRuleSetModeOff CoreRuleSetMode = "off"
|
CoreRuleSetModeOff = off
|
||||||
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
||||||
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
|
CoreRuleSetModeDetect = "detect"
|
||||||
// CoreRuleSetModeBlock refuses a match with 403.
|
// CoreRuleSetModeBlock refuses a match with 403.
|
||||||
CoreRuleSetModeBlock CoreRuleSetMode = "block"
|
CoreRuleSetModeBlock = "block"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
|
||||||
|
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
|
||||||
|
const fileSource = "file"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
day = 24 * time.Hour
|
day = 24 * time.Hour
|
||||||
ipv4Bits = 32
|
ipv4Bits = 32
|
||||||
@@ -512,8 +451,12 @@ var (
|
|||||||
"is not a whole number above zero, such as 60, or off",
|
"is not a whole number above zero, such as 60, or off",
|
||||||
)
|
)
|
||||||
errNotUTF8 = errors.New("is not valid UTF-8")
|
errNotUTF8 = errors.New("is not valid UTF-8")
|
||||||
|
errNotASN = errors.New("is not an AS number such as AS64496")
|
||||||
errNotPercentItem = errors.New(
|
errNotPercentItem = errors.New(
|
||||||
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
||||||
|
errNotPercent = errors.New(
|
||||||
|
"is not a percentage, a whole number from 0 to 100",
|
||||||
|
)
|
||||||
errNotAbuseIPDBThreshold = errors.New(
|
errNotAbuseIPDBThreshold = errors.New(
|
||||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
||||||
"blacklist can be cut at",
|
"blacklist can be cut at",
|
||||||
@@ -615,13 +558,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
),
|
),
|
||||||
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
||||||
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
||||||
BytesCounted: env.bytesCounted(
|
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
|
||||||
"SWW_WAF_BYTES_COUNTED",
|
|
||||||
BytesCountedBoth,
|
|
||||||
),
|
|
||||||
LookupSource: env.lookupSource(
|
LookupSource: env.lookupSource(
|
||||||
"SWW_WAF_LOOKUP_SOURCE",
|
"SWW_WAF_LOOKUP_SOURCE",
|
||||||
LookupSourceGeoJS,
|
"geojs",
|
||||||
),
|
),
|
||||||
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
||||||
ASNAndCountryLookupTimeout: env.milliseconds(
|
ASNAndCountryLookupTimeout: env.milliseconds(
|
||||||
@@ -629,24 +569,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"800",
|
"800",
|
||||||
),
|
),
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
ExclusivelyAllowedCountriesSetting, ""),
|
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||||
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
||||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||||
ASNBytesLimitPercent: env.percents(
|
ASNBytesLimitPercent: env.percents(
|
||||||
ASNBytesLimitPercentSetting,
|
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||||
reputation.ParseASN,
|
ParseASN,
|
||||||
),
|
),
|
||||||
CountryBytesLimitPercent: env.percents(
|
CountryBytesLimitPercent: env.percents(
|
||||||
CountryBytesLimitPercentSetting,
|
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||||
parseCountry,
|
parseCountry,
|
||||||
),
|
),
|
||||||
UnknownCountryLimitPercent: env.percent(
|
UnknownCountryLimitPercent: env.percent(
|
||||||
UnknownCountryLimitPercentSetting,
|
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||||
"100",
|
"100",
|
||||||
),
|
),
|
||||||
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
||||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||||
@@ -818,11 +758,11 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||||
// The app gets the headers by default, unless no client is looked up.
|
// The app gets the headers by default, unless no client is looked up.
|
||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != off))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
BlocklistActionSetting, string(ReputationActionDeny))
|
"SWW_WAF_BLOCKLIST_ACTION", "deny")
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
DNSBLAbuseIPDBActionSetting, "limit:25")
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||||
|
|
||||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||||
env.checkLookupDBPath(cfg)
|
env.checkLookupDBPath(cfg)
|
||||||
@@ -918,12 +858,9 @@ func (e *environment) count(name, defaultValue string) int64 {
|
|||||||
|
|
||||||
// bytesCounted reads the setting that is which body bytes count toward the
|
// bytesCounted reads the setting that is which body bytes count toward the
|
||||||
// byte limits and the byte anomaly thresholds: response, request or both.
|
// byte limits and the byte anomaly thresholds: response, request or both.
|
||||||
func (e *environment) bytesCounted(
|
func (e *environment) bytesCounted(name, defaultValue string) string {
|
||||||
name string, defaultValue BytesCounted,
|
value := e.value(name, defaultValue)
|
||||||
) BytesCounted {
|
if value != "response" && value != "request" && value != "both" {
|
||||||
value := BytesCounted(e.value(name, string(defaultValue)))
|
|
||||||
if value != BytesCountedResponse && value != BytesCountedRequest &&
|
|
||||||
value != BytesCountedBoth {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -968,10 +905,8 @@ func (e *environment) trapPaths(name string) []string {
|
|||||||
|
|
||||||
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
||||||
// detect or block.
|
// detect or block.
|
||||||
func (e *environment) coreRuleSetMode(
|
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
|
||||||
name string, defaultValue CoreRuleSetMode,
|
mode := e.value(name, defaultValue)
|
||||||
) CoreRuleSetMode {
|
|
||||||
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
|
|
||||||
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
||||||
mode != CoreRuleSetModeBlock {
|
mode != CoreRuleSetModeBlock {
|
||||||
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
||||||
@@ -1022,7 +957,7 @@ func (e *environment) percents(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
||||||
func (e *environment) tagLists(name string) []reputation.TagList {
|
func (e *environment) tagLists(name string) []TagList {
|
||||||
lists, err := parseTagLists(e.value(name, ""))
|
lists, err := parseTagLists(e.value(name, ""))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
@@ -1048,7 +983,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
|
|||||||
|
|
||||||
// percent reads a setting that is a percentage, from 0 to 100.
|
// percent reads a setting that is a percentage, from 0 to 100.
|
||||||
func (e *environment) percent(name, defaultValue string) int64 {
|
func (e *environment) percent(name, defaultValue string) int64 {
|
||||||
percent, err := reputation.ParsePercent(e.value(name, defaultValue))
|
percent, err := ParsePercent(e.value(name, defaultValue))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
return percent
|
return percent
|
||||||
@@ -1121,29 +1056,25 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
|
|||||||
// action reads a setting that is what is done with a client a blocklist
|
// action reads a setting that is what is done with a client a blocklist
|
||||||
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
||||||
// as limit and the percentage.
|
// as limit and the percentage.
|
||||||
func (e *environment) action(
|
func (e *environment) action(name, defaultValue string) (string, int64) {
|
||||||
name, defaultValue string,
|
value := e.value(name, defaultValue)
|
||||||
) (ReputationAction, int64) {
|
if value == "deny" || value == "log" {
|
||||||
value := ReputationAction(e.value(name, defaultValue))
|
|
||||||
if value == ReputationActionDeny || value == ReputationActionLog {
|
|
||||||
return value, 0
|
return value, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
percentText, isLimit := strings.CutPrefix(
|
percentText, isLimit := strings.CutPrefix(value, "limit:")
|
||||||
string(value), string(ReputationActionLimit)+":",
|
|
||||||
)
|
|
||||||
|
|
||||||
percent, err := reputation.ParsePercent(percentText)
|
percent, err := ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
|
|
||||||
return ReputationActionLimit, percent
|
return "limit", percent
|
||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
// default. The log shows each zone with its key masked, as
|
// default. The log shows each zone with its key masked, as MaskZoneKey
|
||||||
// reputation.MaskZoneKey masks it.
|
// masks it.
|
||||||
func (e *environment) zones(name string) []string {
|
func (e *environment) zones(name string) []string {
|
||||||
value, _ := e.lookup(name)
|
value, _ := e.lookup(name)
|
||||||
zones, err := parseZones(value)
|
zones, err := parseZones(value)
|
||||||
@@ -1151,7 +1082,7 @@ func (e *environment) zones(name string) []string {
|
|||||||
|
|
||||||
logged := make([]string, len(zones))
|
logged := make([]string, len(zones))
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
logged[i] = reputation.MaskZoneKey(zone)
|
logged[i] = MaskZoneKey(zone)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.settings = append(
|
e.settings = append(
|
||||||
@@ -1189,12 +1120,9 @@ func (e *environment) routeWatchURL(name string) string {
|
|||||||
|
|
||||||
// lookupSource reads the setting that is where clients are looked up:
|
// lookupSource reads the setting that is where clients are looked up:
|
||||||
// geojs, file, or off.
|
// geojs, file, or off.
|
||||||
func (e *environment) lookupSource(
|
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||||
name string, defaultValue LookupSource,
|
source := e.value(name, defaultValue)
|
||||||
) LookupSource {
|
if source != "geojs" && source != fileSource && source != off {
|
||||||
source := LookupSource(e.value(name, string(defaultValue)))
|
|
||||||
if source != LookupSourceGeoJS && source != LookupSourceFile &&
|
|
||||||
source != LookupSourceOff {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1206,12 +1134,12 @@ func (e *environment) lookupSource(
|
|||||||
// one source at a time.
|
// one source at a time.
|
||||||
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||||
switch {
|
switch {
|
||||||
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
|
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
|
||||||
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
||||||
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
||||||
errNeedsDBPath,
|
errNeedsDBPath,
|
||||||
))
|
))
|
||||||
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
|
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
|
||||||
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
||||||
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
||||||
cfg.LookupSource,
|
cfg.LookupSource,
|
||||||
@@ -1231,12 +1159,12 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
|||||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||||
e.check(ExclusivelyAllowedCountriesSetting,
|
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.LookupSource != LookupSourceOff {
|
if cfg.LookupSource != off {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1250,15 +1178,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
}{
|
}{
|
||||||
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||||
{TagLimitPercentSetting, slices.ContainsFunc(
|
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
||||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||||
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
||||||
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
||||||
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
||||||
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||||
@@ -1278,7 +1206,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||||
e.check(ASNLimitPercentURLSetting,
|
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
||||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1305,18 +1233,14 @@ func (e *environment) checkTags(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for tag := range cfg.TagLimitPercent {
|
for tag := range cfg.TagLimitPercent {
|
||||||
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkTag returns an error when tag would never apply, as checkTags says.
|
// checkTag returns an error when tag would never apply, as checkTags says.
|
||||||
func checkTag(cfg *WAF, tag string) error {
|
func checkTag(cfg *WAF, tag string) error {
|
||||||
if !requestCanGetTag(cfg, tag) {
|
if !requestCanGetTag(cfg, tag) {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
|
||||||
"%q %w",
|
|
||||||
reputation.MaskZoneKey(tag),
|
|
||||||
errNoRequestGetsTag,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -1324,7 +1248,7 @@ func checkTag(cfg *WAF, tag string) error {
|
|||||||
|
|
||||||
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
||||||
func requestCanGetTag(cfg *WAF, tag string) bool {
|
func requestCanGetTag(cfg *WAF, tag string) bool {
|
||||||
isListName := func(list reputation.TagList) bool { return list.Name == tag }
|
isListName := func(list TagList) bool { return list.Name == tag }
|
||||||
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -1348,7 +1272,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
|
|||||||
|
|
||||||
return isZoneTag &&
|
return isZoneTag &&
|
||||||
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
||||||
return strings.ToLower(reputation.MaskZoneKey(each)) == zone
|
return strings.ToLower(MaskZoneKey(each)) == zone
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1846,9 +1770,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
netblock, err := reputation.ParseNetblock(
|
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
|
||||||
strings.TrimSpace(netblockText),
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2054,6 +1976,21 @@ func parseCountry(value string) (string, error) {
|
|||||||
return country, nil
|
return country, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseASN reads an AS number such as AS64496, in either case, and
|
||||||
|
// returns it as the lookup gives it: AS and the number, in capitals and
|
||||||
|
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
||||||
|
// read with it too.
|
||||||
|
func ParseASN(value string) (string, error) {
|
||||||
|
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
||||||
|
|
||||||
|
number, err := strconv.ParseUint(digits, 10, 32)
|
||||||
|
if !hasAS || err != nil {
|
||||||
|
return "", fmt.Errorf("%q %w", value, errNotASN)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "AS" + strconv.FormatUint(number, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
// The tags a request can get that the settings can name, as waf
|
// The tags a request can get that the settings can name, as waf
|
||||||
// writes them. The prefixes are followed by a value.
|
// writes them. The prefixes are followed by a value.
|
||||||
const (
|
const (
|
||||||
@@ -2067,17 +2004,24 @@ const (
|
|||||||
ExploitSignatureTag = "exploitsignature"
|
ExploitSignatureTag = "exploitsignature"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
||||||
|
// requests of the clients it holds, and the URL it is fetched from.
|
||||||
|
type TagList struct {
|
||||||
|
Name string
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
||||||
// torexit=https://example.com/tor.txt, and returns each list with its name
|
// torexit=https://example.com/tor.txt, and returns each list with its name
|
||||||
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
||||||
// = before http. A name or a URL listed twice is an error.
|
// = before http. A name or a URL listed twice is an error.
|
||||||
func parseTagLists(value string) ([]reputation.TagList, error) {
|
func parseTagLists(value string) ([]TagList, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
lists := make([]reputation.TagList, 0, len(items))
|
lists := make([]TagList, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
end := strings.Index(item, "=http")
|
end := strings.Index(item, "=http")
|
||||||
@@ -2085,12 +2029,9 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
||||||
}
|
}
|
||||||
|
|
||||||
list := reputation.TagList{
|
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
|
||||||
Name: strings.ToLower(item[:end]),
|
|
||||||
URL: item[end+1:],
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool {
|
if slices.ContainsFunc(lists, func(earlier TagList) bool {
|
||||||
return earlier.Name == list.Name || earlier.URL == list.URL
|
return earlier.Name == list.Name || earlier.URL == list.URL
|
||||||
}) {
|
}) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||||
@@ -2103,8 +2044,7 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
||||||
// tag listed twice is an error, which shows the tag as
|
// tag listed twice is an error, which shows the tag as MaskZoneKey does.
|
||||||
// reputation.MaskZoneKey does.
|
|
||||||
func parseTags(value string) ([]string, error) {
|
func parseTags(value string) ([]string, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2124,7 +2064,7 @@ func parseTags(value string) ([]string, error) {
|
|||||||
if slices.Contains(tags, tag) {
|
if slices.Contains(tags, tag) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
reputation.MaskZoneKey(item),
|
MaskZoneKey(item),
|
||||||
errListedTwice,
|
errListedTwice,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2193,7 +2133,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if colon <= 0 {
|
if colon <= 0 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
reputation.MaskZoneKey(item),
|
MaskZoneKey(item),
|
||||||
errNotTagPercentItem,
|
errNotTagPercentItem,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2207,7 +2147,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if err != nil || percent < 1 || percent > 99 {
|
if err != nil || percent < 1 || percent > 99 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
reputation.MaskZoneKey(item[colon+1:]),
|
MaskZoneKey(item[colon+1:]),
|
||||||
errNotTagPercent,
|
errNotTagPercent,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2219,11 +2159,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if _, listed := percents[tag]; listed {
|
if _, listed := percents[tag]; listed {
|
||||||
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
||||||
// would not mask.
|
// would not mask.
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
|
||||||
"%q %w",
|
|
||||||
reputation.MaskZoneKey(tag),
|
|
||||||
errListedTwice,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
percents[tag] = percent
|
percents[tag] = percent
|
||||||
@@ -2257,7 +2193,7 @@ func parsePercents(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
percent, err := reputation.ParsePercent(percentText)
|
percent, err := ParsePercent(percentText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2272,6 +2208,38 @@ func parsePercents(
|
|||||||
return percents, nil
|
return percents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
||||||
|
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
||||||
|
func ParsePercent(value string) (int64, error) {
|
||||||
|
percent, err := strconv.ParseInt(value, 10, 64)
|
||||||
|
if err != nil || percent < 0 || percent > 100 {
|
||||||
|
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return percent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// headerNameChars are the characters RFC 9110 allows in a header name:
|
||||||
|
// letters, digits and these marks.
|
||||||
|
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
|
||||||
|
"0123456789!#$%&'*+-.^_`|~"
|
||||||
|
|
||||||
|
// IsHeaderName reports whether name can be a header name: one or more of
|
||||||
|
// the characters RFC 9110 allows in one.
|
||||||
|
func IsHeaderName(name string) bool {
|
||||||
|
if name == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, char := range name {
|
||||||
|
if !strings.ContainsRune(headerNameChars, char) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// parseHeaderNames reads a comma-separated list of header names in either
|
// parseHeaderNames reads a comma-separated list of header names in either
|
||||||
// case, and returns them in lower case. Host and Transfer-Encoding are
|
// case, and returns them in lower case. Host and Transfer-Encoding are
|
||||||
// refused: Go's HTTP server takes them out of the request's headers.
|
// refused: Go's HTTP server takes them out of the request's headers.
|
||||||
@@ -2284,7 +2252,7 @@ func parseHeaderNames(value string) ([]string, error) {
|
|||||||
headers := make([]string, 0, len(items))
|
headers := make([]string, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if !httpguts.ValidHeaderFieldName(item) {
|
if !IsHeaderName(item) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2440,8 +2408,8 @@ const (
|
|||||||
// other name for one that does not exist, so that the zone would list no
|
// other name for one that does not exist, so that the zone would list no
|
||||||
// client. A zone listed twice is an error, whatever the case of its
|
// client. A zone listed twice is an error, whatever the case of its
|
||||||
// letters, which DNS names ignore, and whatever its key, since
|
// letters, which DNS names ignore, and whatever its key, since
|
||||||
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a
|
// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
|
||||||
// zone as it does.
|
// MaskZoneKey does.
|
||||||
func parseZones(value string) ([]string, error) {
|
func parseZones(value string) ([]string, error) {
|
||||||
zones, err := parseList(value)
|
zones, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2451,14 +2419,11 @@ func parseZones(value string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
shown := reputation.MaskZoneKey(zone)
|
shown := MaskZoneKey(zone)
|
||||||
listedBefore := slices.ContainsFunc(
|
listedBefore := slices.ContainsFunc(
|
||||||
zones[:i],
|
zones[:i],
|
||||||
func(earlier string) bool {
|
func(earlier string) bool {
|
||||||
return strings.EqualFold(
|
return strings.EqualFold(MaskZoneKey(earlier), shown)
|
||||||
reputation.MaskZoneKey(earlier),
|
|
||||||
shown,
|
|
||||||
)
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2475,6 +2440,24 @@ func parseZones(value string) ([]string, error) {
|
|||||||
return zones, nil
|
return zones, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
||||||
|
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
||||||
|
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
||||||
|
// other zone it returns as it is. A zone is shown so wherever it leaves
|
||||||
|
// the process: in the log, the alerts and the metrics.
|
||||||
|
func MaskZoneKey(zone string) string {
|
||||||
|
// DNS names ignore case, and a name may be written with a dot at its
|
||||||
|
// end.
|
||||||
|
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||||
|
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
||||||
|
return zone
|
||||||
|
}
|
||||||
|
|
||||||
|
_, rest, _ := strings.Cut(zone, ".")
|
||||||
|
|
||||||
|
return masked + "." + rest
|
||||||
|
}
|
||||||
|
|
||||||
// isZone reports whether each label of zone is as parseZones takes it. It
|
// isZone reports whether each label of zone is as parseZones takes it. It
|
||||||
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
||||||
func isZone(zone string) bool {
|
func isZone(zone string) bool {
|
||||||
@@ -2530,7 +2513,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
|
|||||||
name, headerValue, found := strings.Cut(item, ":")
|
name, headerValue, found := strings.Cut(item, ":")
|
||||||
name = strings.TrimSpace(name)
|
name = strings.TrimSpace(name)
|
||||||
|
|
||||||
if !found || !httpguts.ValidHeaderFieldName(name) ||
|
if !found || !IsHeaderName(name) ||
|
||||||
strings.ContainsAny(headerValue, "\r\n\x00") {
|
strings.ContainsAny(headerValue, "\r\n\x00") {
|
||||||
return nil, "", fmt.Errorf(
|
return nil, "", fmt.Errorf(
|
||||||
"item %d %w",
|
"item %d %w",
|
||||||
|
|||||||
+25
-35
@@ -18,12 +18,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"golang.org/x/net/http/httpguts"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The settings, by name.
|
// The settings, by name.
|
||||||
@@ -616,7 +614,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
environment{
|
environment{
|
||||||
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
|
coreRuleSetMode: config.CoreRuleSetModeDetect,
|
||||||
coreRuleSetParanoiaLevel: "4",
|
coreRuleSetParanoiaLevel: "4",
|
||||||
coreRuleSetScoreThreshold: "10",
|
coreRuleSetScoreThreshold: "10",
|
||||||
coreRuleSetDisabledRules: "942100, 920350",
|
coreRuleSetDisabledRules: "942100, 920350",
|
||||||
@@ -866,32 +864,6 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// config and rules check header names with httpguts.ValidHeaderFieldName.
|
|
||||||
// It takes a name of one or more of the characters RFC 9110 allows in one,
|
|
||||||
// as the check it replaced did: letters, digits and these marks.
|
|
||||||
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const marks = "!#$%&'*+-.^_`|~"
|
|
||||||
|
|
||||||
if httpguts.ValidHeaderFieldName("") {
|
|
||||||
t.Error("the empty name is taken")
|
|
||||||
}
|
|
||||||
|
|
||||||
for b := range 256 {
|
|
||||||
char := byte(b)
|
|
||||||
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
|
|
||||||
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
|
|
||||||
|
|
||||||
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
|
|
||||||
// for a byte above 0x7f.
|
|
||||||
name := string([]byte{char})
|
|
||||||
if got := httpguts.ValidHeaderFieldName(name); got != want {
|
|
||||||
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1722,7 +1694,7 @@ func TestTagSettingsAsSet(t *testing.T) {
|
|||||||
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantLists := []reputation.TagList{
|
wantLists := []config.TagList{
|
||||||
{Name: torExitTag, URL: torURL},
|
{Name: torExitTag, URL: torURL},
|
||||||
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||||
}
|
}
|
||||||
@@ -2061,9 +2033,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
value string
|
value, action string
|
||||||
action config.ReputationAction
|
percent int64
|
||||||
percent int64
|
|
||||||
}{
|
}{
|
||||||
{actionDeny, actionDeny, 0},
|
{actionDeny, actionDeny, 0},
|
||||||
{actionLog, actionLog, 0},
|
{actionLog, actionLog, 0},
|
||||||
@@ -2196,7 +2167,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
|
|||||||
// want are the zones, resolver, action and percent config.WAF gives.
|
// want are the zones, resolver, action and percent config.WAF gives.
|
||||||
want []string
|
want []string
|
||||||
wantResolver string
|
wantResolver string
|
||||||
wantAction config.ReputationAction
|
wantAction string
|
||||||
wantPercent int64
|
wantPercent int64
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -2316,6 +2287,25 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for zone, want := range map[string]string{
|
||||||
|
spamhaus: spamhausMasked,
|
||||||
|
spamhaus + ".": spamhausMasked + ".",
|
||||||
|
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
||||||
|
dronebl: dronebl,
|
||||||
|
"dq.spamhaus.net": "dq.spamhaus.net",
|
||||||
|
spamhaus + ".example": spamhaus + ".example",
|
||||||
|
} {
|
||||||
|
if got := config.MaskZoneKey(zone); got != want {
|
||||||
|
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -3059,7 +3049,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
|||||||
logLevel: "info",
|
logLevel: "info",
|
||||||
ruleFilesDir: "/etc/sww/rules.d",
|
ruleFilesDir: "/etc/sww/rules.d",
|
||||||
ruleFilesEnabled: "true",
|
ruleFilesEnabled: "true",
|
||||||
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
|
coreRuleSetMode: config.CoreRuleSetModeBlock,
|
||||||
coreRuleSetParanoiaLevel: "1",
|
coreRuleSetParanoiaLevel: "1",
|
||||||
coreRuleSetScoreThreshold: "5",
|
coreRuleSetScoreThreshold: "5",
|
||||||
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
||||||
|
|||||||
+37
-60
@@ -84,7 +84,12 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
params Params
|
url string
|
||||||
|
answered func(Answer)
|
||||||
|
now func() time.Time
|
||||||
|
asked func()
|
||||||
|
failed func(err error, askingAgainIn time.Duration)
|
||||||
|
wentWithoutAnswer func()
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -94,13 +99,8 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way, and endAsking
|
// asking is true while a request to GeoJS is under way.
|
||||||
// ends it. stopped is true once StopAsking has been called, and
|
asking bool
|
||||||
// askGoroutines waits for the goroutine that makes the requests.
|
|
||||||
asking bool
|
|
||||||
endAsking context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
askGoroutines sync.WaitGroup
|
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
@@ -140,7 +140,12 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
params: params,
|
url: params.URL,
|
||||||
|
answered: params.Answered,
|
||||||
|
now: params.Now,
|
||||||
|
asked: params.Asked,
|
||||||
|
failed: params.Failed,
|
||||||
|
wentWithoutAnswer: params.WentWithoutAnswer,
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -183,7 +188,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -205,23 +210,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
|
||||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
|
||||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
|
||||||
// asked nothing.
|
|
||||||
func (g *GeoJS) StopAsking() {
|
|
||||||
g.mu.Lock()
|
|
||||||
|
|
||||||
g.stopped = true
|
|
||||||
if g.asking {
|
|
||||||
g.endAsking()
|
|
||||||
}
|
|
||||||
|
|
||||||
g.mu.Unlock()
|
|
||||||
|
|
||||||
g.askGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -263,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -300,19 +288,18 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, or asked nothing after
|
// GeoJS is left alone after a failure, so no answer can come.
|
||||||
// StopAsking, so no answer can come.
|
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -323,7 +310,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -336,28 +323,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way, it is left alone after a failure, or StopAsking has
|
// it is under way or it is left alone after a failure. The requests to
|
||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// GeoJS are for every client waiting, so they go on when the caller whose
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// ctx is given stops waiting.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
if g.asking || g.now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
g.asking = true
|
||||||
g.asking, g.endAsking = true, end
|
|
||||||
|
|
||||||
g.askGoroutines.Go(func() {
|
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||||
defer end()
|
|
||||||
|
|
||||||
g.askAboutWaiting(askCtx)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
// Answered, outside the lock, since Answered takes locks of its own.
|
||||||
// locks of its own.
|
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -366,15 +347,11 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
if err != nil && ctx.Err() != nil {
|
|
||||||
continue // ended by StopAsking, which is no failure
|
|
||||||
}
|
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.params.Answered != nil {
|
if g.answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.params.Answered(answer)
|
g.answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -385,13 +362,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, or StopAsking has been called, it returns none and
|
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||||
// notes that no request to GeoJS is under way.
|
// under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 || g.stopped {
|
if len(g.waiting) == 0 {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -423,7 +400,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -467,7 +444,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.params.Failed(err, g.retryDelay)
|
g.failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -497,7 +474,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.params.URL,
|
g.url,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -506,7 +483,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.params.Asked()
|
g.asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -318,44 +318,6 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
service, _, g, reports := startWithReports()
|
|
||||||
clients := newClients()
|
|
||||||
|
|
||||||
service.set(hanging)
|
|
||||||
g.LookUpWithoutWaiting(t.Context(), clients())
|
|
||||||
waitForRequests(t, service, 1)
|
|
||||||
|
|
||||||
// The request is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
g.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
reports.wantFailures(t)
|
|
||||||
|
|
||||||
// A new client is then not asked about, and its lookup does not
|
|
||||||
// wait.
|
|
||||||
service.set(answering)
|
|
||||||
wantCountry(t, g, clients(), "")
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("the lookup returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
waitForRequests(t, service, 1)
|
|
||||||
reports.wantFailures(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+267
-393
@@ -14,34 +14,14 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Params are what New builds the metrics from. Each part is nil while it is
|
|
||||||
// off, and a part that is nil adds no metric of its own.
|
|
||||||
type Params struct {
|
|
||||||
// BusiestCount is how many countries and how many AS numbers get series
|
|
||||||
// of their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS).
|
|
||||||
BusiestCount int
|
|
||||||
// InstanceName is the label instance every metric carries
|
|
||||||
// (SWW_WAF_INSTANCE_NAME).
|
|
||||||
InstanceName string
|
|
||||||
// Now returns the time at which Bans counts the bans active.
|
|
||||||
Now func() time.Time
|
|
||||||
|
|
||||||
Bans banLedger
|
|
||||||
Clients sized
|
|
||||||
Rules sized
|
|
||||||
Lists fetchedLists
|
|
||||||
DNSBL dnsblZones
|
|
||||||
AbuseIPDB abuseIPDBChecker
|
|
||||||
Alerts alertQueue
|
|
||||||
LookupFile lookupDatabase
|
|
||||||
RemoteLog remoteLogSender
|
|
||||||
}
|
|
||||||
|
|
||||||
// Metrics are sww's metrics. They are safe for concurrent use.
|
// Metrics are sww's metrics. They are safe for concurrent use.
|
||||||
type Metrics struct {
|
type Metrics struct {
|
||||||
// registry gives every metric registered with it the label instance.
|
// registry gives every metric registered with it the label instance.
|
||||||
@@ -57,10 +37,12 @@ type Metrics struct {
|
|||||||
rateLimitHits *prometheus.CounterVec
|
rateLimitHits *prometheus.CounterVec
|
||||||
offences *prometheus.CounterVec
|
offences *prometheus.CounterVec
|
||||||
coreRuleSetMatches *prometheus.CounterVec
|
coreRuleSetMatches *prometheus.CounterVec
|
||||||
ruleMatches *prometheus.CounterVec
|
// ruleMatches are made by AddRules, and reputationHits by
|
||||||
reputationHits *prometheus.CounterVec
|
// AddReputation.
|
||||||
countries *busiest
|
ruleMatches *prometheus.CounterVec
|
||||||
asns *busiest
|
reputationHits *prometheus.CounterVec
|
||||||
|
countries *busiest
|
||||||
|
asns *busiest
|
||||||
|
|
||||||
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
|
||||||
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
// that failed. RequestsWithoutGeoJSAnswer are the requests that needed
|
||||||
@@ -78,18 +60,20 @@ type Metrics struct {
|
|||||||
stateFileEditsSetAside *prometheus.CounterVec
|
stateFileEditsSetAside *prometheus.CounterVec
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns the metrics, with the Go runtime's and the process's own, and
|
// New returns the metrics, with the Go runtime's and the process's own.
|
||||||
// those read from each part in params as the metrics are asked for.
|
// busiestCount is how many countries and how many AS numbers get series of
|
||||||
|
// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric
|
||||||
|
// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance.
|
||||||
//
|
//
|
||||||
//nolint:funlen // a few lines for each metric, a list that grows with them
|
//nolint:funlen // a few lines for each metric, a list that grows with them
|
||||||
func New(params Params) *Metrics {
|
func New(busiestCount int, instanceName string) *Metrics {
|
||||||
byStatus := []string{"status_class", "action"}
|
byStatus := []string{"status_class", "action"}
|
||||||
byFile := []string{"file"}
|
byFile := []string{"file"}
|
||||||
registry := prometheus.NewRegistry()
|
registry := prometheus.NewRegistry()
|
||||||
|
|
||||||
m := &Metrics{
|
m := &Metrics{
|
||||||
registry: prometheus.WrapRegistererWith(
|
registry: prometheus.WrapRegistererWith(
|
||||||
prometheus.Labels{"instance": params.InstanceName}, registry),
|
prometheus.Labels{"instance": instanceName}, registry),
|
||||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
Name: "sww_requests_in_flight",
|
Name: "sww_requests_in_flight",
|
||||||
@@ -130,19 +114,8 @@ func New(params Params) *Metrics {
|
|||||||
"and the rule's id.",
|
"and the rule's id.",
|
||||||
[]string{"mode", "rule_id"},
|
[]string{"mode", "rule_id"},
|
||||||
),
|
),
|
||||||
ruleMatches: counterVec(
|
countries: newCountries(busiestCount),
|
||||||
"sww_rule_matches_total",
|
asns: newASNs(busiestCount),
|
||||||
"Requests that matched a rule of the rule files, by its id and action.",
|
|
||||||
[]string{"rule_id", "action"},
|
|
||||||
),
|
|
||||||
reputationHits: counterVec(
|
|
||||||
"sww_reputation_hits_total",
|
|
||||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
|
||||||
"by the list's URL, the zone, or abuseipdb.",
|
|
||||||
[]string{sourceLabel},
|
|
||||||
),
|
|
||||||
countries: newCountries(params.BusiestCount),
|
|
||||||
asns: newASNs(params.BusiestCount),
|
|
||||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
Name: "sww_geojs_requests_total",
|
Name: "sww_geojs_requests_total",
|
||||||
Help: "Requests to GeoJS.",
|
Help: "Requests to GeoJS.",
|
||||||
@@ -194,7 +167,6 @@ func New(params Params) *Metrics {
|
|||||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||||
m.requestDuration, m.upstreamDuration,
|
m.requestDuration, m.upstreamDuration,
|
||||||
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
m.rateLimitHits, m.offences, m.coreRuleSetMatches,
|
||||||
m.ruleMatches, m.reputationHits,
|
|
||||||
m.countries, m.asns,
|
m.countries, m.asns,
|
||||||
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
m.GeoJSRequests, m.GeoJSFailures, m.RequestsWithoutGeoJSAnswer,
|
||||||
m.stateFileWrites, m.stateFileWriteFailures,
|
m.stateFileWrites, m.stateFileWriteFailures,
|
||||||
@@ -202,45 +174,259 @@ func New(params Params) *Metrics {
|
|||||||
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
|
||||||
)
|
)
|
||||||
|
|
||||||
if params.Bans != nil {
|
|
||||||
m.addBans(params.Bans, params.Now)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Clients != nil {
|
|
||||||
m.addClients(params.Clients)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Rules != nil {
|
|
||||||
m.addRules(params.Rules)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Lists != nil {
|
|
||||||
m.addLists(params.Lists)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.DNSBL != nil {
|
|
||||||
m.addDNSBL(params.DNSBL)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.AbuseIPDB != nil {
|
|
||||||
m.addAbuseIPDB(params.AbuseIPDB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Alerts != nil {
|
|
||||||
m.addAlerts(params.Alerts)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.LookupFile != nil {
|
|
||||||
m.addLookupFile(params.LookupFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.RemoteLog != nil {
|
|
||||||
m.addRemoteLog(params.RemoteLog)
|
|
||||||
}
|
|
||||||
|
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddBansAndClients adds the metrics read from the ledger and the table
|
||||||
|
// of clients as the metrics are asked for: the bans made since the start,
|
||||||
|
// by cause, the bans active and permanent at now, and the clients in the
|
||||||
|
// table.
|
||||||
|
func (m *Metrics) AddBansAndClients(
|
||||||
|
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||||
|
) {
|
||||||
|
for _, cause := range []string{
|
||||||
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||||
|
} {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_bans_made_total",
|
||||||
|
Help: "Bans made, by cause.",
|
||||||
|
ConstLabels: prometheus.Labels{"cause": cause},
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ledger.Made(cause))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_active_bans",
|
||||||
|
Help: "Bans active now, the permanent ones included.",
|
||||||
|
}, func() float64 {
|
||||||
|
active, _ := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(active)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_permanent_bans",
|
||||||
|
Help: "Permanent bans not lifted.",
|
||||||
|
}, func() float64 {
|
||||||
|
_, permanent := ledger.Count(now())
|
||||||
|
|
||||||
|
return float64(permanent)
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_tracked_clients",
|
||||||
|
Help: "Clients in the table of clients.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(limiter.Len())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRules adds the metrics of the rule files: the requests that matched
|
||||||
|
// each rule, which RuleMatched counts, and the rules loaded from
|
||||||
|
// ruleFiles, read as the metrics are asked for. It is called once, before
|
||||||
|
// RuleMatched.
|
||||||
|
func (m *Metrics) AddRules(ruleFiles *rules.Files) {
|
||||||
|
m.ruleMatches = counterVec(
|
||||||
|
"sww_rule_matches_total",
|
||||||
|
"Requests that matched a rule of the rule files, by its id and action.",
|
||||||
|
[]string{"rule_id", "action"},
|
||||||
|
)
|
||||||
|
|
||||||
|
m.registry.MustRegister(m.ruleMatches,
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_rules_loaded",
|
||||||
|
Help: "Rules loaded from the rule files.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(ruleFiles.Len())
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRemoteLog adds the metrics of sending the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
||||||
|
// lines sent, those dropped, and those waiting in the buffer.
|
||||||
|
func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_sent_total",
|
||||||
|
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Sent())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_remote_log_lines_dropped_total",
|
||||||
|
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
||||||
|
"whose sending failed.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Dropped())
|
||||||
|
}),
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_remote_log_lines_waiting",
|
||||||
|
Help: "Log lines in the buffer, waiting to be sent.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(remote.Depth())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddLookupFile adds the metrics of the lookup database, read as the
|
||||||
|
// metrics are asked for: when the file in use was read, which lastRead
|
||||||
|
// returns, and the replacements of it that could not be read, which
|
||||||
|
// readFailures returns.
|
||||||
|
func (m *Metrics) AddLookupFile(
|
||||||
|
lastRead func() time.Time,
|
||||||
|
readFailures func() int,
|
||||||
|
) {
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
||||||
|
Help: "When the lookup database in use was read, in seconds since 1970.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(lastRead().Unix())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_lookup_database_read_failures_total",
|
||||||
|
Help: "Replacements of the lookup database that could not be read.",
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(readFailures())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
||||||
|
// DNSBL zone, its key masked, or abuseipdb.
|
||||||
|
const sourceLabel = "source"
|
||||||
|
|
||||||
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
|
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||||
|
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||||
|
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||||
|
// the fetches that failed and when the copy in use was fetched, and for a
|
||||||
|
// zone, the queries made and those that failed. It is called once, before
|
||||||
|
// ReputationHit.
|
||||||
|
func (m *Metrics) AddReputation(
|
||||||
|
lists *reputation.Lists,
|
||||||
|
dnsbl *reputation.DNSBL,
|
||||||
|
) {
|
||||||
|
m.reputationHits = counterVec(
|
||||||
|
"sww_reputation_hits_total",
|
||||||
|
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
||||||
|
"by the list's URL, the zone, or abuseipdb.",
|
||||||
|
[]string{sourceLabel},
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
|
for _, zone := range dnsbl.Zones() {
|
||||||
|
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
|
||||||
|
|
||||||
|
m.addReputationQueries(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Queries(zone) },
|
||||||
|
)
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return dnsbl.Failures(zone) },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, listURL := range lists.URLs() {
|
||||||
|
source := prometheus.Labels{sourceLabel: listURL}
|
||||||
|
|
||||||
|
m.addReputationFailures(
|
||||||
|
source,
|
||||||
|
func() int { return lists.Failures(listURL) },
|
||||||
|
)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
||||||
|
Help: "When the copy of the list in use was fetched, in seconds since " +
|
||||||
|
"1970, or 0 while there is none.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
fetched := lists.Fetched(listURL)
|
||||||
|
if fetched.IsZero() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return float64(fetched.Unix())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
||||||
|
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
||||||
|
// that failed, and how many checks the day's budget has left. It is
|
||||||
|
// called once, after AddReputation, while SWW_WAF_ABUSEIPDB_KEY is set.
|
||||||
|
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
||||||
|
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
||||||
|
|
||||||
|
m.addReputationQueries(source, abuseIPDB.Checked)
|
||||||
|
m.addReputationFailures(source, abuseIPDB.Failures)
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "sww_reputation_daily_budget_remaining",
|
||||||
|
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
||||||
|
ConstLabels: source,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(abuseIPDB.BudgetLeft())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||||
|
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||||
|
func (m *Metrics) ReputationHit(source string) {
|
||||||
|
m.reputationHits.WithLabelValues(source).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAlerts adds the metrics of the alerts sent to each destination set,
|
||||||
|
// read from queue as the metrics are asked for, by destination: the
|
||||||
|
// alerts sent, the requests to the destination that failed, the alerts
|
||||||
|
// held back, which are the same for every destination, and those
|
||||||
|
// dropped. With no destination set, it adds none.
|
||||||
|
func (m *Metrics) AddAlerts(queue *alerts.Queue) {
|
||||||
|
for _, name := range queue.DestinationsSet() {
|
||||||
|
destination := prometheus.Labels{"destination": name}
|
||||||
|
|
||||||
|
m.registry.MustRegister(
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_sent_total",
|
||||||
|
Help: "Alerts the destination took.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Sent)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alert_send_failures_total",
|
||||||
|
Help: "Requests to the destination that failed.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).SendFailures)
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_suppressed_total",
|
||||||
|
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
||||||
|
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Suppressed())
|
||||||
|
}),
|
||||||
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
Name: "sww_alerts_dropped_total",
|
||||||
|
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
||||||
|
"alerts given up as the destination refused them.",
|
||||||
|
ConstLabels: destination,
|
||||||
|
}, func() float64 {
|
||||||
|
return float64(queue.Counts(name).Dropped)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ServeHTTP answers with the metrics in the Prometheus text format.
|
// ServeHTTP answers with the metrics in the Prometheus text format.
|
||||||
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
m.handler.ServeHTTP(w, r)
|
m.handler.ServeHTTP(w, r)
|
||||||
@@ -310,12 +496,6 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
|
|||||||
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
|
||||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
|
||||||
func (m *Metrics) ReputationHit(source string) {
|
|
||||||
m.reputationHits.WithLabelValues(source).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
// StateFileWritten counts a write of the state or cache file name, of size
|
// StateFileWritten counts a write of the state or cache file name, of size
|
||||||
// bytes, that ended with err.
|
// bytes, that ended with err.
|
||||||
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
func (m *Metrics) StateFileWritten(name string, size int, err error) {
|
||||||
@@ -347,312 +527,6 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
|||||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// banLedger is what the metrics read from the ledger of bans.
|
|
||||||
type banLedger interface {
|
|
||||||
// Made returns the bans made since the start with cause.
|
|
||||||
Made(cause string) int
|
|
||||||
// Count returns the bans active at now, the permanent ones included,
|
|
||||||
// and the permanent ones.
|
|
||||||
Count(now time.Time) (active, permanent int)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addBans adds the metrics read from ledger as the metrics are asked for:
|
|
||||||
// the bans made since the start, by cause, and the bans active and
|
|
||||||
// permanent at now.
|
|
||||||
func (m *Metrics) addBans(ledger banLedger, now func() time.Time) {
|
|
||||||
for _, cause := range []string{
|
|
||||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
|
||||||
} {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_bans_made_total",
|
|
||||||
Help: "Bans made, by cause.",
|
|
||||||
ConstLabels: prometheus.Labels{"cause": cause},
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(ledger.Made(cause))
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_active_bans",
|
|
||||||
Help: "Bans active now, the permanent ones included.",
|
|
||||||
}, func() float64 {
|
|
||||||
active, _ := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(active)
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_permanent_bans",
|
|
||||||
Help: "Permanent bans not lifted.",
|
|
||||||
}, func() float64 {
|
|
||||||
_, permanent := ledger.Count(now())
|
|
||||||
|
|
||||||
return float64(permanent)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sized is what the metrics read from the table of clients and from the
|
|
||||||
// rule files.
|
|
||||||
type sized interface {
|
|
||||||
// Len returns how many clients the table holds, or how many rules were
|
|
||||||
// loaded from the rule files.
|
|
||||||
Len() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addClients adds the metric of the clients in table, read as the metrics
|
|
||||||
// are asked for.
|
|
||||||
func (m *Metrics) addClients(table sized) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_tracked_clients",
|
|
||||||
Help: "Clients in the table of clients.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(table.Len())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRules adds the metric of the rules loaded from files, read as the
|
|
||||||
// metrics are asked for.
|
|
||||||
func (m *Metrics) addRules(files sized) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_rules_loaded",
|
|
||||||
Help: "Rules loaded from the rule files.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(files.Len())
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// remoteLogSender is what the metrics read from what sends the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL.
|
|
||||||
type remoteLogSender interface {
|
|
||||||
// Sent returns the lines sent.
|
|
||||||
Sent() int64
|
|
||||||
// Dropped returns the lines dropped.
|
|
||||||
Dropped() int64
|
|
||||||
// Depth returns the lines waiting in the buffer.
|
|
||||||
Depth() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addRemoteLog adds the metrics of sending the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
|
|
||||||
// lines sent, those dropped, and those waiting in the buffer.
|
|
||||||
func (m *Metrics) addRemoteLog(remote remoteLogSender) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_sent_total",
|
|
||||||
Help: "Log lines sent to SWW_WAF_LOG_REMOTE_URL.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Sent())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_remote_log_lines_dropped_total",
|
|
||||||
Help: "Log lines dropped: the oldest in a full buffer, and those " +
|
|
||||||
"whose sending failed.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Dropped())
|
|
||||||
}),
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_remote_log_lines_waiting",
|
|
||||||
Help: "Log lines in the buffer, waiting to be sent.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(remote.Depth())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookupDatabase is what the metrics read from the lookup database.
|
|
||||||
type lookupDatabase interface {
|
|
||||||
// LastRead returns when the file in use was read.
|
|
||||||
LastRead() time.Time
|
|
||||||
// ReadFailures returns the replacements of the file that could not be
|
|
||||||
// read.
|
|
||||||
ReadFailures() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLookupFile adds the metrics of the lookup database, read from file as
|
|
||||||
// the metrics are asked for: when the file in use was read, and the
|
|
||||||
// replacements of it that could not be read.
|
|
||||||
func (m *Metrics) addLookupFile(file lookupDatabase) {
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_lookup_database_last_read_timestamp_seconds",
|
|
||||||
Help: "When the lookup database in use was read, in seconds since 1970.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(file.LastRead().Unix())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_lookup_database_read_failures_total",
|
|
||||||
Help: "Replacements of the lookup database that could not be read.",
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(file.ReadFailures())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceLabel is the label of the reputation metrics: a list's URL, a
|
|
||||||
// DNSBL zone, its key masked, or abuseipdb.
|
|
||||||
const sourceLabel = "source"
|
|
||||||
|
|
||||||
// dnsblZones is what the metrics read from what asks the DNSBL zones.
|
|
||||||
type dnsblZones interface {
|
|
||||||
// Zones returns the zones asked, their keys unmasked.
|
|
||||||
Zones() []string
|
|
||||||
// Queries returns the queries made to zone.
|
|
||||||
Queries(zone string) int
|
|
||||||
// Failures returns the queries to zone that failed.
|
|
||||||
Failures(zone string) int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDNSBL adds the metrics of each zone dnsbl asks, by source, the zone,
|
|
||||||
// its key masked as reputation.MaskZoneKey masks it, read as the metrics are
|
|
||||||
// asked for: the queries made and those that failed.
|
|
||||||
func (m *Metrics) addDNSBL(dnsbl dnsblZones) {
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
|
||||||
source := prometheus.Labels{
|
|
||||||
sourceLabel: reputation.MaskZoneKey(zone),
|
|
||||||
}
|
|
||||||
|
|
||||||
m.addReputationQueries(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Queries(zone) },
|
|
||||||
)
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return dnsbl.Failures(zone) },
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchedLists is what the metrics read from the lists fetched from URLs.
|
|
||||||
type fetchedLists interface {
|
|
||||||
// URLs returns the lists' URLs.
|
|
||||||
URLs() []string
|
|
||||||
// Failures returns the fetches of the list at listURL that failed.
|
|
||||||
Failures(listURL string) int
|
|
||||||
// Fetched returns when the copy in use of the list at listURL was
|
|
||||||
// fetched, or the zero time while there is none.
|
|
||||||
Fetched(listURL string) time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLists adds the metrics of each list fetched from a URL, by source, the
|
|
||||||
// list's URL, read from lists as the metrics are asked for: the fetches
|
|
||||||
// that failed and when the copy in use was fetched.
|
|
||||||
func (m *Metrics) addLists(lists fetchedLists) {
|
|
||||||
for _, listURL := range lists.URLs() {
|
|
||||||
source := prometheus.Labels{sourceLabel: listURL}
|
|
||||||
|
|
||||||
m.addReputationFailures(
|
|
||||||
source,
|
|
||||||
func() int { return lists.Failures(listURL) },
|
|
||||||
)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_last_fetch_timestamp_seconds",
|
|
||||||
Help: "When the copy of the list in use was fetched, in seconds since " +
|
|
||||||
"1970, or 0 while there is none.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
fetched := lists.Fetched(listURL)
|
|
||||||
if fetched.IsZero() {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return float64(fetched.Unix())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// abuseIPDBChecker is what the metrics read from what checks clients with
|
|
||||||
// AbuseIPDB.
|
|
||||||
type abuseIPDBChecker interface {
|
|
||||||
// Checked returns the checks made.
|
|
||||||
Checked() int
|
|
||||||
// Failures returns the checks that failed.
|
|
||||||
Failures() int
|
|
||||||
// BudgetLeft returns how many checks the day's budget has left.
|
|
||||||
BudgetLeft() int
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
|
|
||||||
// read from abuseIPDB as the metrics are asked for: the checks made, those
|
|
||||||
// that failed, and how many checks the day's budget has left.
|
|
||||||
func (m *Metrics) addAbuseIPDB(abuseIPDB abuseIPDBChecker) {
|
|
||||||
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
|
|
||||||
|
|
||||||
m.addReputationQueries(source, abuseIPDB.Checked)
|
|
||||||
m.addReputationFailures(source, abuseIPDB.Failures)
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
|
||||||
Name: "sww_reputation_daily_budget_remaining",
|
|
||||||
Help: "Checks of the day's SWW_WAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
|
|
||||||
ConstLabels: source,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(abuseIPDB.BudgetLeft())
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// alertQueue is what the metrics read from the queue of alerts.
|
|
||||||
type alertQueue interface {
|
|
||||||
// DestinationsSet returns the names of the destinations set.
|
|
||||||
DestinationsSet() []string
|
|
||||||
// Counts returns the alerts sent to the destination name, the requests
|
|
||||||
// to it that failed, and the alerts dropped.
|
|
||||||
Counts(name string) alerts.Counts
|
|
||||||
// Suppressed returns the alerts held back.
|
|
||||||
Suppressed() int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAlerts adds the metrics of the alerts sent to each destination set,
|
|
||||||
// read from queue as the metrics are asked for, by destination: the
|
|
||||||
// alerts sent, the requests to the destination that failed, the alerts
|
|
||||||
// held back, which are the same for every destination, and those
|
|
||||||
// dropped. With no destination set, it adds none.
|
|
||||||
func (m *Metrics) addAlerts(queue alertQueue) {
|
|
||||||
for _, name := range queue.DestinationsSet() {
|
|
||||||
destination := prometheus.Labels{"destination": name}
|
|
||||||
|
|
||||||
m.registry.MustRegister(
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_sent_total",
|
|
||||||
Help: "Alerts the destination took.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Sent)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alert_send_failures_total",
|
|
||||||
Help: "Requests to the destination that failed.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).SendFailures)
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_suppressed_total",
|
|
||||||
Help: "Alerts held back: repeats within SWW_WAF_ALERT_COOLDOWN, and " +
|
|
||||||
"alerts past SWW_WAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Suppressed())
|
|
||||||
}),
|
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
|
||||||
Name: "sww_alerts_dropped_total",
|
|
||||||
Help: "Alerts dropped, the oldest first, from a full queue, and " +
|
|
||||||
"alerts given up as the destination refused them.",
|
|
||||||
ConstLabels: destination,
|
|
||||||
}, func() float64 {
|
|
||||||
return float64(queue.Counts(name).Dropped)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// count counts a request that has ended, as RequestEnded and RequestReported
|
// count counts a request that has ended, as RequestEnded and RequestReported
|
||||||
// describe.
|
// describe.
|
||||||
func (m *Metrics) count(
|
func (m *Metrics) count(
|
||||||
|
|||||||
@@ -1,160 +0,0 @@
|
|||||||
package metrics_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
listURL = "https://lists.example/block.txt"
|
|
||||||
zone = "dnsbl.example"
|
|
||||||
destination = "webhook"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Each stand-in answers with a number of its own, so that each metric
|
|
||||||
// shows which part it was read from.
|
|
||||||
|
|
||||||
type ledgerStandIn struct{}
|
|
||||||
|
|
||||||
func (ledgerStandIn) Made(string) int { return 1 }
|
|
||||||
|
|
||||||
func (ledgerStandIn) Count(time.Time) (int, int) { return 2, 1 }
|
|
||||||
|
|
||||||
type clientTableStandIn struct{}
|
|
||||||
|
|
||||||
func (clientTableStandIn) Len() int { return 3 }
|
|
||||||
|
|
||||||
type ruleFilesStandIn struct{}
|
|
||||||
|
|
||||||
func (ruleFilesStandIn) Len() int { return 4 }
|
|
||||||
|
|
||||||
type listsStandIn struct{}
|
|
||||||
|
|
||||||
func (listsStandIn) URLs() []string { return []string{listURL} }
|
|
||||||
|
|
||||||
func (listsStandIn) Failures(string) int { return 5 }
|
|
||||||
|
|
||||||
func (listsStandIn) Fetched(string) time.Time { return time.Time{} }
|
|
||||||
|
|
||||||
type dnsblStandIn struct{}
|
|
||||||
|
|
||||||
func (dnsblStandIn) Zones() []string { return []string{zone} }
|
|
||||||
|
|
||||||
func (dnsblStandIn) Queries(string) int { return 6 }
|
|
||||||
|
|
||||||
func (dnsblStandIn) Failures(string) int { return 0 }
|
|
||||||
|
|
||||||
type abuseIPDBStandIn struct{}
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) Checked() int { return 0 }
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) Failures() int { return 0 }
|
|
||||||
|
|
||||||
func (abuseIPDBStandIn) BudgetLeft() int { return 7 }
|
|
||||||
|
|
||||||
type alertQueueStandIn struct{}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) DestinationsSet() []string {
|
|
||||||
return []string{destination}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) Counts(string) alerts.Counts {
|
|
||||||
return alerts.Counts{Sent: 8}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (alertQueueStandIn) Suppressed() int64 { return 0 }
|
|
||||||
|
|
||||||
type lookupDatabaseStandIn struct{}
|
|
||||||
|
|
||||||
func (lookupDatabaseStandIn) LastRead() time.Time { return time.Time{} }
|
|
||||||
|
|
||||||
func (lookupDatabaseStandIn) ReadFailures() int { return 9 }
|
|
||||||
|
|
||||||
type remoteLogStandIn struct{}
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Sent() int64 { return 10 }
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Dropped() int64 { return 0 }
|
|
||||||
|
|
||||||
func (remoteLogStandIn) Depth() int { return 0 }
|
|
||||||
|
|
||||||
func TestMetricsAreReadFromEachPart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m := metrics.New(metrics.Params{
|
|
||||||
BusiestCount: 1,
|
|
||||||
InstanceName: "app",
|
|
||||||
Now: func() time.Time { return time.Time{} },
|
|
||||||
Bans: ledgerStandIn{},
|
|
||||||
Clients: clientTableStandIn{},
|
|
||||||
Rules: ruleFilesStandIn{},
|
|
||||||
Lists: listsStandIn{},
|
|
||||||
DNSBL: dnsblStandIn{},
|
|
||||||
AbuseIPDB: abuseIPDBStandIn{},
|
|
||||||
Alerts: alertQueueStandIn{},
|
|
||||||
LookupFile: lookupDatabaseStandIn{},
|
|
||||||
RemoteLog: remoteLogStandIn{},
|
|
||||||
})
|
|
||||||
scraped := scrape(t, m)
|
|
||||||
|
|
||||||
for _, line := range []string{
|
|
||||||
`sww_bans_made_total{cause="limit",instance="app"} 1`,
|
|
||||||
`sww_active_bans{instance="app"} 2`,
|
|
||||||
`sww_tracked_clients{instance="app"} 3`,
|
|
||||||
`sww_rules_loaded{instance="app"} 4`,
|
|
||||||
`sww_reputation_failures_total{instance="app",source="` + listURL + `"} 5`,
|
|
||||||
`sww_reputation_queries_total{instance="app",source="` + zone + `"} 6`,
|
|
||||||
`sww_reputation_daily_budget_remaining{instance="app",source="abuseipdb"} 7`,
|
|
||||||
`sww_alerts_sent_total{destination="` + destination + `",instance="app"} 8`,
|
|
||||||
`sww_lookup_database_read_failures_total{instance="app"} 9`,
|
|
||||||
`sww_remote_log_lines_sent_total{instance="app"} 10`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(scraped, "\n"+line+"\n") {
|
|
||||||
t.Errorf("metrics\n%s\nwant\n%s", scraped, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPartThatIsOffAddsNoMetric(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
scraped := scrape(t, metrics.New(metrics.Params{
|
|
||||||
BusiestCount: 1,
|
|
||||||
InstanceName: "app",
|
|
||||||
}))
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"sww_bans_made_total",
|
|
||||||
"sww_tracked_clients",
|
|
||||||
"sww_rules_loaded",
|
|
||||||
"sww_reputation_",
|
|
||||||
"sww_alert",
|
|
||||||
"sww_lookup_database_",
|
|
||||||
"sww_remote_log_",
|
|
||||||
} {
|
|
||||||
if strings.Contains(scraped, name) {
|
|
||||||
t.Errorf("metrics\n%s\nshow %s", scraped, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// scrape returns the metrics m serves.
|
|
||||||
func scrape(t *testing.T, m *metrics.Metrics) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
scraped := httptest.NewRecorder()
|
|
||||||
m.ServeHTTP(
|
|
||||||
scraped,
|
|
||||||
httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, "/", http.NoBody,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
return scraped.Body.String()
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
// Package processlog builds the logger for the process's own messages,
|
|
||||||
// which it writes as JSON lines marked "type":"process". The request log
|
|
||||||
// is package requestlog's.
|
|
||||||
package processlog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Params are what New needs.
|
|
||||||
type Params struct {
|
|
||||||
// Output is where the lines are written.
|
|
||||||
Output io.Writer
|
|
||||||
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
|
||||||
InstanceName string
|
|
||||||
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
|
||||||
Level slog.Level
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns the logger for the process's own messages: JSON lines on
|
|
||||||
// params.Output, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's. It writes only the messages at params.Level or more
|
|
||||||
// severe; the request lines requestlog.Write writes are never held back.
|
|
||||||
func New(params Params) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
|
||||||
Level: params.Level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
requestlog.FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", params.InstanceName)
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package processlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// instanceName is the instance the tests' loggers are built with.
|
|
||||||
const instanceName = "host1/blog"
|
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: slog.LevelInfo,
|
|
||||||
}).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: level,
|
|
||||||
})
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,28 +156,16 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params are what New needs.
|
// New returns a Limiter for limits, with no client counted yet, whose
|
||||||
type Params struct {
|
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
||||||
// Limits are what each client's requests and bytes are counted against.
|
// it, the least recently seen client is dropped, with its history, and
|
||||||
Limits Limits
|
// starts afresh if it comes back.
|
||||||
// MaxClients is the most clients the table holds,
|
func New(limits Limits, maxClients int) *Limiter {
|
||||||
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
||||||
// is dropped, with its history, and starts afresh if it comes back.
|
|
||||||
MaxClients int
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns a Limiter with no client counted yet.
|
|
||||||
func New(params Params) *Limiter {
|
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
|
||||||
params.MaxClients,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
limits := params.Limits
|
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{
|
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
||||||
PerMinute: 5,
|
tableSize,
|
||||||
BytesPerDay: math.MaxInt64,
|
)
|
||||||
},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
||||||
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
||||||
Limits: ratelimit.Limits{PerMinute: 1},
|
|
||||||
MaxClients: maxClients,
|
|
||||||
})
|
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Params{
|
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Params{
|
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -110,12 +110,8 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way, each with the
|
// checking are the clients whose check is under way.
|
||||||
// function that ends it. stopped is true once StopChecking has been
|
checking map[netip.Prefix]bool
|
||||||
// called, and checkGoroutines waits for the goroutine of each check.
|
|
||||||
checking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
checkGoroutines sync.WaitGroup
|
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -138,7 +134,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]context.CancelFunc{},
|
checking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,11 +144,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, the day's checks have
|
// way, a check failed less than failureDelay ago, or the day's checks
|
||||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||||
// for a check. The check that uses the budget up is logged and raised as a
|
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||||
// source_failure alert. ctx is the context of the client's request, and a
|
// is the context of the client's request, and a check goes on after the
|
||||||
// check goes on after the request ends, until StopChecking.
|
// request ends.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -171,21 +167,14 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
_, checking := a.checking[client]
|
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||||
|
a.spent < a.params.DailyBudget
|
||||||
check := offender && !checking && !a.stopped &&
|
|
||||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
|
||||||
if check {
|
if check {
|
||||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
a.checking[client] = true
|
||||||
a.checking[client] = end
|
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
a.checkGoroutines.Go(func() {
|
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
a.check(checkCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -200,24 +189,6 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopChecking ends the checks under way, and returns once their goroutines
|
|
||||||
// have. A check it ends gives no score and is no failure: it is neither
|
|
||||||
// counted, logged nor raised as an alert, and checks are not left alone
|
|
||||||
// after it. The ended check stays spent from the day's budget. After
|
|
||||||
// StopChecking, Hit checks no client.
|
|
||||||
func (a *AbuseIPDB) StopChecking() {
|
|
||||||
a.mu.Lock()
|
|
||||||
|
|
||||||
a.stopped = true
|
|
||||||
for _, end := range a.checking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
a.mu.Unlock()
|
|
||||||
|
|
||||||
a.checkGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -344,8 +315,7 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay. A check
|
// source_failure alert, and no client is checked for failureDelay.
|
||||||
// ended as ctx ends, by StopChecking, is no failure.
|
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -353,26 +323,24 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
case isFailure:
|
} else {
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := abuseIPDBParams()
|
|
||||||
p.Alerts = queue
|
|
||||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
|
||||||
|
|
||||||
wantScore(t, checker, suspect, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The check is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
checker.StopChecking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantFailures(t, checker, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// An offender without a score is then not checked.
|
|
||||||
wantScore(t, checker, other, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
wantChecked(t, abuseIPDB, suspect)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -445,9 +408,12 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
|||||||
scores: map[string]int64{suspect: 100},
|
scores: map[string]int64{suspect: 100},
|
||||||
}
|
}
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
m.AddReputation(
|
||||||
})
|
reputation.New(params()),
|
||||||
|
reputation.NewDNSBL(dnsblParams()),
|
||||||
|
)
|
||||||
|
m.AddAbuseIPDB(checker)
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one it refuses with an answer
|
// One check that AbuseIPDB answers, and one it refuses with an answer
|
||||||
// that names the key.
|
// that names the key.
|
||||||
@@ -500,9 +466,12 @@ func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) {
|
|||||||
p := abuseIPDBParams()
|
p := abuseIPDBParams()
|
||||||
p.DailyBudget = 5
|
p.DailyBudget = 5
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, AbuseIPDB: checker,
|
m.AddReputation(
|
||||||
})
|
reputation.New(params()),
|
||||||
|
reputation.NewDNSBL(dnsblParams()),
|
||||||
|
)
|
||||||
|
m.AddAbuseIPDB(checker)
|
||||||
|
|
||||||
// One check that AbuseIPDB answers, and one that fails.
|
// One check that AbuseIPDB answers, and one that fails.
|
||||||
hitFrom(t, checker, suspect, true)
|
hitFrom(t, checker, suspect, true)
|
||||||
@@ -805,9 +774,6 @@ func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// instanceName is the label instance of the metrics the tests build.
|
|
||||||
const instanceName = "app"
|
|
||||||
|
|
||||||
// scrapeMetrics returns the metrics m serves.
|
// scrapeMetrics returns the metrics m serves.
|
||||||
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
|
|
||||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -31,8 +32,6 @@ const (
|
|||||||
// it fails, and no client is checked with AbuseIPDB after a check
|
// it fails, and no client is checked with AbuseIPDB after a check
|
||||||
// fails, so that a source refusing them is not asked on every request.
|
// fails, so that a source refusing them is not asked on every request.
|
||||||
failureDelay = time.Minute
|
failureDelay = time.Minute
|
||||||
// maskedKey is what MaskZoneKey shows in place of a zone's key.
|
|
||||||
maskedKey = "********"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -83,12 +82,8 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way, each with the function that ends
|
// asking are the queries under way.
|
||||||
// it. stopped is true once StopAsking has been called, and
|
asking map[query]bool
|
||||||
// queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[query]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -125,31 +120,13 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]context.CancelFunc{},
|
asking: map[query]bool{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
|
||||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
|
||||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
|
||||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
|
||||||
// the process: in the log, the alerts and the metrics.
|
|
||||||
func MaskZoneKey(zone string) string {
|
|
||||||
// DNS names ignore case, and a name may be written with a dot at its
|
|
||||||
// end.
|
|
||||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
|
||||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
|
||||||
return zone
|
|
||||||
}
|
|
||||||
|
|
||||||
_, rest, _ := strings.Cut(zone, ".")
|
|
||||||
|
|
||||||
return maskedKey + "." + rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
||||||
func (d *DNSBL) Zones() []string {
|
func (d *DNSBL) Zones() []string {
|
||||||
return slices.Clone(d.params.Zones)
|
return slices.Clone(d.params.Zones)
|
||||||
@@ -157,14 +134,13 @@ func (d *DNSBL) Zones() []string {
|
|||||||
|
|
||||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||||
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
||||||
// masked, as MaskZoneKey masks it, since they go to the request
|
// masked, as config.MaskZoneKey masks it, since they go to the request
|
||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, maxQueries are under way, or
|
// zone is left alone after a failure, or maxQueries are under way;
|
||||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
// ListedBy never waits for a query. ctx is the context of the client's
|
||||||
// context of the client's request, and a query goes on after the request
|
// request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -177,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
_, asking := d.asking[q]
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, MaskZoneKey(zone))
|
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||||
len(d.asking) < maxQueries:
|
d.asking[q] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
d.asking[q] = end
|
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
d.queryGoroutines.Go(func() {
|
go d.ask(context.WithoutCancel(ctx), q)
|
||||||
defer end()
|
|
||||||
|
|
||||||
d.ask(queryCtx, q)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their
|
|
||||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
|
||||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
|
||||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
|
||||||
func (d *DNSBL) StopAsking() {
|
|
||||||
d.mu.Lock()
|
|
||||||
|
|
||||||
d.stopped = true
|
|
||||||
for _, end := range d.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
d.mu.Unlock()
|
|
||||||
|
|
||||||
d.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -286,32 +238,30 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
// failureDelay.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := MaskZoneKey(q.zone)
|
shown := config.MaskZoneKey(q.zone)
|
||||||
|
|
||||||
// Raised before it is logged, so that the alert is there once the
|
// Raised before it is logged, so that the alert is there once the
|
||||||
// log line is.
|
// log line is.
|
||||||
|
|||||||
@@ -210,45 +210,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := dnsblParams(zone)
|
|
||||||
p.Alerts = queue
|
|
||||||
resolver := &resolverStandIn{hanging: true}
|
|
||||||
dnsbl := newDNSBL(resolver, p)
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
dnsbl.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A client without a verdict is then asked about in no zone.
|
|
||||||
wantZones(t, dnsbl, unlisted)
|
|
||||||
synctest.Wait()
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
wantAsked(t, resolver, listedName)
|
|
||||||
waitForTheResolver()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
@@ -358,9 +319,8 @@ func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) {
|
|||||||
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
"99.2.0.192." + otherZone + ".": {rcode: serverFailure},
|
||||||
}}
|
}}
|
||||||
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone))
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
m.AddReputation(reputation.New(params()), dnsbl)
|
||||||
})
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
wantZones(t, dnsbl, listed)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
@@ -405,9 +365,8 @@ func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) {
|
|||||||
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
"99.2.0.192." + keyed + ".": {addrs: []string{listing}},
|
||||||
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
"100.2.0.192." + keyed + ".": {rcode: serverFailure},
|
||||||
}}, p)
|
}}, p)
|
||||||
m := metrics.New(metrics.Params{
|
m := metrics.New(1, "app")
|
||||||
BusiestCount: 1, InstanceName: instanceName, DNSBL: dnsbl,
|
m.AddReputation(reputation.New(params()), dnsbl)
|
||||||
})
|
|
||||||
|
|
||||||
// Both clients are asked about before either answer comes, so that
|
// Both clients are asked about before either answer comes, so that
|
||||||
// the failure does not keep the zone from the other query.
|
// the failure does not keep the zone from the other query.
|
||||||
@@ -862,28 +821,3 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
|||||||
t.Errorf("asked about %v, want %v", got, want)
|
t.Errorf("asked about %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
dronebl = "dnsbl.dronebl.org"
|
|
||||||
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
|
||||||
spamhausMasked = "********.xbl.dq.spamhaus.net"
|
|
||||||
)
|
|
||||||
|
|
||||||
for zone, want := range map[string]string{
|
|
||||||
spamhaus: spamhausMasked,
|
|
||||||
spamhaus + ".": spamhausMasked + ".",
|
|
||||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
|
||||||
dronebl: dronebl,
|
|
||||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
|
||||||
spamhaus + ".example": spamhaus + ".example",
|
|
||||||
} {
|
|
||||||
if got := reputation.MaskZoneKey(zone); got != want {
|
|
||||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,10 +7,8 @@
|
|||||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||||
// today. The state package writes all of these to files and reads them
|
// today. The state package writes all of these to files and reads them
|
||||||
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
|
// from there, so that a restart keeps them too. Last, it fetches
|
||||||
// blacklist, which it does not keep. Last, it reads the netblocks, AS
|
// AbuseIPDB's blacklist, which it does not keep.
|
||||||
// numbers and percentages that both the settings and the lists hold, and
|
|
||||||
// masks the keys of DNSBL zones.
|
|
||||||
package reputation
|
package reputation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -22,12 +20,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -42,28 +40,15 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errStatus = errors.New("the server answered")
|
errStatus = errors.New("the server answered")
|
||||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||||
errNotNetblockLine = errors.New(
|
errNotNetblock = errors.New(
|
||||||
"is not an address or a netblock, such as 192.0.2.0/24",
|
"is not an address or a netblock, such as 192.0.2.0/24",
|
||||||
)
|
)
|
||||||
errNotASNPercent = errors.New(
|
errNotASNPercent = errors.New(
|
||||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||||
errNotNetblock = errors.New(
|
|
||||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
|
||||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
|
||||||
errNotPercent = errors.New(
|
|
||||||
"is not a percentage, a whole number from 0 to 100",
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
|
||||||
// requests of the clients it holds, and the URL it is fetched from.
|
|
||||||
type TagList struct {
|
|
||||||
Name string
|
|
||||||
URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
// List is a list as lists.json holds it: the URL it is fetched from,
|
// List is a list as lists.json holds it: the URL it is fetched from,
|
||||||
// when it was last tried, the fetch failed or not, and its last good copy:
|
// when it was last tried, the fetch failed or not, and its last good copy:
|
||||||
// when that was fetched, and its lines, as fetched, comment lines
|
// when that was fetched, and its lines, as fetched, comment lines
|
||||||
@@ -83,7 +68,7 @@ type Params struct {
|
|||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
ASNLimitPercentURL string
|
ASNLimitPercentURL string
|
||||||
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
||||||
TagLists []TagList
|
TagLists []config.TagList
|
||||||
// Refresh is how long after a list was last fetched or tried it is
|
// Refresh is how long after a list was last fetched or tried it is
|
||||||
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
||||||
Refresh time.Duration
|
Refresh time.Duration
|
||||||
@@ -480,11 +465,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
|
|
||||||
netblock, ok := parseNetblock(text)
|
netblock, ok := parseNetblock(text)
|
||||||
if !ok {
|
if !ok {
|
||||||
return entries{}, fmt.Errorf(
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
|
||||||
"line %d %w",
|
|
||||||
i+1,
|
|
||||||
errNotNetblockLine,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
found.netblocks[netblock] = true
|
found.netblocks[netblock] = true
|
||||||
@@ -504,7 +485,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
||||||
// stands for none, is refused.
|
// stands for none, is refused.
|
||||||
func parseNetblock(text string) (netip.Prefix, bool) {
|
func parseNetblock(text string) (netip.Prefix, bool) {
|
||||||
netblock, err := ParseNetblock(text)
|
netblock, err := config.ParseNetblock(text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netip.Prefix{}, false
|
return netip.Prefix{}, false
|
||||||
}
|
}
|
||||||
@@ -539,9 +520,9 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
asnText, percentText, _ := strings.Cut(text, ":")
|
asnText, percentText, _ := strings.Cut(text, ":")
|
||||||
asn, asnErr := ParseASN(asnText)
|
asn, asnErr := config.ParseASN(asnText)
|
||||||
|
|
||||||
percent, percentErr := ParsePercent(percentText)
|
percent, percentErr := config.ParsePercent(percentText)
|
||||||
if asnErr != nil || percentErr != nil {
|
if asnErr != nil || percentErr != nil {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
||||||
}
|
}
|
||||||
@@ -555,53 +536,6 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
return found, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
|
||||||
// address is a netblock of that address alone, a /32 or a /128. A
|
|
||||||
// blocklist's lines are read with it too.
|
|
||||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
|
||||||
if strings.Contains(value, "/") {
|
|
||||||
netblock, err := netip.ParsePrefix(value)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netblock.Masked(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(value)
|
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
|
||||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
|
||||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
|
||||||
// read with it too.
|
|
||||||
func ParseASN(value string) (string, error) {
|
|
||||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
|
||||||
|
|
||||||
number, err := strconv.ParseUint(digits, 10, 32)
|
|
||||||
if !hasAS || err != nil {
|
|
||||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
return "AS" + strconv.FormatUint(number, 10), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
|
||||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
|
||||||
func ParsePercent(value string) (int64, error) {
|
|
||||||
percent, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || percent < 0 || percent > 100 {
|
|
||||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
|
||||||
}
|
|
||||||
|
|
||||||
return percent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// withoutComment returns line without anything after a ; or a #, and
|
// withoutComment returns line without anything after a ; or a #, and
|
||||||
// without the spaces around what is left.
|
// without the spaces around what is left.
|
||||||
func withoutComment(line string) string {
|
func withoutComment(line string) string {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the request log sww prints on stdout: JSON
|
// Package requestlog writes the lines sww prints on stdout: one
|
||||||
// lines marked "type":"request", at most two from waf and a short one
|
// JSON object per request, two from waf and a short one from rproxy,
|
||||||
// from rproxy for each request. Package processlog writes the process's
|
// marked "type":"request", and the process's own messages as JSON lines marked
|
||||||
// own messages.
|
// "type":"process".
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewProcessLogger returns the logger for the process's own messages:
|
||||||
|
// JSON lines on w, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
||||||
|
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
||||||
|
// the request lines Write writes are never held back.
|
||||||
|
func NewProcessLogger(
|
||||||
|
w io.Writer,
|
||||||
|
instanceName string,
|
||||||
|
level slog.Level,
|
||||||
|
) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
||||||
|
Level: level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", instanceName)
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := requestlog.NewProcessLogger(
|
||||||
|
&out,
|
||||||
|
"host1/blog",
|
||||||
|
level,
|
||||||
|
)
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -99,12 +99,8 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way, each with the
|
// asking are the clients whose query is under way.
|
||||||
// function that ends it. stopped is true once StopAsking has been
|
asking map[netip.Prefix]bool
|
||||||
// called, and queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -120,7 +116,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]context.CancelFunc{},
|
asking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,9 +124,8 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||||
// is the context of the client's request, and a query goes on after the request
|
// client's request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -152,38 +147,15 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
_, asking := c.asking[client]
|
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
c.asking[client] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
c.asking[client] = end
|
|
||||||
|
|
||||||
c.queryGoroutines.Go(func() {
|
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
c.ask(queryCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their goroutines
|
|
||||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
|
||||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
|
||||||
// routewatch nothing.
|
|
||||||
func (c *Client) StopAsking() {
|
|
||||||
c.mu.Lock()
|
|
||||||
|
|
||||||
c.stopped = true
|
|
||||||
for _, end := range c.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
c.mu.Unlock()
|
|
||||||
|
|
||||||
c.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
// those under way is logged.
|
||||||
// failure.
|
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -242,20 +213,18 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
|
||||||
// says.
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
f := &fixture{t: t, log: &syncBuffer{}}
|
|
||||||
f.client = routewatch.New(routewatch.Params{
|
|
||||||
URL: silentURL, Now: time.Now,
|
|
||||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
|
||||||
})
|
|
||||||
|
|
||||||
f.route()
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
f.client.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A query started now would fail within waitLimit. None is, and the
|
|
||||||
// one ended is no failure: nothing is logged, and routewatch is not
|
|
||||||
// left alone.
|
|
||||||
for range 2 {
|
|
||||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
|
||||||
t.Errorf(
|
|
||||||
"Route gave %q, want %q",
|
|
||||||
missing,
|
|
||||||
routewatch.NoAnswerYet,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(waitLimit)
|
|
||||||
synctest.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.log.String() != "" {
|
|
||||||
t.Errorf("process log %q, want nothing", f.log.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package rproxy
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
@@ -22,7 +23,9 @@ type requestBody struct {
|
|||||||
// body is the visitor's body, ending in an *http.MaxBytesError past
|
// body is the visitor's body, ending in an *http.MaxBytesError past
|
||||||
// SWW_RPROXY_MAX_REQUEST_SIZE.
|
// SWW_RPROXY_MAX_REQUEST_SIZE.
|
||||||
body io.ReadCloser
|
body io.ReadCloser
|
||||||
rq *request
|
// refuse refuses the request with a status, as the request's own refuse
|
||||||
|
// does.
|
||||||
|
refuse func(status int)
|
||||||
// held is the part read before the request goes to the app, which Read gives
|
// held is the part read before the request goes to the app, which Read gives
|
||||||
// first.
|
// first.
|
||||||
held []byte
|
held []byte
|
||||||
@@ -68,7 +71,7 @@ func (b *requestBody) Read(p []byte) (int, error) {
|
|||||||
case errors.Is(err, io.EOF):
|
case errors.Is(err, io.EOF):
|
||||||
b.received.Store(true)
|
b.received.Store(true)
|
||||||
case errors.As(err, &tooLarge):
|
case errors.As(err, &tooLarge):
|
||||||
b.rq.refuse(http.StatusRequestEntityTooLarge)
|
b.refuse(http.StatusRequestEntityTooLarge)
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
@@ -86,7 +89,13 @@ type responseBody struct {
|
|||||||
// body is the app's body, ending in an *http.MaxBytesError past
|
// body is the app's body, ending in an *http.MaxBytesError past
|
||||||
// SWW_RPROXY_MAX_RESPONSE_SIZE.
|
// SWW_RPROXY_MAX_RESPONSE_SIZE.
|
||||||
body io.ReadCloser
|
body io.ReadCloser
|
||||||
rq *request
|
// refuse refuses the request with a status, as the request's own refuse
|
||||||
|
// does.
|
||||||
|
refuse func(status int)
|
||||||
|
// responseReceived is called once the app's whole answer has arrived.
|
||||||
|
responseReceived func()
|
||||||
|
// ctx is the visitor's request's context.
|
||||||
|
ctx context.Context //nolint:containedctx // the body ends with the request
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read reads from the app's body.
|
// Read reads from the app's body.
|
||||||
@@ -100,15 +109,15 @@ func (b *responseBody) Read(p []byte) (int, error) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, io.EOF):
|
case errors.Is(err, io.EOF):
|
||||||
b.rq.responseReceived()
|
b.responseReceived()
|
||||||
case errors.As(err, &tooLarge):
|
case errors.As(err, &tooLarge):
|
||||||
b.rq.refuse(http.StatusBadGateway)
|
b.refuse(http.StatusBadGateway)
|
||||||
|
|
||||||
return n, errResponseTooLarge
|
return n, errResponseTooLarge
|
||||||
case b.rq.in.Context().Err() == nil:
|
case b.ctx.Err() == nil:
|
||||||
// The answer broke off, not because the visitor went away. If a time
|
// The answer broke off, not because the visitor went away. If a time
|
||||||
// limit cut it, that refusal came first and is the one kept.
|
// limit cut it, that refusal came first and is the one kept.
|
||||||
b.rq.refuse(http.StatusBadGateway)
|
b.refuse(http.StatusBadGateway)
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
|
|||||||
@@ -2,12 +2,6 @@ package rproxy
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
|
||||||
// long requests take cannot matter to it. It is called before s serves.
|
|
||||||
func (s *Server) StopBreakerClock(at time.Time) {
|
|
||||||
s.h.breakerClock = func() time.Time { return at }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Breaker is the circuit breaker, for tests that give it the time.
|
// Breaker is the circuit breaker, for tests that give it the time.
|
||||||
type Breaker struct {
|
type Breaker struct {
|
||||||
b breaker
|
b breaker
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
@@ -870,14 +870,10 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
|
|||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// rproxy is waiting for the rest of the body well within this limit, even on
|
|
||||||
// a busy host, and answers when it runs out, well before dial's deadline.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxRequestTime: "3000",
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1266,9 +1262,7 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The app's answer arrives well within this limit, even on a busy host, and
|
const limit = 2 * time.Second
|
||||||
// the connection closes when it runs out, well before dial's deadline.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, answerBeforeTheBody)
|
app := startApp(t, answerBeforeTheBody)
|
||||||
@@ -1433,20 +1427,17 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
|
|||||||
|
|
||||||
const answer = "the end of a long answer"
|
const answer = "the end of a long answer"
|
||||||
|
|
||||||
// The request reaches the app well within this limit, even on a busy host.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
_ = http.NewResponseController(w).Flush()
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(limit + 100*time.Millisecond)
|
time.Sleep(2100 * time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
_, _ = io.WriteString(w, answer)
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxRequestTime: "2000",
|
||||||
})
|
})
|
||||||
|
|
||||||
got := visit(
|
got := visit(
|
||||||
|
|||||||
@@ -126,8 +126,8 @@ func (h *handler) newRequest(
|
|||||||
|
|
||||||
if r.Body != http.NoBody {
|
if r.Body != http.NoBody {
|
||||||
rq.body = &requestBody{
|
rq.body = &requestBody{
|
||||||
body: limitBody(r.Body, h.config.MaxRequestSize),
|
body: limitBody(r.Body, h.config.MaxRequestSize),
|
||||||
rq: rq,
|
refuse: rq.refuse,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,7 +294,12 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
|||||||
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
||||||
rq.startResponseTimer(requestEnd)
|
rq.startResponseTimer(requestEnd)
|
||||||
|
|
||||||
res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
|
res.Body = &responseBody{
|
||||||
|
body: limitBody(res.Body, maxBytes),
|
||||||
|
refuse: rq.refuse,
|
||||||
|
responseReceived: rq.responseReceived,
|
||||||
|
ctx: rq.in.Context(),
|
||||||
|
}
|
||||||
rq.out.afterStatus = rq.setResponseDeadline
|
rq.out.afterStatus = rq.setResponseDeadline
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -94,7 +94,6 @@ func New(params Params) *Server {
|
|||||||
reportURL: (&url.URL{
|
reportURL: (&url.URL{
|
||||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||||
}).String(),
|
}).String(),
|
||||||
breakerClock: time.Now,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Server{
|
return &Server{
|
||||||
@@ -134,10 +133,8 @@ type handler struct {
|
|||||||
wafTransport http.RoundTripper
|
wafTransport http.RoundTripper
|
||||||
// reportURL is where the reports go.
|
// reportURL is where the reports go.
|
||||||
reportURL string
|
reportURL string
|
||||||
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
// breaker is the circuit breaker for asks to waf.
|
||||||
// the time.
|
breaker breaker
|
||||||
breaker breaker
|
|
||||||
breakerClock func() time.Time
|
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
// reports waits for the reports being sent. reportsMu guards starting one
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
// against reportsStopped, which is set once nothing is to wait for another.
|
||||||
reports sync.WaitGroup
|
reports sync.WaitGroup
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -356,19 +356,6 @@ func startServerLogging(
|
|||||||
) (string, *rproxy.Server) {
|
) (string, *rproxy.Server) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
|
||||||
|
|
||||||
return serve(t, server), server
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServer returns rproxy, not yet serving, with its request lines going to
|
|
||||||
// requestLog and its process lines to processLog.
|
|
||||||
func newServer(
|
|
||||||
t *testing.T, backend, appURL string, env map[string]string,
|
|
||||||
requestLog, processLog io.Writer,
|
|
||||||
) *rproxy.Server {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
wafBackendTimeout: "60000",
|
wafBackendTimeout: "60000",
|
||||||
@@ -380,20 +367,15 @@ func newServer(
|
|||||||
t.Fatalf("settings %v: %v", settings, err)
|
t.Fatalf("settings %v: %v", settings, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: processLog,
|
processLog,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
|
||||||
// serve has server serve until the test ends, and returns its address.
|
|
||||||
func serve(t *testing.T, server *rproxy.Server) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
t.Context(),
|
t.Context(),
|
||||||
@@ -414,7 +396,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
|
|||||||
server.WaitForReports()
|
server.WaitForReports()
|
||||||
})
|
})
|
||||||
|
|
||||||
return listener.Addr().String()
|
return listener.Addr().String(), server
|
||||||
}
|
}
|
||||||
|
|
||||||
// answered is the visitor's answer.
|
// answered is the visitor's answer.
|
||||||
@@ -1452,18 +1434,14 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|||||||
|
|
||||||
greet(w, r)
|
greet(w, r)
|
||||||
})
|
})
|
||||||
out := &output{}
|
addr, out := startRProxy(
|
||||||
server := newServer(
|
t,
|
||||||
t, standIn.URL, app.URL,
|
standIn.URL,
|
||||||
map[string]string{wafBackendTimeout: "200"},
|
app.URL,
|
||||||
out, out,
|
map[string]string{
|
||||||
|
wafBackendTimeout: "200",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
// The breaker's clock stands still, as in
|
|
||||||
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
|
|
||||||
// failures could take more than five seconds, and the next request could
|
|
||||||
// come more than a second later.
|
|
||||||
server.StopBreakerClock(time.Now())
|
|
||||||
addr := serve(t, server)
|
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
// An ask can time out before it reaches the stand-in, so these
|
||||||
// may leave fewer than five questions, or leave some late.
|
// may leave fewer than five questions, or leave some late.
|
||||||
@@ -1554,13 +1532,9 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
|||||||
tooLarge(w, r)
|
tooLarge(w, r)
|
||||||
})
|
})
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
out := &output{}
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
wafBackendTimeout: "60000",
|
||||||
// The breaker's clock stands still. On a slow host, the five failures could
|
})
|
||||||
// otherwise take more than the five seconds that open the breaker. The next
|
|
||||||
// request could come more than a second later, when it may ask waf again.
|
|
||||||
server.StopBreakerClock(time.Now())
|
|
||||||
addr := serve(t, server)
|
|
||||||
|
|
||||||
for range 5 {
|
for range 5 {
|
||||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||||
@@ -1835,15 +1809,11 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The app's answer to the upgrade arrives well within these limits, even on a
|
|
||||||
// busy host.
|
|
||||||
const limit = 5 * time.Second
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, echoAfterUpgrade)
|
app := startApp(t, echoAfterUpgrade)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxRequestTime: "2000",
|
||||||
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
maxResponseTime: "2000",
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1868,10 +1838,7 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Past both time limits, the connection still carries what is sent.
|
// Past both time limits, the connection still carries what is sent.
|
||||||
time.Sleep(limit + 100*time.Millisecond)
|
time.Sleep(2100 * time.Millisecond)
|
||||||
|
|
||||||
// dial's deadline ran from before the upgrade and this wait.
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
|
|
||||||
const message = "still here\n"
|
const message = "still here\n"
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"maps"
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -96,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
) (*wafAnswer, error) {
|
) (*wafAnswer, error) {
|
||||||
b := &rq.h.breaker
|
b := &rq.h.breaker
|
||||||
if !b.mayAsk(rq.h.breakerClock()) {
|
if !b.mayAsk(time.Now()) {
|
||||||
return nil, errBreakerOpen
|
return nil, errBreakerOpen
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
case err == nil:
|
case err == nil:
|
||||||
b.noteAnswer()
|
b.noteAnswer()
|
||||||
case ctx.Err() == nil:
|
case ctx.Err() == nil:
|
||||||
b.noteFailure(rq.h.breakerClock())
|
b.noteFailure(time.Now())
|
||||||
}
|
}
|
||||||
|
|
||||||
return answer, err
|
return answer, err
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
"github.com/fsnotify/fsnotify"
|
||||||
"golang.org/x/net/http/httpguts"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
|
|||||||
rule.Target,
|
rule.Target,
|
||||||
errNotTarget,
|
errNotTarget,
|
||||||
)
|
)
|
||||||
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
|
case isHeader && !config.IsHeaderName(headerName):
|
||||||
return Rule{}, false, fmt.Errorf(
|
return Rule{}, false, fmt.Errorf(
|
||||||
"the target %q %w",
|
"the target %q %w",
|
||||||
rule.Target,
|
rule.Target,
|
||||||
|
|||||||
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
|
|||||||
|
|
||||||
// A second ban, made while the first waits to be written, puts the
|
// A second ban, made while the first waits to be written, puts the
|
||||||
// write off no further, and is written with it.
|
// write off no further, and is written with it.
|
||||||
first, _ := params.Ledger.Ban(
|
first, _ := params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
second, _ := params.Ledger.Ban(
|
second, _ := params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5*time.Second - time.Nanosecond)
|
time.Sleep(5*time.Second - time.Nanosecond)
|
||||||
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
|
|||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// A ban, and bans.json written with it.
|
// A ban, and bans.json written with it.
|
||||||
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
|
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
midnight(), bans.Notes{})
|
||||||
time.Sleep(params.BansWriteDelay)
|
time.Sleep(params.BansWriteDelay)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
|
|
||||||
// A second ban is to be written BansWriteDelay later. Just before
|
// A second ban is to be written BansWriteDelay later. Just before
|
||||||
// then, an admin saves bans.json with the first ban lifted and
|
// then, an admin saves bans.json with the first ban lifted and
|
||||||
// another added.
|
// another added.
|
||||||
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
|
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
midnight(), bans.Notes{})
|
||||||
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
edit(t, dir, bansJSON, permanentBansJSON)
|
edit(t, dir, bansJSON, permanentBansJSON)
|
||||||
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.Ban(
|
params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
params.Limiter.Count(
|
params.Limiter.Count(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.Ban(
|
params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
err = files.WriteAll()
|
err = files.WriteAll()
|
||||||
@@ -2135,11 +2135,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
|||||||
|
|
||||||
// The ban that liftedBansJSON lifts, before it is lifted.
|
// The ban that liftedBansJSON lifts, before it is lifted.
|
||||||
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
||||||
params.Ledger.Ban(
|
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
||||||
if !banned {
|
if !banned {
|
||||||
@@ -2430,7 +2426,7 @@ func midnight() time.Time {
|
|||||||
// counts, with thresholds fill does not reach.
|
// counts, with thresholds fill does not reach.
|
||||||
func newParams(dir string) state.Params {
|
func newParams(dir string) state.Params {
|
||||||
discard := slog.New(slog.DiscardHandler)
|
discard := slog.New(slog.DiscardHandler)
|
||||||
m := metrics.New(metrics.Params{BusiestCount: 1, InstanceName: "app"})
|
m := metrics.New(1, "app")
|
||||||
queue := alerts.New(alerts.Params{
|
queue := alerts.New(alerts.Params{
|
||||||
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
|
||||||
Events: alerts.Events(),
|
Events: alerts.Events(),
|
||||||
@@ -2452,7 +2448,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
@@ -2514,15 +2510,11 @@ func fill(params state.Params) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
params.Ledger.Load([]bans.Ban{permanentBan()})
|
params.Ledger.Load([]bans.Ban{permanentBan()})
|
||||||
params.Ledger.Ban(client, now, bans.Grounds{
|
params.Ledger.BanForLimit(client, now, bans.Notes{
|
||||||
Cause: bans.CauseLimit,
|
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
|
||||||
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
|
|
||||||
})
|
})
|
||||||
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
|
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
|
||||||
bans.Grounds{
|
bans.Notes{RuleID: "env-file", Target: "path"})
|
||||||
Cause: bans.CauseAttack,
|
|
||||||
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
||||||
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
||||||
@@ -2708,11 +2700,7 @@ func wantLiftedBanKept(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Were the lifted ban counted, the next would last three hours.
|
// Were the lifted ban counted, the next would last three hours.
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
afterLifting(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour {
|
if ban.Expires.Sub(ban.Start) != time.Hour {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban lasts %s, want 1h",
|
"the next ban lasts %s, want 1h",
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: params.Stdout,
|
params.Stdout,
|
||||||
InstanceName: hostname,
|
hostname,
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
})
|
)
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -755,7 +755,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
||||||
|
// read from it ends with an error once waitLimit is up.
|
||||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -768,6 +769,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("set the deadline: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
_, err = io.WriteString(conn, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send: %v", err)
|
t.Fatalf("send: %v", err)
|
||||||
@@ -832,9 +838,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
|
||||||
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
text, err := io.ReadAll(conn)
|
||||||
if err != nil || len(text) != 0 {
|
if err != nil || len(text) != 0 {
|
||||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||||
|
|||||||
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
|
|||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE": string(
|
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
|
||||||
config.CoreRuleSetModeOff,
|
|
||||||
),
|
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
|
|||||||
+43
-105
@@ -2,7 +2,6 @@ package sww
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -19,10 +18,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,11 +42,8 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(params.Stdout,
|
||||||
Output: params.Stdout,
|
config.InstanceName(params.Viper), slog.LevelError)
|
||||||
InstanceName: config.InstanceName(params.Viper),
|
|
||||||
Level: slog.LevelError,
|
|
||||||
})
|
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -66,11 +62,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = processlog.New(processlog.Params{
|
processLog = requestlog.NewProcessLogger(
|
||||||
Output: stdout,
|
stdout,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -121,8 +117,7 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
// The last write follows the stop of every hook, and so the end of every
|
// The last write follows the stop of every hook, and so the end of every
|
||||||
// loop that changes the state or raises an alert, and of every ask of
|
// loop that changes the state or raises an alert. A hook of the state
|
||||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
|
||||||
// files would stop before the loops of the parts built ahead of them.
|
// files would stop before the loops of the parts built ahead of them.
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
err := files.WriteAll()
|
err := files.WriteAll()
|
||||||
@@ -180,35 +175,11 @@ func newRuleFiles(
|
|||||||
return ruleFiles, nil
|
return ruleFiles, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
|
||||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
||||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
||||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
||||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
||||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
||||||
// set file is loaded with the Core Rule Set only to check it.
|
|
||||||
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
crs, err := waf.NewCoreRuleSet(cfg)
|
||||||
cfg.CoreRuleSetExclusionsFile == "" {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
crs, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, startError{
|
return nil, startError{"cannot use the Core Rule Set", err}
|
||||||
"cannot use the Core Rule Set",
|
|
||||||
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return crs, nil
|
return crs, nil
|
||||||
@@ -221,7 +192,7 @@ func newLookupFile(
|
|||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue,
|
alertQueue *alerts.Queue,
|
||||||
) (*lookup.File, error) {
|
) (*lookup.File, error) {
|
||||||
if cfg.LookupSource != config.LookupSourceFile {
|
if cfg.LookupSource != "file" {
|
||||||
return nil, nil //nolint:nilnil // no lookup database, and no error
|
return nil, nil //nolint:nilnil // no lookup database, and no error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,17 +213,14 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Params{
|
return ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
@@ -304,13 +272,11 @@ func newLists(
|
|||||||
return lists
|
return lists
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
||||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
|
||||||
func newDNSBL(
|
func newDNSBL(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) *reputation.DNSBL {
|
) *reputation.DNSBL {
|
||||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
return reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
Zones: cfg.DNSBLZones,
|
Zones: cfg.DNSBLZones,
|
||||||
Resolver: cfg.DNSBLResolver,
|
Resolver: cfg.DNSBLResolver,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
@@ -319,17 +285,12 @@ func newDNSBL(
|
|||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
Alerts: alertQueue,
|
Alerts: alertQueue,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
|
||||||
|
|
||||||
return dnsbl
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
||||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
|
||||||
func newAbuseIPDB(
|
func newAbuseIPDB(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||||
@@ -355,30 +316,24 @@ func newAbuseIPDB(
|
|||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
|
||||||
|
|
||||||
return abuseIPDB
|
return abuseIPDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
// way, as routewatch.Client.StopAsking says.
|
|
||||||
func newRouteWatch(
|
func newRouteWatch(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger,
|
||||||
) *routewatch.Client {
|
) *routewatch.Client {
|
||||||
if cfg.RouteWatchURL == "" {
|
if cfg.RouteWatchURL == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
routeWatch := routewatch.New(routewatch.Params{
|
return routewatch.New(routewatch.Params{
|
||||||
URL: cfg.RouteWatchURL,
|
URL: cfg.RouteWatchURL,
|
||||||
Now: utcNow,
|
Now: utcNow,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
|
||||||
|
|
||||||
return routeWatch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newMetrics returns the metrics, with those read from the other parts as
|
// newMetrics returns the metrics, with those read from the other parts as
|
||||||
@@ -392,49 +347,36 @@ func newMetrics(
|
|||||||
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
||||||
remote *remotelog.Sender,
|
remote *remotelog.Sender,
|
||||||
) *metrics.Metrics {
|
) *metrics.Metrics {
|
||||||
params := metrics.Params{
|
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
||||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
m.AddBansAndClients(ledger, limiter, utcNow)
|
||||||
InstanceName: cfg.InstanceName,
|
m.AddRules(ruleFiles)
|
||||||
Now: utcNow,
|
m.AddReputation(lists, dnsbl)
|
||||||
Bans: ledger,
|
|
||||||
Clients: limiter,
|
if cfg.AbuseIPDBKey != "" {
|
||||||
Rules: ruleFiles,
|
m.AddAbuseIPDB(abuseIPDB)
|
||||||
Lists: lists,
|
|
||||||
DNSBL: dnsbl,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// A nil pointer in an interface field is not a nil interface, so a part
|
m.AddAlerts(alertQueue)
|
||||||
// that is off is left out rather than set to nil.
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
|
||||||
params.AbuseIPDB = abuseIPDB
|
|
||||||
}
|
|
||||||
|
|
||||||
if lookupFile != nil {
|
if lookupFile != nil {
|
||||||
params.LookupFile = lookupFile
|
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
||||||
}
|
}
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
params.RemoteLog = remote
|
m.AddRemoteLog(remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics.New(params)
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
// failure is handled as waf.GeoJSFailed says.
|
||||||
// under way, as geojs.GeoJS.StopAsking says.
|
|
||||||
func newGeoJS(
|
func newGeoJS(
|
||||||
lc fx.Lifecycle,
|
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
||||||
cfg *config.WAF,
|
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
||||||
m *metrics.Metrics,
|
|
||||||
limiter *ratelimit.Limiter,
|
|
||||||
ledger *bans.Ledger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) *geojs.GeoJS {
|
) *geojs.GeoJS {
|
||||||
geoJS := geojs.New(geojs.Params{
|
return geojs.New(geojs.Params{
|
||||||
URL: geojs.URL,
|
URL: geojs.URL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
@@ -446,10 +388,6 @@ func newGeoJS(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
|
||||||
|
|
||||||
return geoJS
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWAFServer returns the server waf runs, with its parts and the time
|
// newWAFServer returns the server waf runs, with its parts and the time
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
if rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
|||||||
"",
|
"",
|
||||||
map[string]string{metricsToken: token},
|
map[string]string{metricsToken: token},
|
||||||
)
|
)
|
||||||
server.Ledger.Ban(
|
server.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix(otherClient+"/32"),
|
netip.MustParsePrefix(otherClient+"/32"),
|
||||||
clk.Now(),
|
clk.Now(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
before := server.Ledger.Snapshot()
|
before := server.Ledger.Snapshot()
|
||||||
|
|
||||||
|
|||||||
@@ -130,9 +130,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
|
|||||||
// A ban for a clear sign of attack, which a request under it would make
|
// A ban for a clear sign of attack, which a request under it would make
|
||||||
// permanent.
|
// permanent.
|
||||||
group := netip.MustParsePrefix(ipv6Group)
|
group := netip.MustParsePrefix(ipv6Group)
|
||||||
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
|
attackBan, _ := server.Ledger.BanForAttack(
|
||||||
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
|
group,
|
||||||
})
|
start,
|
||||||
|
bans.Notes{RuleID: "probe"},
|
||||||
|
)
|
||||||
|
|
||||||
// The third request breaks the limit, and so does the fourth, within the
|
// The third request breaks the limit, and so does the fourth, within the
|
||||||
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
||||||
|
|||||||
@@ -686,10 +686,10 @@ func askDirectly(
|
|||||||
|
|
||||||
// banClient bans client at now.
|
// banClient bans client at now.
|
||||||
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||||
server.Ledger.Ban(
|
server.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix(client+"/32"),
|
netip.MustParsePrefix(client+"/32"),
|
||||||
now,
|
now,
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+57
-77
@@ -86,14 +86,13 @@ func (rq *request) countedBytes() int64 {
|
|||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes
|
||||||
|
|
||||||
switch rq.h.config.BytesCounted {
|
switch rq.h.config.BytesCounted {
|
||||||
case config.BytesCountedResponse:
|
case "response":
|
||||||
return response
|
return response
|
||||||
case config.BytesCountedRequest:
|
case "request":
|
||||||
return request
|
return request
|
||||||
case config.BytesCountedBoth:
|
default: // both
|
||||||
|
return response + request
|
||||||
}
|
}
|
||||||
|
|
||||||
return response + request
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||||
@@ -103,8 +102,9 @@ func (rq *request) countedBytes() int64 {
|
|||||||
// one whose bytes broke a byte limit, the refusal for one that broke the
|
// one whose bytes broke a byte limit, the refusal for one that broke the
|
||||||
// refusal limit. The ban's notes give the client's limit percentage for a
|
// refusal limit. The ban's notes give the client's limit percentage for a
|
||||||
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
||||||
// the client's counters back to zero. In observe mode, banClient says what
|
// the client's counters back to zero. In observe mode it makes no ban and
|
||||||
// happens instead.
|
// sets nothing back, and raises the alert for the ban it would have made,
|
||||||
|
// if that alert would be sent.
|
||||||
func (rq *request) banForLimit(
|
func (rq *request) banForLimit(
|
||||||
now time.Time,
|
now time.Time,
|
||||||
hit ratelimit.Hit,
|
hit ratelimit.Hit,
|
||||||
@@ -122,11 +122,23 @@ func (rq *request) banForLimit(
|
|||||||
rq.line.Offence = requestlog.OffenceLimit
|
rq.line.Offence = requestlog.OffenceLimit
|
||||||
rq.h.metrics.LimitHit(hit)
|
rq.h.metrics.LimitHit(hit)
|
||||||
|
|
||||||
|
netblock := banNetblock(rq.h.config, rq.client)
|
||||||
|
if rq.h.config.Observe &&
|
||||||
|
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
notes := bans.Notes{
|
notes := bans.Notes{
|
||||||
Kind: hit.Kind,
|
ASN: rq.line.ASN,
|
||||||
Limit: hit.Limit,
|
ASName: rq.line.ASName,
|
||||||
Window: hit.Window,
|
Country: rq.line.Country,
|
||||||
Count: hit.Count,
|
Kind: hit.Kind,
|
||||||
|
Limit: hit.Limit,
|
||||||
|
Window: hit.Window,
|
||||||
|
Count: hit.Count,
|
||||||
|
Reputation: rq.reputation,
|
||||||
|
Request: rq.noted(now, status),
|
||||||
|
Requests: rq.netblockRequests(netblock),
|
||||||
}
|
}
|
||||||
|
|
||||||
switch hit.Kind {
|
switch hit.Kind {
|
||||||
@@ -136,54 +148,47 @@ func (rq *request) banForLimit(
|
|||||||
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banClient(
|
if rq.h.config.Observe {
|
||||||
now,
|
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
|
||||||
status,
|
if wouldBan {
|
||||||
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
|
rq.alertBan(ban)
|
||||||
)
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
|
||||||
|
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
||||||
|
rq.line.BanExpires = banExpires(ban)
|
||||||
|
|
||||||
|
if made {
|
||||||
|
rq.alertBan(ban)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForAttack bans the client's netblock at now for a clear sign of
|
// banForAttack bans the client's netblock at now for a clear sign of
|
||||||
// attack, which notes name: the ban rule that matched, or the trap path
|
// attack, which notes name: the ban rule that matched, or the trap path
|
||||||
// asked for. It returns the action for that ban. In observe mode, banClient
|
// asked for. It fills in the rest of the notes. In observe mode it makes
|
||||||
// says what happens instead.
|
// no ban, and raises the alert for the ban it would have made, if that
|
||||||
|
// alert would be sent. It returns the action for that ban.
|
||||||
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
||||||
return rq.banClient(
|
|
||||||
now,
|
|
||||||
rq.h.config.BanResponse,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// banClient bans the client's netblock at now on grounds, and returns the
|
|
||||||
// action for that ban. It adds to the notes what the request tells of the
|
|
||||||
// client, with status, what the client was sent. A ban for a broken limit
|
|
||||||
// sets the client's counters back to zero.
|
|
||||||
//
|
|
||||||
// In observe mode it makes no ban and sets nothing back. It raises the
|
|
||||||
// alert for the ban it would have made, if that alert would be sent.
|
|
||||||
func (rq *request) banClient(
|
|
||||||
now time.Time,
|
|
||||||
status int,
|
|
||||||
grounds bans.Grounds,
|
|
||||||
) string {
|
|
||||||
netblock := banNetblock(rq.h.config, rq.client)
|
netblock := banNetblock(rq.h.config, rq.client)
|
||||||
|
if rq.h.config.Observe &&
|
||||||
|
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
||||||
|
return banAction(
|
||||||
|
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
notes.ASN = rq.line.ASN
|
||||||
|
notes.ASName = rq.line.ASName
|
||||||
|
notes.Country = rq.line.Country
|
||||||
|
notes.Reputation = rq.reputation
|
||||||
|
notes.Request = rq.noted(now, rq.h.config.BanResponse)
|
||||||
|
notes.Requests = rq.netblockRequests(netblock)
|
||||||
|
|
||||||
if rq.h.config.Observe {
|
if rq.h.config.Observe {
|
||||||
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
|
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
|
||||||
return banAction(
|
|
||||||
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
grounds.Notes = rq.withClientNotes(
|
|
||||||
grounds.Notes,
|
|
||||||
now,
|
|
||||||
status,
|
|
||||||
netblock,
|
|
||||||
)
|
|
||||||
|
|
||||||
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
|
|
||||||
if wouldBan {
|
if wouldBan {
|
||||||
rq.alertBan(ban)
|
rq.alertBan(ban)
|
||||||
}
|
}
|
||||||
@@ -191,13 +196,7 @@ func (rq *request) banClient(
|
|||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
|
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
|
||||||
|
|
||||||
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
|
|
||||||
if grounds.Cause == bans.CauseLimit {
|
|
||||||
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.line.BanExpires = banExpires(ban)
|
rq.line.BanExpires = banExpires(ban)
|
||||||
|
|
||||||
if made {
|
if made {
|
||||||
@@ -207,25 +206,6 @@ func (rq *request) banClient(
|
|||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
// withClientNotes is notes with what the request tells of the client
|
|
||||||
// added: its AS, country and reputation, the request at now with status,
|
|
||||||
// and how many requests netblock, the client's, has sent.
|
|
||||||
func (rq *request) withClientNotes(
|
|
||||||
notes bans.Notes,
|
|
||||||
now time.Time,
|
|
||||||
status int,
|
|
||||||
netblock netip.Prefix,
|
|
||||||
) bans.Notes {
|
|
||||||
notes.ASN = rq.line.ASN
|
|
||||||
notes.ASName = rq.line.ASName
|
|
||||||
notes.Country = rq.line.Country
|
|
||||||
notes.Reputation = rq.reputation
|
|
||||||
notes.Request = rq.noted(now, status)
|
|
||||||
notes.Requests = rq.netblockRequests(netblock)
|
|
||||||
|
|
||||||
return notes
|
|
||||||
}
|
|
||||||
|
|
||||||
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
||||||
// made at now would be sent. In observe mode the ban the request would
|
// made at now would be sent. In observe mode the ban the request would
|
||||||
// have made is worked out only then, at most once per
|
// have made is worked out only then, at most once per
|
||||||
|
|||||||
@@ -548,10 +548,6 @@ func (s *sender) requestWithHeader(
|
|||||||
// body is sent as it is after the headers, header holding its
|
// body is sent as it is after the headers, header holding its
|
||||||
// Content-Length or Transfer-Encoding. header may hold several lines,
|
// Content-Length or Transfer-Encoding. header may hold several lines,
|
||||||
// separated by "\r\n". It returns the whole answer.
|
// separated by "\r\n". It returns the whole answer.
|
||||||
//
|
|
||||||
// It waits for the answer as long as it takes, since a busy host can slow
|
|
||||||
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
|
|
||||||
// whose answer never comes.
|
|
||||||
func (s *sender) requestWithBody(
|
func (s *sender) requestWithBody(
|
||||||
method, from, path, header, body string, status int, action string,
|
method, from, path, header, body string, status int, action string,
|
||||||
) (logLine, answer) {
|
) (logLine, answer) {
|
||||||
@@ -570,6 +566,11 @@ func (s *sender) requestWithBody(
|
|||||||
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
||||||
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
||||||
|
|
||||||
|
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
|
||||||
|
if err != nil {
|
||||||
|
s.t.Fatalf("set read deadline: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
var got answer
|
var got answer
|
||||||
|
|
||||||
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
||||||
|
|||||||
+9
-10
@@ -53,34 +53,33 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
unknown = percentage{
|
unknown = percentage{
|
||||||
cfg.UnknownCountryLimitPercent,
|
cfg.UnknownCountryLimitPercent,
|
||||||
config.UnknownCountryLimitPercentSetting,
|
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched := percentage{percent: whole}
|
fetched := percentage{percent: whole}
|
||||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||||
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
if rq.blocklisted &&
|
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
config.BlocklistActionSetting,
|
"SWW_WAF_BLOCKLIST_ACTION",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
cfg.DNSBLAbuseIPDBAction == "limit" {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
config.DNSBLAbuseIPDBActionSetting,
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const tagSetting = config.TagLimitPercentSetting
|
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||||
|
|
||||||
asnRequests := lowest(
|
asnRequests := lowest(
|
||||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||||
@@ -108,7 +107,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
asnBytes = given(
|
asnBytes = given(
|
||||||
cfg.ASNBytesLimitPercent,
|
cfg.ASNBytesLimitPercent,
|
||||||
asn,
|
asn,
|
||||||
config.ASNBytesLimitPercentSetting,
|
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,7 +115,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
countryBytes = given(
|
countryBytes = given(
|
||||||
cfg.CountryBytesLimitPercent,
|
cfg.CountryBytesLimitPercent,
|
||||||
country,
|
country,
|
||||||
config.CountryBytesLimitPercentSetting,
|
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -42,6 +41,5 @@ func (blocklists) check(
|
|||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
||||||
config.ReputationActionDeny
|
|
||||||
}
|
}
|
||||||
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
|
|||||||
rq.line.CoreRuleSetScore = &result.Score
|
rq.line.CoreRuleSetScore = &result.Score
|
||||||
|
|
||||||
for _, id := range result.RuleIDs {
|
for _, id := range result.RuleIDs {
|
||||||
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
threshold := cfg.CoreRuleSetScoreThreshold
|
||||||
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
||||||
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
detail["mode"] = config.CoreRuleSetModeDetect
|
||||||
case rq.h.config.Observe:
|
case rq.h.config.Observe:
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,8 +33,7 @@ func (countries) check(
|
|||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionCountryDenied,
|
action: requestlog.ActionCountryDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
"country": country,
|
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||||
"setting": config.ExclusivelyAllowedCountriesSetting,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed ||
|
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -104,6 +104,12 @@ func onReport() []heuristic {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
||||||
|
// it is called.
|
||||||
|
func (rq *request) runAfterAnswer(ctx context.Context) {
|
||||||
|
rq.runBanningHeuristics(ctx, afterAnswer())
|
||||||
|
}
|
||||||
|
|
||||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||||
// and returns the first verdict, or nil when none gives one. The log line's
|
// and returns the first verdict, or nil when none gives one. The log line's
|
||||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||||
|
|||||||
@@ -49,8 +49,7 @@ func (rq *request) startLookUp(
|
|||||||
) <-chan lookupResult {
|
) <-chan lookupResult {
|
||||||
result := make(chan lookupResult, 1)
|
result := make(chan lookupResult, 1)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceOff ||
|
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
||||||
!canBePlaced(rq.client) {
|
|
||||||
result <- lookupResult{}
|
result <- lookupResult{}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@@ -58,7 +57,7 @@ func (rq *request) startLookUp(
|
|||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(rq.h.config, rq.client)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == "file" {
|
||||||
result <- lookupResult{
|
result <- lookupResult{
|
||||||
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
||||||
}
|
}
|
||||||
|
|||||||
+37
-2
@@ -6,6 +6,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -82,8 +83,8 @@ type Params struct {
|
|||||||
// LookupFile is the lookup database they are looked up in while
|
// LookupFile is the lookup database they are looked up in while
|
||||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||||
LookupFile *lookup.File
|
LookupFile *lookup.File
|
||||||
// CoreRuleSet is what inspects requests while
|
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||||
CoreRuleSet CoreRuleSetInspector
|
CoreRuleSet CoreRuleSetInspector
|
||||||
// Now tells the time by which requests are counted for the rate
|
// Now tells the time by which requests are counted for the rate
|
||||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||||
@@ -173,6 +174,40 @@ func newHandler(params Params) *handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||||
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
||||||
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
||||||
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
||||||
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
||||||
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
||||||
|
// set file is loaded with the Core Rule Set only to check it.
|
||||||
|
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
||||||
|
cfg.CoreRuleSetExclusionsFile == "" {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
||||||
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||||
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||||
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||||
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||||
|
Apps: cfg.Apps,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
return coreRuleSet, nil
|
||||||
|
}
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
// belongs to one request is in a request.
|
// belongs to one request is in a request.
|
||||||
type handler struct {
|
type handler struct {
|
||||||
|
|||||||
+57
-85
@@ -20,11 +20,9 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -194,11 +192,11 @@ func (o *output) requestLine(t *testing.T) logLine {
|
|||||||
// endedLines waits for count requests that went through rproxy, or to waf's
|
// endedLines waits for count requests that went through rproxy, or to waf's
|
||||||
// own endpoints, to have ended, and returns the last log line of each, in the
|
// own endpoints, to have ended, and returns the last log line of each, in the
|
||||||
// order waf took them. A question's last line is the one its report writes.
|
// order waf took them. A question's last line is the one its report writes.
|
||||||
// It waits as long as it takes, as requestWithBody does.
|
|
||||||
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
for {
|
deadline := time.Now().Add(waitLimit)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
found := o.lastLines(t)
|
found := o.lastLines(t)
|
||||||
|
|
||||||
ended := len(found) >= count
|
ended := len(found) >= count
|
||||||
@@ -214,6 +212,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
|
|||||||
|
|
||||||
time.Sleep(pollInterval)
|
time.Sleep(pollInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastLines returns the last request log line written so far of each request,
|
// lastLines returns the last request log line written so far of each request,
|
||||||
@@ -395,11 +397,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "rproxy",
|
"rproxy",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -468,11 +470,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: out,
|
out,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -502,20 +504,24 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Core Rule Set: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
params := waf.Params{
|
params := waf.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: out,
|
RequestLog: out,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
LookupFile: lookupFile,
|
LookupFile: lookupFile,
|
||||||
Now: now,
|
CoreRuleSet: coreRuleSet,
|
||||||
Rules: ruleFiles,
|
Now: now,
|
||||||
Alerts: alertQueue,
|
Rules: ruleFiles,
|
||||||
|
Alerts: alertQueue,
|
||||||
// The tests of the limit set it back, and hold the Core Rule Set
|
// The tests of the limit set it back, and hold the Core Rule Set
|
||||||
// with a stand-in.
|
// with a stand-in.
|
||||||
BesideLookupTimeLimit: time.Hour,
|
BesideLookupTimeLimit: time.Hour,
|
||||||
}
|
}
|
||||||
|
|
||||||
addCoreRuleSet(t, ¶ms)
|
|
||||||
addCounters(¶ms)
|
addCounters(¶ms)
|
||||||
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||||
|
|
||||||
@@ -524,46 +530,24 @@ func newWAFWith(
|
|||||||
return server, out, alertQueue
|
return server, out, alertQueue
|
||||||
}
|
}
|
||||||
|
|
||||||
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
|
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||||
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
|
// and the anomaly counters, as sww builds them from params' settings. The
|
||||||
func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
// metrics read the ledger, the table of clients and the rules.
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cfg := params.Config
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Core Rule Set: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
params.CoreRuleSet = coreRuleSet
|
|
||||||
}
|
|
||||||
|
|
||||||
// addCounters adds to params the table of clients, the ledger and the
|
|
||||||
// anomaly counters, as sww builds them from params' settings.
|
|
||||||
func addCounters(params *waf.Params) {
|
func addCounters(params *waf.Params) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
|
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Metrics = metrics.New(
|
||||||
Limits: ratelimit.Limits{
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
cfg.InstanceName,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
)
|
||||||
PerDay: cfg.RateLimitPerDay,
|
params.Limiter = ratelimit.New(ratelimit.Limits{
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
},
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
})
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
}, cfg.MaxTrackedClients)
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
@@ -582,18 +566,26 @@ func addCounters(params *waf.Params) {
|
|||||||
NamedNetblocks: cfg.AnomalyWatchedNets,
|
NamedNetblocks: cfg.AnomalyWatchedNets,
|
||||||
Alerts: params.Alerts,
|
Alerts: params.Alerts,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
params.Metrics.AddBansAndClients(
|
||||||
|
params.Ledger,
|
||||||
|
params.Limiter,
|
||||||
|
params.Now,
|
||||||
|
)
|
||||||
|
params.Metrics.AddRules(params.Rules)
|
||||||
}
|
}
|
||||||
|
|
||||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||||
// setting is set, the metrics newMetrics returns, and GeoJS at geojsURL, as
|
// setting is set, and GeoJS at geojsURL, as sww builds
|
||||||
// sww builds them from params' settings and the parts addCounters added.
|
// them from params' settings and the parts addCounters added. The metrics
|
||||||
|
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||||
func addOutsideSources(
|
func addOutsideSources(
|
||||||
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
|
||||||
) {
|
) {
|
||||||
cfg := params.Config
|
cfg := params.Config
|
||||||
limiter, ledger := params.Limiter, params.Ledger
|
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
|
||||||
processLog, alertQueue := params.ProcessLog, params.Alerts
|
processLog, alertQueue := params.ProcessLog, params.Alerts
|
||||||
|
|
||||||
params.Lists = reputation.New(reputation.Params{
|
params.Lists = reputation.New(reputation.Params{
|
||||||
@@ -642,9 +634,6 @@ func addOutsideSources(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
m := newMetrics(params)
|
|
||||||
params.Metrics = m
|
|
||||||
|
|
||||||
params.GeoJS = geojs.New(geojs.Params{
|
params.GeoJS = geojs.New(geojs.Params{
|
||||||
URL: geojsURL,
|
URL: geojsURL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
@@ -657,29 +646,12 @@ func addOutsideSources(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
|
||||||
// newMetrics returns the metrics, as sww builds them from params' settings,
|
m.AddReputation(params.Lists, params.DNSBL)
|
||||||
// read from the ledger, the table of clients, the rules, the lists, the
|
|
||||||
// zones' verdicts, and AbuseIPDB's while its key is set.
|
|
||||||
func newMetrics(params *waf.Params) *metrics.Metrics {
|
|
||||||
cfg := params.Config
|
|
||||||
metricsParams := metrics.Params{
|
|
||||||
BusiestCount: cfg.MetricsBusiestCountriesAndASNs,
|
|
||||||
InstanceName: cfg.InstanceName,
|
|
||||||
Now: params.Now,
|
|
||||||
Bans: params.Ledger,
|
|
||||||
Clients: params.Limiter,
|
|
||||||
Rules: params.Rules,
|
|
||||||
Lists: params.Lists,
|
|
||||||
DNSBL: params.DNSBL,
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
if cfg.AbuseIPDBKey != "" {
|
||||||
metricsParams.AbuseIPDB = params.AbuseIPDB
|
m.AddAbuseIPDB(params.AbuseIPDB)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics.New(metricsParams)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newClient returns an HTTP client that sends requests as they are made,
|
// newClient returns an HTTP client that sends requests as they are made,
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
|
||||||
|
// refuses the requests of a client a source lists.
|
||||||
|
const deny = "deny"
|
||||||
|
|
||||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||||
// with reason.
|
// with reason.
|
||||||
|
|||||||
+1
-11
@@ -345,16 +345,6 @@ func timing(start, end time.Time) *float64 {
|
|||||||
return new(requestlog.Milliseconds(end.Sub(start)))
|
return new(requestlog.Milliseconds(end.Sub(start)))
|
||||||
}
|
}
|
||||||
|
|
||||||
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
|
|
||||||
// It then counts the request for the anomaly thresholds and adds it to its
|
|
||||||
// client's history. list runs first, so that a refusal limit it finds broken
|
|
||||||
// is in that history. ctx is the request's own context.
|
|
||||||
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
|
|
||||||
rq.runBanningHeuristics(ctx, list)
|
|
||||||
rq.countAnomalies()
|
|
||||||
rq.addToHistory()
|
|
||||||
}
|
|
||||||
|
|
||||||
// addToHistory adds the request, which has ended, to its client's
|
// addToHistory adds the request, which has ended, to its client's
|
||||||
// history, and counts its offences in the metrics, and then the lookup's
|
// history, and counts its offences in the metrics, and then the lookup's
|
||||||
// answer about the client, as answerAtTheEnd gives it, to that history and
|
// answer about the client, as answerAtTheEnd gives it, to that history and
|
||||||
@@ -431,7 +421,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
|||||||
return geojs.Answer{}, false
|
return geojs.Answer{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == "file" {
|
||||||
return rq.lookupAnswer, true
|
return rq.lookupAnswer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+28
-21
@@ -177,13 +177,11 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
peer.Unmap(),
|
peer.Unmap(),
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
// These run even when a check panics. finish runs first, so that what is
|
// The last deferred runs first, so that what is held has the finished log
|
||||||
// held has the finished log line. Go's server sends the answer only once
|
// line. Go's server sends the answer only once this function has
|
||||||
// this function has returned, so the report cannot come before it is held.
|
// returned, so the report cannot come before it is held.
|
||||||
defer func() {
|
defer rq.holdForReport()
|
||||||
rq.finish()
|
defer rq.finish()
|
||||||
rq.holdForReport()
|
|
||||||
}()
|
|
||||||
|
|
||||||
rq.site = r.Host
|
rq.site = r.Host
|
||||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||||
@@ -192,28 +190,35 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|||||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
||||||
|
|
||||||
if refused == nil && !underOwnPrefix {
|
if refused == nil && !underOwnPrefix {
|
||||||
rq.allow()
|
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
// In observe mode, what enforce mode would have refused counts here.
|
||||||
rq.runBanningHeuristics(r.Context(), afterAnswer())
|
defer rq.runAfterAnswer(r.Context())
|
||||||
|
|
||||||
|
rq.allow()
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
||||||
|
// a broken refusal limit is in the client's history.
|
||||||
|
defer rq.addToHistory()
|
||||||
|
defer rq.countAnomalies()
|
||||||
|
defer rq.runAfterAnswer(r.Context())
|
||||||
|
|
||||||
if refused != nil {
|
if refused != nil {
|
||||||
rq.deny(*refused)
|
rq.deny(*refused)
|
||||||
} else {
|
|
||||||
// The app never sees this prefix. waf's own endpoints are at its
|
return
|
||||||
// own address, so this answers as a path with no endpoint does.
|
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
http.StatusText(http.StatusNotFound),
|
|
||||||
http.StatusNotFound,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banAndCount(r.Context(), afterAnswer())
|
// The app never sees this prefix. waf's own endpoints are at its
|
||||||
|
// own address, so this answers as a path with no endpoint does.
|
||||||
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
|
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||||
|
http.Error(
|
||||||
|
rq.out,
|
||||||
|
http.StatusText(http.StatusNotFound),
|
||||||
|
http.StatusNotFound,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readQuestion returns the visitor's address and the body a question carries,
|
// readQuestion returns the visitor's address and the body a question carries,
|
||||||
@@ -486,7 +491,9 @@ func (h *handler) finishWith(
|
|||||||
if rq.allowed {
|
if rq.allowed {
|
||||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||||
|
|
||||||
rq.banAndCount(ctx, onReport())
|
rq.runBanningHeuristics(ctx, onReport())
|
||||||
|
rq.countAnomalies()
|
||||||
|
rq.addToHistory()
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(
|
rq.h.metrics.RequestReported(
|
||||||
&rq.line,
|
&rq.line,
|
||||||
|
|||||||
Reference in new issue
Block a user