Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
249d6a1142 | ||
|
|
0a0850f797 | ||
|
|
56a57125f1 | ||
|
|
9a8ed745bd | ||
|
|
0a230c86ce | ||
|
|
40386dce6e | ||
|
|
433d5cc0a9 | ||
|
|
419ee1c441 | ||
|
|
aaa28f541b | ||
|
|
98d7432077 | ||
|
|
11ac8aee50 | ||
|
|
e7cc0adbdb |
No files matched your search
@@ -902,13 +902,17 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
- `internal/rproxy`: the reverse proxy in front of one app.
|
- `internal/rproxy`: the reverse proxy in front of one app.
|
||||||
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
||||||
endpoints.
|
endpoints.
|
||||||
|
- `internal/wafprotocol`: the question, the answer and the report that `rproxy`
|
||||||
|
and `waf` send each other.
|
||||||
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
- `internal/state`: the state and cache files.
|
||||||
`internal/metrics`: the state and cache files, the log lines, the syslog
|
- `internal/requestlog`: the request log.
|
||||||
sender and the metrics.
|
- `internal/processlog`: the process's own messages.
|
||||||
|
- `internal/remotelog`: the syslog sender.
|
||||||
|
- `internal/metrics`: the metrics.
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -73,13 +73,16 @@ never as a time.
|
|||||||
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||||
and stops their background work.
|
and stops their background work.
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
|
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
|
||||||
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
|
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
|
||||||
`answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes
|
and anomalies, and writes the log lines.
|
||||||
the log lines.
|
|
||||||
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
||||||
about each request. It passes a request to the app with the standard
|
about each request. It passes a request to the app with the standard
|
||||||
library's `httputil.ReverseProxy`, within the time and size limits.
|
library's `httputil.ReverseProxy`, within the time and size limits.
|
||||||
|
- `internal/wafprotocol`: the question, the answer and the report that
|
||||||
|
`rproxy` and `waf` send each other (see "`waf`"). It writes and reads each
|
||||||
|
one, and declares every header, path and limit both sides use.
|
||||||
|
`internal/rproxy` imports it, not `internal/waf`.
|
||||||
- `internal/ratelimit`: the table of clients. It counts requests, bytes and
|
- `internal/ratelimit`: the table of clients. It counts requests, bytes and
|
||||||
refusals, and keeps each client's history.
|
refusals, and keeps each client's history.
|
||||||
- `internal/bans`: the bans. It holds each netblock's bans with their notes,
|
- `internal/bans`: the bans. It holds each netblock's bans with their notes,
|
||||||
@@ -104,8 +107,10 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
||||||
sending them to the syslog server.
|
- `internal/processlog`: the process lines on stdout, which hold the
|
||||||
|
process's own messages.
|
||||||
|
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ require (
|
|||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
|
golang.org/x/net v0.60.0
|
||||||
golang.org/x/sys v0.48.0
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -50,7 +51,6 @@ require (
|
|||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||||
golang.org/x/net v0.60.0 // indirect
|
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
|
|||||||
@@ -54,7 +54,11 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
|
|||||||
now := midnight()
|
now := midnight()
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
scores := map[netip.Prefix]int64{netblock: 100}
|
||||||
|
|
||||||
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
limitBan, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
||||||
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
||||||
@@ -119,7 +123,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
|
|||||||
expires := now.Add(30 * day)
|
expires := now.Add(30 * day)
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
||||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
// listed is off the blacklist when it is fetched after the ban's end.
|
// listed is off the blacklist when it is fetched after the ban's end.
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
||||||
@@ -144,7 +148,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
||||||
}
|
}
|
||||||
|
|
||||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
||||||
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
||||||
@@ -154,7 +158,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
|
|
||||||
// The ban for an attack takes the place of the one for a limit, and no
|
// The ban for an attack takes the place of the one for a limit, and no
|
||||||
// blacklist ban goes.
|
// blacklist ban goes.
|
||||||
ledger.BanForAttack(attacker, now, bans.Notes{})
|
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
|
||||||
|
|
||||||
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
||||||
for netblock := range scores {
|
for netblock := range scores {
|
||||||
|
|||||||
+64
-23
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
||||||
|
|
||||||
// A new ban drops the ban sww made, and only that one.
|
// A new ban drops the ban sww made, and only that one.
|
||||||
ledger.BanForLimit(second, midnight(), bans.Notes{})
|
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// And the next drops that one.
|
// And the next drops that one.
|
||||||
ledger.BanForLimit(third, midnight(), bans.Notes{})
|
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -98,20 +98,28 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
|
|||||||
// twice, each time after its last ban has ended.
|
// twice, each time after its last ban has ended.
|
||||||
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
||||||
"probes for logins")
|
"probes for logins")
|
||||||
first, _ := ledger.BanForLimit(
|
first, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
|
||||||
|
|
||||||
// A ban on another netblock fills the ledger, and the netblock's first
|
// A ban on another netblock fills the ledger, and the netblock's first
|
||||||
// ban for a limit, its earliest that is not an admin's, goes.
|
// ban for a limit, its earliest that is not an admin's, goes.
|
||||||
ledger.BanForLimit(other, second.Start, bans.Notes{})
|
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
||||||
|
|
||||||
// The netblock's next ban still counts it.
|
// The netblock's next ban still counts it.
|
||||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
third, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
second.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -127,15 +135,27 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
|
|||||||
// The netblock breaks a limit twice, the second time after its first
|
// The netblock breaks a limit twice, the second time after its first
|
||||||
// ban has ended, and the second ban's notes count the first. An admin
|
// ban has ended, and the second ban's notes count the first. An admin
|
||||||
// then keeps the first ban.
|
// then keeps the first ban.
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
kept := first
|
kept := first
|
||||||
kept.Cause = bans.CauseAdmin
|
kept.Cause = bans.CauseAdmin
|
||||||
ledger.LoadEdit([]bans.Ban{kept, second})
|
ledger.LoadEdit([]bans.Ban{kept, second})
|
||||||
|
|
||||||
// The netblock's next ban counts the kept ban once, as an admin's.
|
// The netblock's next ban counts the kept ban once, as an admin's.
|
||||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
third, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
second.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -147,20 +167,37 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
limit, _ := ledger.BanForLimit(
|
limit, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{
|
||||||
|
Kind: "requests",
|
||||||
|
Limit: 1000,
|
||||||
|
Window: "minute",
|
||||||
|
},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
byteLimit, _ := ledger.BanForLimit(
|
byteLimit, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.3/32"),
|
netip.MustParsePrefix("203.0.113.3/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{
|
||||||
|
Kind: "bytes",
|
||||||
|
Limit: 10 << 30,
|
||||||
|
Window: "hour",
|
||||||
|
},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
attack, _ := ledger.BanForAttack(
|
attack, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.2/32"),
|
netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{RuleID: "git-dir", Target: "path"},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseAttack,
|
||||||
|
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
for _, tc := range []struct{ got, want string }{
|
for _, tc := range []struct{ got, want string }{
|
||||||
@@ -200,7 +237,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
|
|||||||
t.Error("the lifted ban refuses")
|
t.Error("the lifted ban refuses")
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -251,7 +288,11 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The next clear sign of attack bans for seven days, as a first does.
|
// The next clear sign of attack bans for seven days, as a first does.
|
||||||
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != 7*day {
|
if ban.Expires.Sub(ban.Start) != 7*day {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban for an attack ends at %s, want seven days on",
|
"the next ban for an attack ends at %s, want seven days on",
|
||||||
@@ -264,10 +305,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
made, _ := ledger.BanForLimit(
|
made, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
atStart := bans.Ban{
|
atStart := bans.Ban{
|
||||||
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
@@ -311,7 +352,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// An hour's ban for a broken limit.
|
// An hour's ban for a broken limit.
|
||||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A minute later an admin bans the netblock for good, named by an
|
// A minute later an admin bans the netblock for good, named by an
|
||||||
@@ -431,10 +472,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// The temporary ban is made first, so that its netblock is looked at first.
|
// The temporary ban is made first, so that its netblock is looked at first.
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
ledger.BanForAdmin(
|
ledger.BanForAdmin(
|
||||||
netip.MustParsePrefix("203.0.113.0/24"),
|
netip.MustParsePrefix("203.0.113.0/24"),
|
||||||
|
|||||||
+54
-82
@@ -300,81 +300,50 @@ func activeBan(bans []Ban, now time.Time) *Ban {
|
|||||||
return newest
|
return newest
|
||||||
}
|
}
|
||||||
|
|
||||||
// BanForLimit bans netblock at now for a broken limit, with notes, and
|
// Grounds are why a request makes a ban: its cause, CauseLimit or
|
||||||
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
|
// CauseAttack, and its notes.
|
||||||
// made within LimitBanRepeatWindow after the netblock's ban that ended
|
type Grounds struct {
|
||||||
// last, other than one for a clear sign of attack or a lifted one, lasts
|
Cause string
|
||||||
// repeatFactor times as long as that one. A ban that would be longer
|
Notes Notes
|
||||||
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
|
|
||||||
// active, as when two of its requests break a limit at once, that ban is
|
|
||||||
// returned with false, and no other is made. The ledger fills in the
|
|
||||||
// notes' Refused and EarlierBans itself, and gives the ban the reason
|
|
||||||
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
|
|
||||||
// as "requests per minute over the limit of 1000".
|
|
||||||
func (l *Ledger) BanForLimit(
|
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
|
||||||
) (Ban, bool) {
|
|
||||||
return l.ban(
|
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseLimit,
|
|
||||||
limitReason(notes),
|
|
||||||
notes,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBanForLimit returns what BanForLimit would, without making the ban:
|
// Ban bans netblock at now on grounds, and returns the ban, and true. If
|
||||||
// what observe mode would have done.
|
// a ban on netblock is still active, as when two of its requests break a
|
||||||
func (l *Ledger) WouldBanForLimit(
|
// limit at once, that ban is returned with false, and no other is made.
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
// The ledger fills in the notes' Refused and EarlierBans itself.
|
||||||
|
//
|
||||||
|
// A ban for a broken limit:
|
||||||
|
// - lasts LimitBanDuration the first time;
|
||||||
|
// - lasts repeatFactor times as long as the netblock's ban that ended
|
||||||
|
// last, when made within LimitBanRepeatWindow after it, unless that
|
||||||
|
// ban was for a clear sign of attack or was lifted;
|
||||||
|
// - is permanent instead when it would be longer than
|
||||||
|
// LimitBanPermanentAbove;
|
||||||
|
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
|
||||||
|
// the notes, such as "requests per minute over the limit of 1000".
|
||||||
|
//
|
||||||
|
// A ban for a clear sign of attack:
|
||||||
|
// - lasts AttackBanDuration the first time;
|
||||||
|
// - is permanent once the netblock has had one that was not lifted;
|
||||||
|
// - has the reason "matched the rule <RuleID>", or "asked for the trap
|
||||||
|
// path <TrapPath>".
|
||||||
|
func (l *Ledger) Ban(
|
||||||
|
netblock netip.Prefix, now time.Time, grounds Grounds,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(
|
return l.ban(netblock, now, grounds, true)
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseLimit,
|
|
||||||
limitReason(notes),
|
|
||||||
notes,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BanForAttack bans netblock at now for a clear sign of attack, with
|
// WouldBan returns what Ban would, without making the ban: what observe
|
||||||
// notes, and returns the ban, and whether it made it, as BanForLimit
|
// mode would have done.
|
||||||
// does. A first ban lasts AttackBanDuration; once the netblock has had
|
func (l *Ledger) WouldBan(
|
||||||
// one that was not lifted, the next is permanent. Its reason is "matched
|
netblock netip.Prefix, now time.Time, grounds Grounds,
|
||||||
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
|
|
||||||
func (l *Ledger) BanForAttack(
|
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(
|
return l.ban(netblock, now, grounds, false)
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseAttack,
|
|
||||||
attackReason(notes),
|
|
||||||
notes,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WouldBanForAttack returns what BanForAttack would, without making the
|
|
||||||
// ban: what observe mode would have done.
|
|
||||||
func (l *Ledger) WouldBanForAttack(
|
|
||||||
netblock netip.Prefix, now time.Time, notes Notes,
|
|
||||||
) (Ban, bool) {
|
|
||||||
return l.ban(
|
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
CauseAttack,
|
|
||||||
attackReason(notes),
|
|
||||||
notes,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
||||||
// or CauseAttack, made at now would be permanent, as BanForLimit or
|
// or CauseAttack, made at now would be permanent, as Ban would make it. It
|
||||||
// BanForAttack would make it. It works out nothing else of the ban.
|
// works out nothing else of the ban.
|
||||||
func (l *Ledger) WouldBePermanent(
|
func (l *Ledger) WouldBePermanent(
|
||||||
netblock netip.Prefix, now time.Time, cause string,
|
netblock netip.Prefix, now time.Time, cause string,
|
||||||
) bool {
|
) bool {
|
||||||
@@ -411,10 +380,9 @@ func attackReason(notes Notes) string {
|
|||||||
|
|
||||||
// BanForAdmin bans netblock at now for an admin, with reason, until
|
// BanForAdmin bans netblock at now for an admin, with reason, until
|
||||||
// expires, or for good when expires is zero, and returns the ban, whose
|
// expires, or for good when expires is zero, and returns the ban, whose
|
||||||
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
|
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
|
||||||
// ban even while another on netblock is active, since the admin asked
|
// on netblock is active, since the admin asked for this one. The ledger
|
||||||
// for this one. The ledger fills in the notes' EarlierBans, and counts
|
// fills in the notes' EarlierBans, and counts the ban among those made.
|
||||||
// the ban among those made.
|
|
||||||
func (l *Ledger) BanForAdmin(
|
func (l *Ledger) BanForAdmin(
|
||||||
netblock netip.Prefix, now, expires time.Time, reason string,
|
netblock netip.Prefix, now, expires time.Time, reason string,
|
||||||
) Ban {
|
) Ban {
|
||||||
@@ -759,15 +727,13 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
|
|||||||
return forgot
|
return forgot
|
||||||
}
|
}
|
||||||
|
|
||||||
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
|
// ban bans netblock at now on grounds, as Ban describes, and returns the
|
||||||
// reason and notes, as BanForLimit and BanForAttack describe, and returns
|
// ban, and whether it made it. Unless keep is true, the ban is not made,
|
||||||
// the ban, and whether it made it. Unless keep is true, the ban is not
|
// only returned: it is the ban that would have been made.
|
||||||
// made, only returned: it is the ban that would have been made.
|
|
||||||
func (l *Ledger) ban(
|
func (l *Ledger) ban(
|
||||||
netblock netip.Prefix,
|
netblock netip.Prefix,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
cause, reason string,
|
grounds Grounds,
|
||||||
notes Notes,
|
|
||||||
keep bool,
|
keep bool,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
@@ -776,6 +742,8 @@ func (l *Ledger) ban(
|
|||||||
// held are the netblock's bans, none of them active.
|
// held are the netblock's bans, none of them active.
|
||||||
var held []Ban
|
var held []Ban
|
||||||
|
|
||||||
|
notes := grounds.Notes
|
||||||
|
|
||||||
bans, found := l.netblocks.Get(netblock)
|
bans, found := l.netblocks.Get(netblock)
|
||||||
if found {
|
if found {
|
||||||
active := activeBan(*bans, now)
|
active := activeBan(*bans, now)
|
||||||
@@ -791,15 +759,19 @@ func (l *Ledger) ban(
|
|||||||
ban := Ban{
|
ban := Ban{
|
||||||
Netblock: netblock,
|
Netblock: netblock,
|
||||||
Start: now,
|
Start: now,
|
||||||
Cause: cause,
|
Cause: grounds.Cause,
|
||||||
Reason: reason,
|
|
||||||
Notes: notes,
|
Notes: notes,
|
||||||
}
|
}
|
||||||
|
|
||||||
if cause == CauseAttack {
|
switch grounds.Cause {
|
||||||
ban.Expires = l.attackExpiry(held, now)
|
case CauseLimit:
|
||||||
} else {
|
ban.Reason = limitReason(notes)
|
||||||
ban.Expires = l.limitExpiry(held, now)
|
ban.Expires = l.limitExpiry(held, now)
|
||||||
|
case CauseAttack:
|
||||||
|
ban.Reason = attackReason(notes)
|
||||||
|
ban.Expires = l.attackExpiry(held, now)
|
||||||
|
default:
|
||||||
|
panic("bans: a request's ban has the cause " + grounds.Cause)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !keep {
|
if !keep {
|
||||||
@@ -807,7 +779,7 @@ func (l *Ledger) ban(
|
|||||||
}
|
}
|
||||||
|
|
||||||
l.add(ban)
|
l.add(ban)
|
||||||
l.made[cause]++
|
l.made[grounds.Cause]++
|
||||||
l.markChanged()
|
l.markChanged()
|
||||||
|
|
||||||
return ban, true
|
return ban, true
|
||||||
|
|||||||
+121
-51
@@ -22,7 +22,11 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
||||||
// 81 hours.
|
// 81 hours.
|
||||||
for i, hours := range []int{1, 3, 9, 27, 81} {
|
for i, hours := range []int{1, 3, 9, 27, 81} {
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
length := time.Duration(hours) * time.Hour
|
length := time.Duration(hours) * time.Hour
|
||||||
if !ban.Expires.Equal(now.Add(length)) ||
|
if !ban.Expires.Equal(now.Add(length)) ||
|
||||||
@@ -42,7 +46,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
|
|
||||||
// The sixth would last 243 hours, more than seven days: it is
|
// The sixth would last 243 hours, more than seven days: it is
|
||||||
// permanent, and never ends.
|
// permanent, and never ends.
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -71,15 +75,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(
|
first, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
second, _ := ledger.BanForLimit(
|
second, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires.Add(tc.gap),
|
first.Expires.Add(tc.gap),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != tc.want ||
|
if second.Expires.Sub(second.Start) != tc.want ||
|
||||||
@@ -104,10 +108,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
|
|||||||
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
||||||
@@ -127,7 +131,11 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
for i := range 14 {
|
for i := range 14 {
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if !ban.Expires.After(ban.Start) {
|
if !ban.Expires.After(ban.Start) {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"ban %d starts at %s and ends at %s",
|
"ban %d starts at %s and ends at %s",
|
||||||
@@ -140,7 +148,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -152,15 +160,19 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, made := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if !made {
|
if !made {
|
||||||
t.Error("the first ban was not made")
|
t.Error("the first ban was not made")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made := ledger.BanForLimit(
|
again, made := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
if made || !reflect.DeepEqual(again, first) ||
|
if made || !reflect.DeepEqual(again, first) ||
|
||||||
@@ -175,10 +187,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made = ledger.BanForAttack(
|
again, made = ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
)
|
)
|
||||||
if made || !reflect.DeepEqual(again, first) {
|
if made || !reflect.DeepEqual(again, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -195,10 +207,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Requests: 5},
|
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
||||||
)
|
)
|
||||||
|
|
||||||
for range 3 {
|
for range 3 {
|
||||||
@@ -237,10 +249,10 @@ func TestFindCountsNothing(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Requests: 5},
|
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
||||||
)
|
)
|
||||||
|
|
||||||
got, banned, _ := ledger.Find(
|
got, banned, _ := ledger.Find(
|
||||||
@@ -287,24 +299,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
|
|||||||
d := netip.MustParsePrefix("2001:db8::/64")
|
d := netip.MustParsePrefix("2001:db8::/64")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
|
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
ledger.BanForLimit(b, now, bans.Notes{})
|
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
ledger.BanForLimit(c, now, bans.Notes{})
|
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
// A request from a makes b the netblock seen longest ago, and its ban
|
// A request from a makes b the netblock seen longest ago, and its ban
|
||||||
// goes to make room for d's.
|
// goes to make room for d's.
|
||||||
ledger.Check(a.Addr(), now)
|
ledger.Check(a.Addr(), now)
|
||||||
ledger.BanForLimit(d, now, bans.Notes{})
|
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
||||||
|
|
||||||
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
||||||
ledger.BanForLimit(a, first.Expires, bans.Notes{})
|
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
||||||
|
|
||||||
// With d seen since, a is seen longest ago, and its earlier ban goes
|
// With d seen since, a is seen longest ago, and its earlier ban goes
|
||||||
// first.
|
// first.
|
||||||
ledger.Check(d.Addr(), first.Expires)
|
ledger.Check(d.Addr(), first.Expires)
|
||||||
ledger.BanForLimit(b, first.Expires, bans.Notes{})
|
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
||||||
|
|
||||||
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
||||||
@@ -325,8 +337,16 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
|
|||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
held := ledger.Bans(netblock)
|
held := ledger.Bans(netblock)
|
||||||
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
||||||
@@ -343,7 +363,11 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
||||||
|
|
||||||
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
||||||
|
)
|
||||||
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
||||||
ban.Cause != bans.CauseAttack ||
|
ban.Cause != bans.CauseAttack ||
|
||||||
ban.Notes.RuleID != "env-file" ||
|
ban.Notes.RuleID != "env-file" ||
|
||||||
@@ -418,8 +442,16 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// A ban for a broken limit before does not count.
|
// A ban for a broken limit before does not count.
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
second, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
first.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != 7*day {
|
if second.Expires.Sub(second.Start) != 7*day {
|
||||||
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
||||||
@@ -435,10 +467,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
|
|
||||||
// Its notes show the earlier ban for an attack that makes it permanent,
|
// Its notes show the earlier ban for an attack that makes it permanent,
|
||||||
// beside the one for a limit.
|
// beside the one for a limit.
|
||||||
third, _ := ledger.BanForAttack(
|
third, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
second.Expires.Add(30*day),
|
second.Expires.Add(30*day),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
)
|
)
|
||||||
if !third.Permanent() ||
|
if !third.Permanent() ||
|
||||||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
||||||
@@ -452,14 +484,18 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// While the first ban lasts, none would be made.
|
// While the first ban lasts, none would be made.
|
||||||
during, would := ledger.WouldBanForAttack(
|
during, would := ledger.WouldBan(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
)
|
)
|
||||||
if would || !reflect.DeepEqual(during, first) {
|
if would || !reflect.DeepEqual(during, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -472,13 +508,19 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
// As it ends, a clear sign of attack would ban for seven days, and a
|
// As it ends, a clear sign of attack would ban for seven days, and a
|
||||||
// limit broken again for three hours, but neither is made.
|
// limit broken again for three hours, but neither is made.
|
||||||
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
|
limitGrounds := bans.Grounds{
|
||||||
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
|
Cause: bans.CauseLimit,
|
||||||
bans.Notes{RuleID: "git-dir"})
|
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
|
||||||
limit, wouldLimit := ledger.WouldBanForLimit(
|
}
|
||||||
|
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
|
||||||
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseAttack,
|
||||||
|
Notes: bans.Notes{RuleID: "git-dir"},
|
||||||
|
})
|
||||||
|
limit, wouldLimit := ledger.WouldBan(
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires,
|
first.Expires,
|
||||||
limitNotes,
|
limitGrounds,
|
||||||
)
|
)
|
||||||
|
|
||||||
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
||||||
@@ -505,7 +547,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// The ban made is the one that would have been.
|
// The ban made is the one that would have been.
|
||||||
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
|
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
|
||||||
if !reflect.DeepEqual(made, limit) {
|
if !reflect.DeepEqual(made, limit) {
|
||||||
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
||||||
}
|
}
|
||||||
@@ -526,7 +568,11 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,7 +585,11 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Once a first ban for an attack has ended, the next would be permanent.
|
// Once a first ban for an attack has ended, the next would be permanent.
|
||||||
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
attack, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
||||||
t.Error("a second ban for an attack would not be permanent")
|
t.Error("a second ban for an attack would not be permanent")
|
||||||
}
|
}
|
||||||
@@ -553,8 +603,16 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
|
|||||||
|
|
||||||
// Three times the seven days would be permanent; a limit broken as the
|
// Three times the seven days would be permanent; a limit broken as the
|
||||||
// ban for an attack ends bans for an hour, as a first broken limit does.
|
// ban for an attack ends bans for an hour, as a first broken limit does.
|
||||||
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
|
attack, _ := ledger.Ban(
|
||||||
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack},
|
||||||
|
)
|
||||||
|
limit, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
attack.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
||||||
limit.Cause != bans.CauseLimit {
|
limit.Cause != bans.CauseLimit {
|
||||||
@@ -578,10 +636,13 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
|
|||||||
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{Request: request},
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{Request: request},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
cut := long[:256]
|
cut := long[:256]
|
||||||
@@ -607,11 +668,20 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
|
|||||||
|
|
||||||
// A ban made with the client's lookup, one made before it came, after
|
// A ban made with the client's lookup, one made before it came, after
|
||||||
// the first ended, and one on another netblock.
|
// the first ended, and one on another netblock.
|
||||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{
|
ledger.Ban(netblock, midnight(), bans.Grounds{
|
||||||
ASN: "AS64497", ASName: "Other Net", Country: "FR",
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{
|
||||||
|
ASN: "AS64497",
|
||||||
|
ASName: "Other Net",
|
||||||
|
Country: "FR",
|
||||||
|
},
|
||||||
})
|
})
|
||||||
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
|
ledger.Ban(
|
||||||
ledger.BanForLimit(other, midnight(), bans.Notes{})
|
netblock,
|
||||||
|
midnight().Add(time.Hour),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
||||||
|
|
||||||
|
|||||||
@@ -19,25 +19,29 @@ func TestChangedAfterABanIsMade(t *testing.T) {
|
|||||||
|
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A limit broken during the ban makes no other, and a refusal changes
|
// A limit broken during the ban makes no other, and a refusal changes
|
||||||
// only the counts in the notes, which wait for the interval's write.
|
// only the counts in the notes, which wait for the interval's write.
|
||||||
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
|
ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight().Add(time.Minute),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// Two bans before the value is read leave one.
|
// Two bans before the value is read leave one.
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.11/32"),
|
netip.MustParsePrefix("203.0.113.11/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
@@ -51,10 +55,14 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
|
|||||||
high := netip.MustParsePrefix("203.0.113.10/32")
|
high := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
low := netip.MustParsePrefix("203.0.113.9/32")
|
low := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
|
first, _ := ledger.Ban(
|
||||||
ledger.BanForLimit(high, midnight(), bans.Notes{})
|
v6,
|
||||||
ledger.BanForLimit(low, midnight(), bans.Notes{})
|
midnight(),
|
||||||
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
|
|
||||||
snapshot := ledger.Snapshot()
|
snapshot := ledger.Snapshot()
|
||||||
|
|
||||||
@@ -80,7 +88,10 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
|
|
||||||
before := bans.New(defaultParams())
|
before := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
|
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
|
||||||
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{Limit: 1},
|
||||||
|
})
|
||||||
|
|
||||||
// Loaded into a new ledger, as across a restart, the ban still refuses
|
// Loaded into a new ledger, as across a restart, the ban still refuses
|
||||||
// while it lasts, and once it has ended a broken limit bans for three
|
// while it lasts, and once it has ended a broken limit bans for three
|
||||||
@@ -96,7 +107,11 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
t.Error("the loaded ban does not refuse")
|
t.Error("the loaded ban does not refuse")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
|
again, _ := after.Ban(
|
||||||
|
netblock,
|
||||||
|
ban.Expires,
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
||||||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -124,10 +139,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
|
|||||||
Start: midnight(),
|
Start: midnight(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
ledger.BanForLimit(
|
ledger.Ban(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
for client, want := range map[string]bool{
|
for client, want := range map[string]bool{
|
||||||
@@ -200,7 +215,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A limit broken now makes no shorter ban over the permanent one.
|
// A limit broken now makes no shorter ban over the permanent one.
|
||||||
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
|
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
||||||
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"a broken limit returned %+v and left the netblock %d bans, "+
|
"a broken limit returned %+v and left the netblock %d bans, "+
|
||||||
@@ -236,10 +251,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
|
|||||||
// Once both have ended, a limit broken within the repeat window bans
|
// Once both have ended, a limit broken within the repeat window bans
|
||||||
// for three times the 9 hours, and the notes count the two bans
|
// for three times the 9 hours, and the notes count the two bans
|
||||||
// before the 9-hour one and it, for a limit, and the admin's.
|
// before the 9-hour one and it, for a limit, and the admin's.
|
||||||
ban, _ := ledger.BanForLimit(
|
ban, _ := ledger.Ban(
|
||||||
netblock,
|
netblock,
|
||||||
nineHours.Expires.Add(time.Hour),
|
nineHours.Expires.Add(time.Hour),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
||||||
@@ -313,15 +328,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The ledger holds one ban, so it makes two more without dropping any.
|
// The ledger holds one ban, so it makes two more without dropping any.
|
||||||
first, _ := ledger.BanForLimit(
|
first, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
second, _ := ledger.BanForLimit(
|
second, _ := ledger.Ban(
|
||||||
netip.MustParsePrefix("198.51.100.8/32"),
|
netip.MustParsePrefix("198.51.100.8/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
want := []bans.Ban{first, second, kept}
|
want := []bans.Ban{first, second, kept}
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// off is the value that switches a timeout, a size limit or a rate limit
|
// off is the value that switches a timeout, a size limit or a rate limit
|
||||||
@@ -40,9 +42,7 @@ var (
|
|||||||
errNotPositive = errors.New("must be more than zero, or off")
|
errNotPositive = errors.New("must be more than zero, or off")
|
||||||
errNotMilliseconds = errors.New(
|
errNotMilliseconds = errors.New(
|
||||||
"is not a whole number of milliseconds above zero, such as 2000")
|
"is not a whole number of milliseconds above zero, such as 2000")
|
||||||
errEmptyItem = errors.New("has an empty item in its list")
|
errEmptyItem = errors.New("has an empty item in its list")
|
||||||
errNotNetblock = errors.New(
|
|
||||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
|
||||||
errNotListenAddr = errors.New(
|
errNotListenAddr = errors.New(
|
||||||
"is not an address to listen on, such as :8080")
|
"is not an address to listen on, such as :8080")
|
||||||
errNotCommand = errors.New("is not rproxy or waf")
|
errNotCommand = errors.New("is not rproxy or waf")
|
||||||
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
netblocks := make([]netip.Prefix, 0, len(items))
|
netblocks := make([]netip.Prefix, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
netblock, err := ParseNetblock(item)
|
netblock, err := reputation.ParseNetblock(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -264,27 +264,6 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
return netblocks, nil
|
return netblocks, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
|
||||||
// address is a netblock of that address alone, a /32 or a /128. A
|
|
||||||
// blocklist's lines are read with it too.
|
|
||||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
|
||||||
if strings.Contains(value, "/") {
|
|
||||||
netblock, err := netip.ParsePrefix(value)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netblock.Masked(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(value)
|
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseListenAddr checks an address to listen on: an optional host and a
|
// parseListenAddr checks an address to listen on: an optional host and a
|
||||||
// port number.
|
// port number.
|
||||||
func parseListenAddr(value string) (string, error) {
|
func parseListenAddr(value string) (string, error) {
|
||||||
|
|||||||
+77
-121
@@ -20,11 +20,26 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The names of the settings that waf writes into log lines and ban notes,
|
||||||
|
// as the setting that refused a request or lowered a client's limits.
|
||||||
|
const (
|
||||||
|
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
|
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||||
|
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
||||||
|
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
||||||
|
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
||||||
|
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
||||||
|
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
||||||
|
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||||
@@ -100,7 +115,7 @@ type WAF struct {
|
|||||||
// with a tag that percentage of every rate limit and byte limit, from 1
|
// with a tag that percentage of every rate limit and byte limit, from 1
|
||||||
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
||||||
// lower case, a country code in capitals.
|
// lower case, a country code in capitals.
|
||||||
TagLists []TagList
|
TagLists []reputation.TagList
|
||||||
TagDeny []string
|
TagDeny []string
|
||||||
TagLimitPercent map[string]int64
|
TagLimitPercent map[string]int64
|
||||||
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
||||||
@@ -497,12 +512,8 @@ var (
|
|||||||
"is not a whole number above zero, such as 60, or off",
|
"is not a whole number above zero, such as 60, or off",
|
||||||
)
|
)
|
||||||
errNotUTF8 = errors.New("is not valid UTF-8")
|
errNotUTF8 = errors.New("is not valid UTF-8")
|
||||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
|
||||||
errNotPercentItem = errors.New(
|
errNotPercentItem = errors.New(
|
||||||
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
||||||
errNotPercent = errors.New(
|
|
||||||
"is not a percentage, a whole number from 0 to 100",
|
|
||||||
)
|
|
||||||
errNotAbuseIPDBThreshold = errors.New(
|
errNotAbuseIPDBThreshold = errors.New(
|
||||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
||||||
"blacklist can be cut at",
|
"blacklist can be cut at",
|
||||||
@@ -618,24 +629,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"800",
|
"800",
|
||||||
),
|
),
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
ExclusivelyAllowedCountriesSetting, ""),
|
||||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||||
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
||||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||||
ASNBytesLimitPercent: env.percents(
|
ASNBytesLimitPercent: env.percents(
|
||||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
ASNBytesLimitPercentSetting,
|
||||||
ParseASN,
|
reputation.ParseASN,
|
||||||
),
|
),
|
||||||
CountryBytesLimitPercent: env.percents(
|
CountryBytesLimitPercent: env.percents(
|
||||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
CountryBytesLimitPercentSetting,
|
||||||
parseCountry,
|
parseCountry,
|
||||||
),
|
),
|
||||||
UnknownCountryLimitPercent: env.percent(
|
UnknownCountryLimitPercent: env.percent(
|
||||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
UnknownCountryLimitPercentSetting,
|
||||||
"100",
|
"100",
|
||||||
),
|
),
|
||||||
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
||||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||||
@@ -809,9 +820,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
BlocklistActionSetting, string(ReputationActionDeny))
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
DNSBLAbuseIPDBActionSetting, "limit:25")
|
||||||
|
|
||||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||||
env.checkLookupDBPath(cfg)
|
env.checkLookupDBPath(cfg)
|
||||||
@@ -1011,7 +1022,7 @@ func (e *environment) percents(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
||||||
func (e *environment) tagLists(name string) []TagList {
|
func (e *environment) tagLists(name string) []reputation.TagList {
|
||||||
lists, err := parseTagLists(e.value(name, ""))
|
lists, err := parseTagLists(e.value(name, ""))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
@@ -1037,7 +1048,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
|
|||||||
|
|
||||||
// percent reads a setting that is a percentage, from 0 to 100.
|
// percent reads a setting that is a percentage, from 0 to 100.
|
||||||
func (e *environment) percent(name, defaultValue string) int64 {
|
func (e *environment) percent(name, defaultValue string) int64 {
|
||||||
percent, err := ParsePercent(e.value(name, defaultValue))
|
percent, err := reputation.ParsePercent(e.value(name, defaultValue))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
return percent
|
return percent
|
||||||
@@ -1122,7 +1133,7 @@ func (e *environment) action(
|
|||||||
string(value), string(ReputationActionLimit)+":",
|
string(value), string(ReputationActionLimit)+":",
|
||||||
)
|
)
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := reputation.ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
@@ -1131,8 +1142,8 @@ func (e *environment) action(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
// default. The log shows each zone with its key masked, as MaskZoneKey
|
// default. The log shows each zone with its key masked, as
|
||||||
// masks it.
|
// reputation.MaskZoneKey masks it.
|
||||||
func (e *environment) zones(name string) []string {
|
func (e *environment) zones(name string) []string {
|
||||||
value, _ := e.lookup(name)
|
value, _ := e.lookup(name)
|
||||||
zones, err := parseZones(value)
|
zones, err := parseZones(value)
|
||||||
@@ -1140,7 +1151,7 @@ func (e *environment) zones(name string) []string {
|
|||||||
|
|
||||||
logged := make([]string, len(zones))
|
logged := make([]string, len(zones))
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
logged[i] = MaskZoneKey(zone)
|
logged[i] = reputation.MaskZoneKey(zone)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.settings = append(
|
e.settings = append(
|
||||||
@@ -1220,7 +1231,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
|||||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||||
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
e.check(ExclusivelyAllowedCountriesSetting,
|
||||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1239,15 +1250,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
}{
|
}{
|
||||||
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||||
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
{TagLimitPercentSetting, slices.ContainsFunc(
|
||||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||||
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
||||||
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
||||||
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
||||||
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||||
@@ -1267,7 +1278,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||||
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
e.check(ASNLimitPercentURLSetting,
|
||||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1294,14 +1305,18 @@ func (e *environment) checkTags(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for tag := range cfg.TagLimitPercent {
|
for tag := range cfg.TagLimitPercent {
|
||||||
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkTag returns an error when tag would never apply, as checkTags says.
|
// checkTag returns an error when tag would never apply, as checkTags says.
|
||||||
func checkTag(cfg *WAF, tag string) error {
|
func checkTag(cfg *WAF, tag string) error {
|
||||||
if !requestCanGetTag(cfg, tag) {
|
if !requestCanGetTag(cfg, tag) {
|
||||||
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
|
return fmt.Errorf(
|
||||||
|
"%q %w",
|
||||||
|
reputation.MaskZoneKey(tag),
|
||||||
|
errNoRequestGetsTag,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -1309,7 +1324,7 @@ func checkTag(cfg *WAF, tag string) error {
|
|||||||
|
|
||||||
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
||||||
func requestCanGetTag(cfg *WAF, tag string) bool {
|
func requestCanGetTag(cfg *WAF, tag string) bool {
|
||||||
isListName := func(list TagList) bool { return list.Name == tag }
|
isListName := func(list reputation.TagList) bool { return list.Name == tag }
|
||||||
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -1333,7 +1348,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
|
|||||||
|
|
||||||
return isZoneTag &&
|
return isZoneTag &&
|
||||||
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
||||||
return strings.ToLower(MaskZoneKey(each)) == zone
|
return strings.ToLower(reputation.MaskZoneKey(each)) == zone
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1831,7 +1846,9 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
|
netblock, err := reputation.ParseNetblock(
|
||||||
|
strings.TrimSpace(netblockText),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2037,21 +2054,6 @@ func parseCountry(value string) (string, error) {
|
|||||||
return country, nil
|
return country, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
|
||||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
|
||||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
|
||||||
// read with it too.
|
|
||||||
func ParseASN(value string) (string, error) {
|
|
||||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
|
||||||
|
|
||||||
number, err := strconv.ParseUint(digits, 10, 32)
|
|
||||||
if !hasAS || err != nil {
|
|
||||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
return "AS" + strconv.FormatUint(number, 10), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tags a request can get that the settings can name, as waf
|
// The tags a request can get that the settings can name, as waf
|
||||||
// writes them. The prefixes are followed by a value.
|
// writes them. The prefixes are followed by a value.
|
||||||
const (
|
const (
|
||||||
@@ -2065,24 +2067,17 @@ const (
|
|||||||
ExploitSignatureTag = "exploitsignature"
|
ExploitSignatureTag = "exploitsignature"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
|
||||||
// requests of the clients it holds, and the URL it is fetched from.
|
|
||||||
type TagList struct {
|
|
||||||
Name string
|
|
||||||
URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
||||||
// torexit=https://example.com/tor.txt, and returns each list with its name
|
// torexit=https://example.com/tor.txt, and returns each list with its name
|
||||||
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
||||||
// = before http. A name or a URL listed twice is an error.
|
// = before http. A name or a URL listed twice is an error.
|
||||||
func parseTagLists(value string) ([]TagList, error) {
|
func parseTagLists(value string) ([]reputation.TagList, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
lists := make([]TagList, 0, len(items))
|
lists := make([]reputation.TagList, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
end := strings.Index(item, "=http")
|
end := strings.Index(item, "=http")
|
||||||
@@ -2090,9 +2085,12 @@ func parseTagLists(value string) ([]TagList, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
||||||
}
|
}
|
||||||
|
|
||||||
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
|
list := reputation.TagList{
|
||||||
|
Name: strings.ToLower(item[:end]),
|
||||||
|
URL: item[end+1:],
|
||||||
|
}
|
||||||
|
|
||||||
if slices.ContainsFunc(lists, func(earlier TagList) bool {
|
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool {
|
||||||
return earlier.Name == list.Name || earlier.URL == list.URL
|
return earlier.Name == list.Name || earlier.URL == list.URL
|
||||||
}) {
|
}) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||||
@@ -2105,7 +2103,8 @@ func parseTagLists(value string) ([]TagList, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
||||||
// tag listed twice is an error, which shows the tag as MaskZoneKey does.
|
// tag listed twice is an error, which shows the tag as
|
||||||
|
// reputation.MaskZoneKey does.
|
||||||
func parseTags(value string) ([]string, error) {
|
func parseTags(value string) ([]string, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2125,7 +2124,7 @@ func parseTags(value string) ([]string, error) {
|
|||||||
if slices.Contains(tags, tag) {
|
if slices.Contains(tags, tag) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item),
|
reputation.MaskZoneKey(item),
|
||||||
errListedTwice,
|
errListedTwice,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2194,7 +2193,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if colon <= 0 {
|
if colon <= 0 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item),
|
reputation.MaskZoneKey(item),
|
||||||
errNotTagPercentItem,
|
errNotTagPercentItem,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2208,7 +2207,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if err != nil || percent < 1 || percent > 99 {
|
if err != nil || percent < 1 || percent > 99 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item[colon+1:]),
|
reputation.MaskZoneKey(item[colon+1:]),
|
||||||
errNotTagPercent,
|
errNotTagPercent,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2220,7 +2219,11 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if _, listed := percents[tag]; listed {
|
if _, listed := percents[tag]; listed {
|
||||||
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
||||||
// would not mask.
|
// would not mask.
|
||||||
return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
|
return nil, fmt.Errorf(
|
||||||
|
"%q %w",
|
||||||
|
reputation.MaskZoneKey(tag),
|
||||||
|
errListedTwice,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
percents[tag] = percent
|
percents[tag] = percent
|
||||||
@@ -2254,7 +2257,7 @@ func parsePercents(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := reputation.ParsePercent(percentText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2269,38 +2272,6 @@ func parsePercents(
|
|||||||
return percents, nil
|
return percents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
|
||||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
|
||||||
func ParsePercent(value string) (int64, error) {
|
|
||||||
percent, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || percent < 0 || percent > 100 {
|
|
||||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
|
||||||
}
|
|
||||||
|
|
||||||
return percent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// headerNameChars are the characters RFC 9110 allows in a header name:
|
|
||||||
// letters, digits and these marks.
|
|
||||||
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
|
|
||||||
"0123456789!#$%&'*+-.^_`|~"
|
|
||||||
|
|
||||||
// IsHeaderName reports whether name can be a header name: one or more of
|
|
||||||
// the characters RFC 9110 allows in one.
|
|
||||||
func IsHeaderName(name string) bool {
|
|
||||||
if name == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, char := range name {
|
|
||||||
if !strings.ContainsRune(headerNameChars, char) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseHeaderNames reads a comma-separated list of header names in either
|
// parseHeaderNames reads a comma-separated list of header names in either
|
||||||
// case, and returns them in lower case. Host and Transfer-Encoding are
|
// case, and returns them in lower case. Host and Transfer-Encoding are
|
||||||
// refused: Go's HTTP server takes them out of the request's headers.
|
// refused: Go's HTTP server takes them out of the request's headers.
|
||||||
@@ -2313,7 +2284,7 @@ func parseHeaderNames(value string) ([]string, error) {
|
|||||||
headers := make([]string, 0, len(items))
|
headers := make([]string, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if !IsHeaderName(item) {
|
if !httpguts.ValidHeaderFieldName(item) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2469,8 +2440,8 @@ const (
|
|||||||
// other name for one that does not exist, so that the zone would list no
|
// other name for one that does not exist, so that the zone would list no
|
||||||
// client. A zone listed twice is an error, whatever the case of its
|
// client. A zone listed twice is an error, whatever the case of its
|
||||||
// letters, which DNS names ignore, and whatever its key, since
|
// letters, which DNS names ignore, and whatever its key, since
|
||||||
// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
|
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a
|
||||||
// MaskZoneKey does.
|
// zone as it does.
|
||||||
func parseZones(value string) ([]string, error) {
|
func parseZones(value string) ([]string, error) {
|
||||||
zones, err := parseList(value)
|
zones, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2480,11 +2451,14 @@ func parseZones(value string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
shown := MaskZoneKey(zone)
|
shown := reputation.MaskZoneKey(zone)
|
||||||
listedBefore := slices.ContainsFunc(
|
listedBefore := slices.ContainsFunc(
|
||||||
zones[:i],
|
zones[:i],
|
||||||
func(earlier string) bool {
|
func(earlier string) bool {
|
||||||
return strings.EqualFold(MaskZoneKey(earlier), shown)
|
return strings.EqualFold(
|
||||||
|
reputation.MaskZoneKey(earlier),
|
||||||
|
shown,
|
||||||
|
)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2501,24 +2475,6 @@ func parseZones(value string) ([]string, error) {
|
|||||||
return zones, nil
|
return zones, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
|
||||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
|
||||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
|
||||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
|
||||||
// the process: in the log, the alerts and the metrics.
|
|
||||||
func MaskZoneKey(zone string) string {
|
|
||||||
// DNS names ignore case, and a name may be written with a dot at its
|
|
||||||
// end.
|
|
||||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
|
||||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
|
||||||
return zone
|
|
||||||
}
|
|
||||||
|
|
||||||
_, rest, _ := strings.Cut(zone, ".")
|
|
||||||
|
|
||||||
return masked + "." + rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// isZone reports whether each label of zone is as parseZones takes it. It
|
// isZone reports whether each label of zone is as parseZones takes it. It
|
||||||
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
||||||
func isZone(zone string) bool {
|
func isZone(zone string) bool {
|
||||||
@@ -2574,7 +2530,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
|
|||||||
name, headerValue, found := strings.Cut(item, ":")
|
name, headerValue, found := strings.Cut(item, ":")
|
||||||
name = strings.TrimSpace(name)
|
name = strings.TrimSpace(name)
|
||||||
|
|
||||||
if !found || !IsHeaderName(name) ||
|
if !found || !httpguts.ValidHeaderFieldName(name) ||
|
||||||
strings.ContainsAny(headerValue, "\r\n\x00") {
|
strings.ContainsAny(headerValue, "\r\n\x00") {
|
||||||
return nil, "", fmt.Errorf(
|
return nil, "", fmt.Errorf(
|
||||||
"item %d %w",
|
"item %d %w",
|
||||||
|
|||||||
+29
-20
@@ -18,10 +18,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The settings, by name.
|
// The settings, by name.
|
||||||
@@ -864,6 +866,32 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// config and rules check header names with httpguts.ValidHeaderFieldName.
|
||||||
|
// It takes a name of one or more of the characters RFC 9110 allows in one,
|
||||||
|
// as the check it replaced did: letters, digits and these marks.
|
||||||
|
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const marks = "!#$%&'*+-.^_`|~"
|
||||||
|
|
||||||
|
if httpguts.ValidHeaderFieldName("") {
|
||||||
|
t.Error("the empty name is taken")
|
||||||
|
}
|
||||||
|
|
||||||
|
for b := range 256 {
|
||||||
|
char := byte(b)
|
||||||
|
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
|
||||||
|
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
|
||||||
|
|
||||||
|
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
|
||||||
|
// for a byte above 0x7f.
|
||||||
|
name := string([]byte{char})
|
||||||
|
if got := httpguts.ValidHeaderFieldName(name); got != want {
|
||||||
|
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1694,7 +1722,7 @@ func TestTagSettingsAsSet(t *testing.T) {
|
|||||||
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantLists := []config.TagList{
|
wantLists := []reputation.TagList{
|
||||||
{Name: torExitTag, URL: torURL},
|
{Name: torExitTag, URL: torURL},
|
||||||
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||||
}
|
}
|
||||||
@@ -2288,25 +2316,6 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for zone, want := range map[string]string{
|
|
||||||
spamhaus: spamhausMasked,
|
|
||||||
spamhaus + ".": spamhausMasked + ".",
|
|
||||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
|
||||||
dronebl: dronebl,
|
|
||||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
|
||||||
spamhaus + ".example": spamhaus + ".example",
|
|
||||||
} {
|
|
||||||
if got := config.MaskZoneKey(zone); got != want {
|
|
||||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+60
-37
@@ -84,12 +84,7 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
url string
|
params Params
|
||||||
answered func(Answer)
|
|
||||||
now func() time.Time
|
|
||||||
asked func()
|
|
||||||
failed func(err error, askingAgainIn time.Duration)
|
|
||||||
wentWithoutAnswer func()
|
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -99,8 +94,13 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way.
|
// asking is true while a request to GeoJS is under way, and endAsking
|
||||||
asking bool
|
// ends it. stopped is true once StopAsking has been called, and
|
||||||
|
// askGoroutines waits for the goroutine that makes the requests.
|
||||||
|
asking bool
|
||||||
|
endAsking context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
askGoroutines sync.WaitGroup
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
@@ -140,12 +140,7 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
url: params.URL,
|
params: params,
|
||||||
answered: params.Answered,
|
|
||||||
now: params.Now,
|
|
||||||
asked: params.Asked,
|
|
||||||
failed: params.Failed,
|
|
||||||
wentWithoutAnswer: params.WentWithoutAnswer,
|
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -188,7 +183,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -210,6 +205,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the request to GeoJS under way, and returns once the
|
||||||
|
// goroutine that made it has. A request it ends is no failure: Failed is not
|
||||||
|
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
||||||
|
// asked nothing.
|
||||||
|
func (g *GeoJS) StopAsking() {
|
||||||
|
g.mu.Lock()
|
||||||
|
|
||||||
|
g.stopped = true
|
||||||
|
if g.asking {
|
||||||
|
g.endAsking()
|
||||||
|
}
|
||||||
|
|
||||||
|
g.mu.Unlock()
|
||||||
|
|
||||||
|
g.askGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -251,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -288,18 +300,19 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, so no answer can come.
|
// GeoJS is left alone after a failure, or asked nothing after
|
||||||
|
// StopAsking, so no answer can come.
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -310,7 +323,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -323,22 +336,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way or it is left alone after a failure. The requests to
|
// it is under way, it is left alone after a failure, or StopAsking has
|
||||||
// GeoJS are for every client waiting, so they go on when the caller whose
|
// been called. The requests to GeoJS are for every client waiting, so they
|
||||||
// ctx is given stops waiting.
|
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.now().Before(g.retryAt) {
|
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
g.asking = true
|
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
g.asking, g.endAsking = true, end
|
||||||
|
|
||||||
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
g.askGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
g.askAboutWaiting(askCtx)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
||||||
// Answered, outside the lock, since Answered takes locks of its own.
|
// answer kept is given to Answered, outside the lock, since Answered takes
|
||||||
|
// locks of its own.
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -347,11 +366,15 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
continue // ended by StopAsking, which is no failure
|
||||||
|
}
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.answered != nil {
|
if g.params.Answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.answered(answer)
|
g.params.Answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,13 +385,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, it returns none and notes that no request to GeoJS is
|
// none is waiting, or StopAsking has been called, it returns none and
|
||||||
// under way.
|
// notes that no request to GeoJS is under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 {
|
if len(g.waiting) == 0 || g.stopped {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -400,7 +423,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -444,7 +467,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.failed(err, g.retryDelay)
|
g.params.Failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -474,7 +497,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.url,
|
g.params.URL,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -483,7 +506,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.asked()
|
g.params.Asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
service, _, g, reports := startWithReports()
|
||||||
|
clients := newClients()
|
||||||
|
|
||||||
|
service.set(hanging)
|
||||||
|
g.LookUpWithoutWaiting(t.Context(), clients())
|
||||||
|
waitForRequests(t, service, 1)
|
||||||
|
|
||||||
|
// The request is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
g.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
reports.wantFailures(t)
|
||||||
|
|
||||||
|
// A new client is then not asked about, and its lookup does not
|
||||||
|
// wait.
|
||||||
|
service.set(answering)
|
||||||
|
wantCountry(t, g, clients(), "")
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("the lookup returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
waitForRequests(t, service, 1)
|
||||||
|
reports.wantFailures(t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
@@ -301,7 +300,7 @@ const sourceLabel = "source"
|
|||||||
|
|
||||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
|
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||||
// the fetches that failed and when the copy in use was fetched, and for a
|
// the fetches that failed and when the copy in use was fetched, and for a
|
||||||
@@ -320,7 +319,9 @@ func (m *Metrics) AddReputation(
|
|||||||
m.registry.MustRegister(m.reputationHits)
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
for _, zone := range dnsbl.Zones() {
|
||||||
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
|
source := prometheus.Labels{
|
||||||
|
sourceLabel: reputation.MaskZoneKey(zone),
|
||||||
|
}
|
||||||
|
|
||||||
m.addReputationQueries(
|
m.addReputationQueries(
|
||||||
source,
|
source,
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
// Package processlog builds the logger for the process's own messages,
|
||||||
|
// which it writes as JSON lines marked "type":"process". The request log
|
||||||
|
// is package requestlog's.
|
||||||
|
package processlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Params are what New needs.
|
||||||
|
type Params struct {
|
||||||
|
// Output is where the lines are written.
|
||||||
|
Output io.Writer
|
||||||
|
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
||||||
|
InstanceName string
|
||||||
|
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
||||||
|
Level slog.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns the logger for the process's own messages: JSON lines on
|
||||||
|
// params.Output, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's. It writes only the messages at params.Level or more
|
||||||
|
// severe; the request lines requestlog.Write writes are never held back.
|
||||||
|
func New(params Params) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
||||||
|
Level: params.Level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
requestlog.FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", params.InstanceName)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package processlog_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// instanceName is the instance the tests' loggers are built with.
|
||||||
|
const instanceName = "host1/blog"
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: slog.LevelInfo,
|
||||||
|
}).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: level,
|
||||||
|
})
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,16 +156,28 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a Limiter for limits, with no client counted yet, whose
|
// Params are what New needs.
|
||||||
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
type Params struct {
|
||||||
// it, the least recently seen client is dropped, with its history, and
|
// Limits are what each client's requests and bytes are counted against.
|
||||||
// starts afresh if it comes back.
|
Limits Limits
|
||||||
func New(limits Limits, maxClients int) *Limiter {
|
// MaxClients is the most clients the table holds,
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
||||||
|
// is dropped, with its history, and starts afresh if it comes back.
|
||||||
|
MaxClients int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a Limiter with no client counted yet.
|
||||||
|
func New(params Params) *Limiter {
|
||||||
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
||||||
|
params.MaxClients,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
|
limits := params.Limits
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
Limits: ratelimit.Limits{
|
||||||
tableSize,
|
PerMinute: 5,
|
||||||
)
|
BytesPerDay: math.MaxInt64,
|
||||||
|
},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: 1},
|
||||||
|
MaxClients: maxClients,
|
||||||
|
})
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
before := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
after := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way.
|
// checking are the clients whose check is under way, each with the
|
||||||
checking map[netip.Prefix]bool
|
// function that ends it. stopped is true once StopChecking has been
|
||||||
|
// called, and checkGoroutines waits for the goroutine of each check.
|
||||||
|
checking map[netip.Prefix]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
checkGoroutines sync.WaitGroup
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]bool{},
|
checking: map[netip.Prefix]context.CancelFunc{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, or the day's checks
|
// way, a check failed less than failureDelay ago, the day's checks have
|
||||||
// have used up DailyBudget; Hit never waits for a check. The check that
|
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
||||||
// uses the budget up is logged and raised as a source_failure alert. ctx
|
// for a check. The check that uses the budget up is logged and raised as a
|
||||||
// is the context of the client's request, and a check goes on after the
|
// source_failure alert. ctx is the context of the client's request, and a
|
||||||
// request ends.
|
// check goes on after the request ends, until StopChecking.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
_, checking := a.checking[client]
|
||||||
a.spent < a.params.DailyBudget
|
|
||||||
|
check := offender && !checking && !a.stopped &&
|
||||||
|
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
||||||
if check {
|
if check {
|
||||||
a.checking[client] = true
|
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
a.checking[client] = end
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
go a.check(context.WithoutCancel(ctx), client, addr)
|
a.checkGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
a.check(checkCtx, client, addr)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopChecking ends the checks under way, and returns once their goroutines
|
||||||
|
// have. A check it ends gives no score and is no failure: it is neither
|
||||||
|
// counted, logged nor raised as an alert, and checks are not left alone
|
||||||
|
// after it. The ended check stays spent from the day's budget. After
|
||||||
|
// StopChecking, Hit checks no client.
|
||||||
|
func (a *AbuseIPDB) StopChecking() {
|
||||||
|
a.mu.Lock()
|
||||||
|
|
||||||
|
a.stopped = true
|
||||||
|
for _, end := range a.checking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
a.checkGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay.
|
// source_failure alert, and no client is checked for failureDelay. A check
|
||||||
|
// ended as ctx ends, by StopChecking, is no failure.
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
} else {
|
case isFailure:
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if isFailure {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
queue := newQueue()
|
||||||
|
p := abuseIPDBParams()
|
||||||
|
p.Alerts = queue
|
||||||
|
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
||||||
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
|
|
||||||
|
wantScore(t, checker, suspect, true, 0, false)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The check is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
checker.StopChecking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopChecking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFailures(t, checker, 0)
|
||||||
|
|
||||||
|
if got := waiting(queue); len(got) != 0 {
|
||||||
|
t.Errorf("alerts waiting %+v, want none", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An offender without a score is then not checked.
|
||||||
|
wantScore(t, checker, other, true, 0, false)
|
||||||
|
synctest.Wait()
|
||||||
|
wantChecked(t, abuseIPDB, suspect)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
|
|
||||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -32,6 +31,8 @@ const (
|
|||||||
// it fails, and no client is checked with AbuseIPDB after a check
|
// it fails, and no client is checked with AbuseIPDB after a check
|
||||||
// fails, so that a source refusing them is not asked on every request.
|
// fails, so that a source refusing them is not asked on every request.
|
||||||
failureDelay = time.Minute
|
failureDelay = time.Minute
|
||||||
|
// maskedKey is what MaskZoneKey shows in place of a zone's key.
|
||||||
|
maskedKey = "********"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -82,8 +83,12 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way.
|
// asking are the queries under way, each with the function that ends
|
||||||
asking map[query]bool
|
// it. stopped is true once StopAsking has been called, and
|
||||||
|
// queryGoroutines waits for the goroutine of each query.
|
||||||
|
asking map[query]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
queryGoroutines sync.WaitGroup
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -120,13 +125,31 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]bool{},
|
asking: map[query]context.CancelFunc{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
||||||
|
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
||||||
|
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
||||||
|
// other zone it returns as it is. A zone is shown so wherever it leaves
|
||||||
|
// the process: in the log, the alerts and the metrics.
|
||||||
|
func MaskZoneKey(zone string) string {
|
||||||
|
// DNS names ignore case, and a name may be written with a dot at its
|
||||||
|
// end.
|
||||||
|
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||||
|
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
||||||
|
return zone
|
||||||
|
}
|
||||||
|
|
||||||
|
_, rest, _ := strings.Cut(zone, ".")
|
||||||
|
|
||||||
|
return maskedKey + "." + rest
|
||||||
|
}
|
||||||
|
|
||||||
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
||||||
func (d *DNSBL) Zones() []string {
|
func (d *DNSBL) Zones() []string {
|
||||||
return slices.Clone(d.params.Zones)
|
return slices.Clone(d.params.Zones)
|
||||||
@@ -134,13 +157,14 @@ func (d *DNSBL) Zones() []string {
|
|||||||
|
|
||||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||||
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
||||||
// masked, as config.MaskZoneKey masks it, since they go to the request
|
// masked, as MaskZoneKey masks it, since they go to the request
|
||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, or maxQueries are under way;
|
// zone is left alone after a failure, maxQueries are under way, or
|
||||||
// ListedBy never waits for a query. ctx is the context of the client's
|
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
||||||
// request, and a query goes on after the request ends.
|
// context of the client's request, and a query goes on after the request
|
||||||
|
// ends, until StopAsking.
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -153,23 +177,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
|
_, asking := d.asking[q]
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
listedBy = append(listedBy, MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||||
d.asking[q] = true
|
len(d.asking) < maxQueries:
|
||||||
|
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
d.asking[q] = end
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
go d.ask(context.WithoutCancel(ctx), q)
|
d.queryGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
d.ask(queryCtx, q)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the queries under way, and returns once their
|
||||||
|
// goroutines have. A query it ends gives no verdict and is no failure: it
|
||||||
|
// is neither counted, logged nor raised as an alert, and the zone is not
|
||||||
|
// left alone after it. After StopAsking, ListedBy asks no zone.
|
||||||
|
func (d *DNSBL) StopAsking() {
|
||||||
|
d.mu.Lock()
|
||||||
|
|
||||||
|
d.stopped = true
|
||||||
|
for _, end := range d.asking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
d.queryGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -238,30 +286,32 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay.
|
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
} else {
|
case isFailure:
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if isFailure {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := config.MaskZoneKey(q.zone)
|
shown := MaskZoneKey(q.zone)
|
||||||
|
|
||||||
// Raised before it is logged, so that the alert is there once the
|
// Raised before it is logged, so that the alert is there once the
|
||||||
// log line is.
|
// log line is.
|
||||||
|
|||||||
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
|
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
queue := newQueue()
|
||||||
|
p := dnsblParams(zone)
|
||||||
|
p.Alerts = queue
|
||||||
|
resolver := &resolverStandIn{hanging: true}
|
||||||
|
dnsbl := newDNSBL(resolver, p)
|
||||||
|
|
||||||
|
wantZones(t, dnsbl, listed)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The query is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
dnsbl.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantQueries(t, dnsbl, 1, 0)
|
||||||
|
|
||||||
|
if got := waiting(queue); len(got) != 0 {
|
||||||
|
t.Errorf("alerts waiting %+v, want none", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client without a verdict is then asked about in no zone.
|
||||||
|
wantZones(t, dnsbl, unlisted)
|
||||||
|
synctest.Wait()
|
||||||
|
wantQueries(t, dnsbl, 1, 0)
|
||||||
|
wantAsked(t, resolver, listedName)
|
||||||
|
waitForTheResolver()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
@@ -821,3 +860,28 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
|||||||
t.Errorf("asked about %v, want %v", got, want)
|
t.Errorf("asked about %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
dronebl = "dnsbl.dronebl.org"
|
||||||
|
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
||||||
|
spamhausMasked = "********.xbl.dq.spamhaus.net"
|
||||||
|
)
|
||||||
|
|
||||||
|
for zone, want := range map[string]string{
|
||||||
|
spamhaus: spamhausMasked,
|
||||||
|
spamhaus + ".": spamhausMasked + ".",
|
||||||
|
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
||||||
|
dronebl: dronebl,
|
||||||
|
"dq.spamhaus.net": "dq.spamhaus.net",
|
||||||
|
spamhaus + ".example": spamhaus + ".example",
|
||||||
|
} {
|
||||||
|
if got := reputation.MaskZoneKey(zone); got != want {
|
||||||
|
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,8 +7,10 @@
|
|||||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||||
// today. The state package writes all of these to files and reads them
|
// today. The state package writes all of these to files and reads them
|
||||||
// from there, so that a restart keeps them too. Last, it fetches
|
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
|
||||||
// AbuseIPDB's blacklist, which it does not keep.
|
// blacklist, which it does not keep. Last, it reads the netblocks, AS
|
||||||
|
// numbers and percentages that both the settings and the lists hold, and
|
||||||
|
// masks the keys of DNSBL zones.
|
||||||
package reputation
|
package reputation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -20,12 +22,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -40,15 +42,28 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errStatus = errors.New("the server answered")
|
errStatus = errors.New("the server answered")
|
||||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||||
errNotNetblock = errors.New(
|
errNotNetblockLine = errors.New(
|
||||||
"is not an address or a netblock, such as 192.0.2.0/24",
|
"is not an address or a netblock, such as 192.0.2.0/24",
|
||||||
)
|
)
|
||||||
errNotASNPercent = errors.New(
|
errNotASNPercent = errors.New(
|
||||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||||
|
errNotNetblock = errors.New(
|
||||||
|
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||||
|
errNotASN = errors.New("is not an AS number such as AS64496")
|
||||||
|
errNotPercent = errors.New(
|
||||||
|
"is not a percentage, a whole number from 0 to 100",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
||||||
|
// requests of the clients it holds, and the URL it is fetched from.
|
||||||
|
type TagList struct {
|
||||||
|
Name string
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
// List is a list as lists.json holds it: the URL it is fetched from,
|
// List is a list as lists.json holds it: the URL it is fetched from,
|
||||||
// when it was last tried, the fetch failed or not, and its last good copy:
|
// when it was last tried, the fetch failed or not, and its last good copy:
|
||||||
// when that was fetched, and its lines, as fetched, comment lines
|
// when that was fetched, and its lines, as fetched, comment lines
|
||||||
@@ -68,7 +83,7 @@ type Params struct {
|
|||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
ASNLimitPercentURL string
|
ASNLimitPercentURL string
|
||||||
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
||||||
TagLists []config.TagList
|
TagLists []TagList
|
||||||
// Refresh is how long after a list was last fetched or tried it is
|
// Refresh is how long after a list was last fetched or tried it is
|
||||||
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
||||||
Refresh time.Duration
|
Refresh time.Duration
|
||||||
@@ -465,7 +480,11 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
|
|
||||||
netblock, ok := parseNetblock(text)
|
netblock, ok := parseNetblock(text)
|
||||||
if !ok {
|
if !ok {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
|
return entries{}, fmt.Errorf(
|
||||||
|
"line %d %w",
|
||||||
|
i+1,
|
||||||
|
errNotNetblockLine,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
found.netblocks[netblock] = true
|
found.netblocks[netblock] = true
|
||||||
@@ -485,7 +504,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
||||||
// stands for none, is refused.
|
// stands for none, is refused.
|
||||||
func parseNetblock(text string) (netip.Prefix, bool) {
|
func parseNetblock(text string) (netip.Prefix, bool) {
|
||||||
netblock, err := config.ParseNetblock(text)
|
netblock, err := ParseNetblock(text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netip.Prefix{}, false
|
return netip.Prefix{}, false
|
||||||
}
|
}
|
||||||
@@ -520,9 +539,9 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
asnText, percentText, _ := strings.Cut(text, ":")
|
asnText, percentText, _ := strings.Cut(text, ":")
|
||||||
asn, asnErr := config.ParseASN(asnText)
|
asn, asnErr := ParseASN(asnText)
|
||||||
|
|
||||||
percent, percentErr := config.ParsePercent(percentText)
|
percent, percentErr := ParsePercent(percentText)
|
||||||
if asnErr != nil || percentErr != nil {
|
if asnErr != nil || percentErr != nil {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
||||||
}
|
}
|
||||||
@@ -536,6 +555,53 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
return found, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||||
|
// address is a netblock of that address alone, a /32 or a /128. A
|
||||||
|
// blocklist's lines are read with it too.
|
||||||
|
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||||
|
if strings.Contains(value, "/") {
|
||||||
|
netblock, err := netip.ParsePrefix(value)
|
||||||
|
if err != nil {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netblock.Masked(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(value)
|
||||||
|
if err != nil || addr.Zone() != "" {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseASN reads an AS number such as AS64496, in either case, and
|
||||||
|
// returns it as the lookup gives it: AS and the number, in capitals and
|
||||||
|
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
||||||
|
// read with it too.
|
||||||
|
func ParseASN(value string) (string, error) {
|
||||||
|
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
||||||
|
|
||||||
|
number, err := strconv.ParseUint(digits, 10, 32)
|
||||||
|
if !hasAS || err != nil {
|
||||||
|
return "", fmt.Errorf("%q %w", value, errNotASN)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "AS" + strconv.FormatUint(number, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
||||||
|
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
||||||
|
func ParsePercent(value string) (int64, error) {
|
||||||
|
percent, err := strconv.ParseInt(value, 10, 64)
|
||||||
|
if err != nil || percent < 0 || percent > 100 {
|
||||||
|
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return percent, nil
|
||||||
|
}
|
||||||
|
|
||||||
// withoutComment returns line without anything after a ; or a #, and
|
// withoutComment returns line without anything after a ; or a #, and
|
||||||
// without the spaces around what is left.
|
// without the spaces around what is left.
|
||||||
func withoutComment(line string) string {
|
func withoutComment(line string) string {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the lines sww prints on stdout: one
|
// Package requestlog writes the request log sww prints on stdout: JSON
|
||||||
// JSON object per request, two from waf and a short one from rproxy,
|
// lines marked "type":"request", at most two from waf and a short one
|
||||||
// marked "type":"request", and the process's own messages as JSON lines marked
|
// from rproxy for each request. Package processlog writes the process's
|
||||||
// "type":"process".
|
// own messages.
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewProcessLogger returns the logger for the process's own messages:
|
|
||||||
// JSON lines on w, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
|
||||||
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
|
||||||
// the request lines Write writes are never held back.
|
|
||||||
func NewProcessLogger(
|
|
||||||
w io.Writer,
|
|
||||||
instanceName string,
|
|
||||||
level slog.Level,
|
|
||||||
) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
|
||||||
Level: level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", instanceName)
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := requestlog.NewProcessLogger(
|
|
||||||
&out,
|
|
||||||
"host1/blog",
|
|
||||||
level,
|
|
||||||
)
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -99,8 +99,12 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way.
|
// asking are the clients whose query is under way, each with the
|
||||||
asking map[netip.Prefix]bool
|
// function that ends it. stopped is true once StopAsking has been
|
||||||
|
// called, and queryGoroutines waits for the goroutine of each query.
|
||||||
|
asking map[netip.Prefix]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
queryGoroutines sync.WaitGroup
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -116,7 +120,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]bool{},
|
asking: map[netip.Prefix]context.CancelFunc{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,8 +128,9 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
||||||
// client's request, and a query goes on after the request ends.
|
// is the context of the client's request, and a query goes on after the request
|
||||||
|
// ends, until StopAsking.
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -147,15 +152,38 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
if !c.asking[client] && len(c.asking) < maxQueries {
|
_, asking := c.asking[client]
|
||||||
c.asking[client] = true
|
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
||||||
|
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
c.asking[client] = end
|
||||||
|
|
||||||
go c.ask(context.WithoutCancel(ctx), client, addr)
|
c.queryGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
c.ask(queryCtx, client, addr)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the queries under way, and returns once their goroutines
|
||||||
|
// have. A query it ends gives no answer and is no failure: it is not logged,
|
||||||
|
// and routewatch is not left alone after it. After StopAsking, Route asks
|
||||||
|
// routewatch nothing.
|
||||||
|
func (c *Client) StopAsking() {
|
||||||
|
c.mu.Lock()
|
||||||
|
|
||||||
|
c.stopped = true
|
||||||
|
for _, end := range c.asking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
c.queryGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged.
|
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
||||||
|
// failure.
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -213,18 +242,20 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := err != nil && !now.Before(c.retryAt)
|
firstFailure := isFailure && !now.Before(c.retryAt)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
} else {
|
case isFailure:
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
||||||
|
// says.
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
f := &fixture{t: t, log: &syncBuffer{}}
|
||||||
|
f.client = routewatch.New(routewatch.Params{
|
||||||
|
URL: silentURL, Now: time.Now,
|
||||||
|
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
||||||
|
})
|
||||||
|
|
||||||
|
f.route()
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The query is ended, not waited out.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
f.client.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A query started now would fail within waitLimit. None is, and the
|
||||||
|
// one ended is no failure: nothing is logged, and routewatch is not
|
||||||
|
// left alone.
|
||||||
|
for range 2 {
|
||||||
|
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
||||||
|
t.Errorf(
|
||||||
|
"Route gave %q, want %q",
|
||||||
|
missing,
|
||||||
|
routewatch.NoAnswerYet,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(waitLimit)
|
||||||
|
synctest.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.log.String() != "" {
|
||||||
|
t.Errorf("process log %q, want nothing", f.log.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -7,16 +7,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The headers rproxy sets on the requests it passes to the app. Go writes every
|
|
||||||
// header name in this form, as it sends it and as it receives it, so
|
|
||||||
// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever
|
|
||||||
// their case; header names are not case-sensitive.
|
|
||||||
const (
|
|
||||||
requestIDHeader = "X-Request-ID"
|
|
||||||
asnHeader = "X-Client-Asn"
|
|
||||||
countryHeader = "X-Client-Country"
|
|
||||||
)
|
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer, normally traefik, without
|
// peerAddress is the address of the request's TCP peer, normally traefik, without
|
||||||
// any IPv6 zone, which waf refuses.
|
// any IPv6 zone, which waf refuses.
|
||||||
func peerAddress(r *http.Request) netip.Addr {
|
func peerAddress(r *http.Request) netip.Addr {
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ package rproxy
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
|
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
||||||
|
// long requests take cannot matter to it. It is called before s serves.
|
||||||
|
func (s *Server) StopBreakerClock(at time.Time) {
|
||||||
|
s.h.breakerClock = func() time.Time { return at }
|
||||||
|
}
|
||||||
|
|
||||||
// Breaker is the circuit breaker, for tests that give it the time.
|
// Breaker is the circuit breaker, for tests that give it the time.
|
||||||
type Breaker struct {
|
type Breaker struct {
|
||||||
b breaker
|
b breaker
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// post is the start of a POST whose headers end with what follows it.
|
// post is the start of a POST whose headers end with what follows it.
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
@@ -492,7 +492,7 @@ func TestASettingOfOffSetsNoLimit(t *testing.T) {
|
|||||||
strings.NewReader("a body"))),
|
strings.NewReader("a body"))),
|
||||||
)
|
)
|
||||||
|
|
||||||
if got := get(t, addr, waf.HealthPath); got.status != http.StatusOK {
|
if got := get(t, addr, wafprotocol.HealthPath); got.status != http.StatusOK {
|
||||||
t.Errorf("answered %+v, want rproxy's own 200", got)
|
t.Errorf("answered %+v, want rproxy's own 200", got)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -870,10 +870,14 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
|
|||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// rproxy is waiting for the rest of the body well within this limit, even on
|
||||||
|
// a busy host, and answers when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "3000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1262,7 +1266,9 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const limit = 2 * time.Second
|
// The app's answer arrives well within this limit, even on a busy host, and
|
||||||
|
// the connection closes when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, answerBeforeTheBody)
|
app := startApp(t, answerBeforeTheBody)
|
||||||
@@ -1427,17 +1433,20 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
|
|||||||
|
|
||||||
const answer = "the end of a long answer"
|
const answer = "the end of a long answer"
|
||||||
|
|
||||||
|
// The request reaches the app well within this limit, even on a busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
_ = http.NewResponseController(w).Flush()
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
_, _ = io.WriteString(w, answer)
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
got := visit(
|
got := visit(
|
||||||
@@ -1471,7 +1480,7 @@ func TestAVisitorThatStopsALittleShortGetsAnAnswerRProxyGivesAtOnce(
|
|||||||
{"a deny", post, http.StatusForbidden},
|
{"a deny", post, http.StatusForbidden},
|
||||||
{
|
{
|
||||||
"the health check",
|
"the health check",
|
||||||
"GET " + waf.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
"GET " + wafprotocol.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -12,11 +12,11 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// questionBodyMaxBytes is the most of a body a question carries.
|
// questionBodyMaxBytes is the most of a body a question carries.
|
||||||
const questionBodyMaxBytes = waf.QuestionBodyMaxBytes
|
const questionBodyMaxBytes = wafprotocol.QuestionBodyMaxBytes
|
||||||
|
|
||||||
// chunked is the encoding of a body whose length the visitor did not announce.
|
// chunked is the encoding of a body whose length the visitor did not announce.
|
||||||
const chunked = "chunked"
|
const chunked = "chunked"
|
||||||
@@ -42,9 +42,9 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
|||||||
req.Header.Set(forwardedForHeader, "198.51.100.1")
|
req.Header.Set(forwardedForHeader, "198.51.100.1")
|
||||||
req.Header.Set(forwardedProtoHeader, "gopher")
|
req.Header.Set(forwardedProtoHeader, "gopher")
|
||||||
req.Header.Set(requestIDHeader, "the-visitor's")
|
req.Header.Set(requestIDHeader, "the-visitor's")
|
||||||
req.Header.Set(waf.ClientHeader, "198.51.100.2")
|
req.Header.Set(wafprotocol.ClientHeader, "198.51.100.2")
|
||||||
req.Header.Set(waf.PeerHeader, "198.51.100.3")
|
req.Header.Set(wafprotocol.PeerHeader, "198.51.100.3")
|
||||||
req.Header.Set(waf.ContentLengthHeader, "12345")
|
req.Header.Set(wafprotocol.ContentLengthHeader, "12345")
|
||||||
// Go's client then sends no User-Agent, so that one in the question would be
|
// Go's client then sends no User-Agent, so that one in the question would be
|
||||||
// Go's own.
|
// Go's own.
|
||||||
req.Header["User-Agent"] = nil
|
req.Header["User-Agent"] = nil
|
||||||
@@ -73,8 +73,8 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
|||||||
want.Set(forwardedForHeader, "198.51.100.1")
|
want.Set(forwardedForHeader, "198.51.100.1")
|
||||||
want.Set(forwardedProtoHeader, "http")
|
want.Set(forwardedProtoHeader, "http")
|
||||||
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
|
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
|
||||||
want.Set(waf.ClientHeader, localhost)
|
want.Set(wafprotocol.ClientHeader, localhost)
|
||||||
want.Set(waf.PeerHeader, localhost)
|
want.Set(wafprotocol.PeerHeader, localhost)
|
||||||
|
|
||||||
if !maps.EqualFunc(asked.header, want, slices.Equal) {
|
if !maps.EqualFunc(asked.header, want, slices.Equal) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -124,8 +124,8 @@ func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) {
|
|||||||
wantGreeting(t, visit(t, req))
|
wantGreeting(t, visit(t, req))
|
||||||
|
|
||||||
asked := standIn.question(t)
|
asked := standIn.question(t)
|
||||||
if asked.header.Get(waf.ClientHeader) != tc.client ||
|
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
|
||||||
asked.header.Get(waf.PeerHeader) != localhost ||
|
asked.header.Get(wafprotocol.PeerHeader) != localhost ||
|
||||||
asked.header.Get(forwardedForHeader) != visitor ||
|
asked.header.Get(forwardedForHeader) != visitor ||
|
||||||
asked.header.Get(forwardedProtoHeader) != tc.proto {
|
asked.header.Get(forwardedProtoHeader) != tc.proto {
|
||||||
t.Errorf("the question's headers are %v", asked.header)
|
t.Errorf("the question's headers are %v", asked.header)
|
||||||
@@ -191,7 +191,7 @@ func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) {
|
|||||||
req.Header[forwardedForHeader] = tc.header
|
req.Header[forwardedForHeader] = tc.header
|
||||||
wantGreeting(t, visit(t, req))
|
wantGreeting(t, visit(t, req))
|
||||||
|
|
||||||
if got := standIn.question(t).header.Get(waf.ClientHeader); got !=
|
if got := standIn.question(t).header.Get(wafprotocol.ClientHeader); got !=
|
||||||
tc.client {
|
tc.client {
|
||||||
t.Errorf("the question names %s, want %s", got, tc.client)
|
t.Errorf("the question names %s, want %s", got, tc.client)
|
||||||
}
|
}
|
||||||
@@ -282,7 +282,7 @@ func wantQuestionBody(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
values := asked.header.Values(waf.ContentLengthHeader)
|
values := asked.header.Values(wafprotocol.ContentLengthHeader)
|
||||||
|
|
||||||
if announced {
|
if announced {
|
||||||
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
|
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
|
||||||
@@ -586,8 +586,8 @@ func TestAQuestionNamesNoZone(t *testing.T) {
|
|||||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|
||||||
asked := standIn.question(t)
|
asked := standIn.question(t)
|
||||||
if asked.header.Get(waf.ClientHeader) != tc.client ||
|
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
|
||||||
asked.header.Get(waf.PeerHeader) != tc.peer {
|
asked.header.Get(wafprotocol.PeerHeader) != tc.peer {
|
||||||
t.Errorf("the question's headers are %v", asked.header)
|
t.Errorf("the question's headers are %v", asked.header)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+11
-13
@@ -18,17 +18,13 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
|
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
|
||||||
// soon as it arrives.
|
// soon as it arrives.
|
||||||
const flushAfterEachWrite time.Duration = -1
|
const flushAfterEachWrite time.Duration = -1
|
||||||
|
|
||||||
// reportStatusMax is the highest status waf takes in a report. An app can
|
|
||||||
// send a higher one, up to 999, and the visitor gets that as it is.
|
|
||||||
const reportStatusMax = 599
|
|
||||||
|
|
||||||
// plainText is the Content-Type of the answers rproxy gives itself.
|
// plainText is the Content-Type of the answers rproxy gives itself.
|
||||||
const plainText = "text/plain; charset=utf-8"
|
const plainText = "text/plain; charset=utf-8"
|
||||||
|
|
||||||
@@ -154,7 +150,7 @@ func (rq *request) readFirstPartOfBody() bool {
|
|||||||
|
|
||||||
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
||||||
held, err := io.ReadAll(
|
held, err := io.ReadAll(
|
||||||
io.LimitReader(rq.body, waf.QuestionBodyMaxBytes),
|
io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
|
||||||
)
|
)
|
||||||
// From here on, the request timer ends a read of the rest of the body, so that
|
// From here on, the request timer ends a read of the rest of the body, so that
|
||||||
// the answer can name the side rproxy was waiting on.
|
// the answer can name the side rproxy was waiting on.
|
||||||
@@ -228,16 +224,17 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
|||||||
setPathAsSent(pr.Out.URL, pr.In)
|
setPathAsSent(pr.Out.URL, pr.In)
|
||||||
|
|
||||||
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
||||||
pr.Out.Header.Set(requestIDHeader, rq.id)
|
pr.Out.Header.Set(wafprotocol.RequestIDHeader, rq.id)
|
||||||
pr.Out.Header.Del(asnHeader)
|
// Del removes a visitor's own, whatever their case.
|
||||||
pr.Out.Header.Del(countryHeader)
|
pr.Out.Header.Del(wafprotocol.ASNHeader)
|
||||||
|
pr.Out.Header.Del(wafprotocol.CountryHeader)
|
||||||
|
|
||||||
if rq.asn != "" {
|
if rq.asn != "" {
|
||||||
pr.Out.Header.Set(asnHeader, rq.asn)
|
pr.Out.Header.Set(wafprotocol.ASNHeader, rq.asn)
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.country != "" {
|
if rq.country != "" {
|
||||||
pr.Out.Header.Set(countryHeader, rq.country)
|
pr.Out.Header.Set(wafprotocol.CountryHeader, rq.country)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -470,9 +467,10 @@ func (rq *request) finish(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
report := &waf.Report{
|
// An app can send a status up to 999, and the visitor gets it as it is.
|
||||||
|
report := &wafprotocol.Report{
|
||||||
RequestID: rq.id,
|
RequestID: rq.id,
|
||||||
Status: min(rq.out.status, reportStatusMax),
|
Status: min(rq.out.status, wafprotocol.ReportStatusMax),
|
||||||
RequestBytes: requestBytes,
|
RequestBytes: requestBytes,
|
||||||
ResponseBytes: responseBytes,
|
ResponseBytes: responseBytes,
|
||||||
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
|
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
|
||||||
|
|||||||
+15
-14
@@ -24,7 +24,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// How rproxy keeps connections to the app and to waf open between
|
// How rproxy keeps connections to the app and to waf open between
|
||||||
@@ -47,10 +47,6 @@ const (
|
|||||||
idleTimeout = 120 * time.Second
|
idleTimeout = 120 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// ownEndpointPrefix starts the path of every request for sww itself, which
|
|
||||||
// never reaches the app.
|
|
||||||
const ownEndpointPrefix = "/_sww/"
|
|
||||||
|
|
||||||
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
|
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
|
||||||
// them. It answers any other method 405 itself.
|
// them. It answers any other method 405 itself.
|
||||||
// The WebDAV ones are for Nextcloud.
|
// The WebDAV ones are for Nextcloud.
|
||||||
@@ -92,8 +88,11 @@ func New(params Params) *Server {
|
|||||||
transport: newTransport(),
|
transport: newTransport(),
|
||||||
wafTransport: newWAFTransport(),
|
wafTransport: newWAFTransport(),
|
||||||
reportURL: (&url.URL{
|
reportURL: (&url.URL{
|
||||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
Scheme: backend.Scheme,
|
||||||
|
Host: backend.Host,
|
||||||
|
Path: wafprotocol.ReportPath,
|
||||||
}).String(),
|
}).String(),
|
||||||
|
breakerClock: time.Now,
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Server{
|
return &Server{
|
||||||
@@ -133,8 +132,10 @@ type handler struct {
|
|||||||
wafTransport http.RoundTripper
|
wafTransport http.RoundTripper
|
||||||
// reportURL is where the reports go.
|
// reportURL is where the reports go.
|
||||||
reportURL string
|
reportURL string
|
||||||
// breaker is the circuit breaker for asks to waf.
|
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
||||||
breaker breaker
|
// the time.
|
||||||
|
breaker breaker
|
||||||
|
breakerClock func() time.Time
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
// reports waits for the reports being sent. reportsMu guards starting one
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
// against reportsStopped, which is set once nothing is to wait for another.
|
||||||
reports sync.WaitGroup
|
reports sync.WaitGroup
|
||||||
@@ -176,7 +177,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
defer rq.readRestOfBody()
|
defer rq.readRestOfBody()
|
||||||
defer rq.finish(r.Context())
|
defer rq.finish(r.Context())
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath {
|
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
||||||
rq.answerHealth()
|
rq.answerHealth()
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -209,9 +210,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
rq.noteAnswer(answer)
|
rq.noteAnswer(answer)
|
||||||
|
|
||||||
switch rq.disposition {
|
switch rq.disposition {
|
||||||
case waf.DispositionAllow:
|
case wafprotocol.DispositionAllow:
|
||||||
rq.passToApp(r.Context())
|
rq.passToApp(r.Context())
|
||||||
case waf.DispositionClose:
|
case wafprotocol.DispositionClose:
|
||||||
// This panic has Go's server close the connection without an answer,
|
// This panic has Go's server close the connection without an answer,
|
||||||
// and log nothing; the deferred finish still runs.
|
// and log nothing; the deferred finish still runs.
|
||||||
panic(http.ErrAbortHandler)
|
panic(http.ErrAbortHandler)
|
||||||
@@ -221,8 +222,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// passToApp passes the request on to the app, unless a limit or its path keeps
|
// passToApp passes the request on to the app, unless a limit or its path keeps
|
||||||
// it from the app. waf answers a path under ownEndpointPrefix itself, so
|
// it from the app. waf answers a path under wafprotocol.OwnEndpointPrefix
|
||||||
// such a request gets here only when
|
// itself, so such a request gets here only when
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
|
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
|
||||||
// ctx is the request's own context.
|
// ctx is the request's own context.
|
||||||
func (rq *request) passToApp(ctx context.Context) {
|
func (rq *request) passToApp(ctx context.Context) {
|
||||||
@@ -231,7 +232,7 @@ func (rq *request) passToApp(ctx context.Context) {
|
|||||||
switch {
|
switch {
|
||||||
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
|
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
|
||||||
rq.answer(http.StatusRequestEntityTooLarge)
|
rq.answer(http.StatusRequestEntityTooLarge)
|
||||||
case strings.HasPrefix(rq.in.URL.Path, ownEndpointPrefix):
|
case strings.HasPrefix(rq.in.URL.Path, wafprotocol.OwnEndpointPrefix):
|
||||||
rq.answer(http.StatusNotFound)
|
rq.answer(http.StatusNotFound)
|
||||||
default:
|
default:
|
||||||
rq.forward(ctx)
|
rq.forward(ctx)
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -56,7 +56,7 @@ const (
|
|||||||
|
|
||||||
// The headers the tests set and look at.
|
// The headers the tests set and look at.
|
||||||
const (
|
const (
|
||||||
dispositionHeader = waf.DispositionHeader
|
dispositionHeader = wafprotocol.DispositionHeader
|
||||||
forwardedForHeader = "X-Forwarded-For"
|
forwardedForHeader = "X-Forwarded-For"
|
||||||
forwardedProtoHeader = "X-Forwarded-Proto"
|
forwardedProtoHeader = "X-Forwarded-Proto"
|
||||||
requestIDHeader = "X-Request-ID"
|
requestIDHeader = "X-Request-ID"
|
||||||
@@ -72,9 +72,9 @@ const (
|
|||||||
|
|
||||||
// What the stand-in for waf answers.
|
// What the stand-in for waf answers.
|
||||||
const (
|
const (
|
||||||
dispositionAllow = waf.DispositionAllow
|
dispositionAllow = wafprotocol.DispositionAllow
|
||||||
dispositionDeny = waf.DispositionDeny
|
dispositionDeny = wafprotocol.DispositionDeny
|
||||||
dispositionClose = waf.DispositionClose
|
dispositionClose = wafprotocol.DispositionClose
|
||||||
// deniedBody is the body of a deny.
|
// deniedBody is the body of a deny.
|
||||||
deniedBody = "refused by waf"
|
deniedBody = "refused by waf"
|
||||||
)
|
)
|
||||||
@@ -175,7 +175,7 @@ type WAF struct {
|
|||||||
*httptest.Server
|
*httptest.Server
|
||||||
|
|
||||||
questions chan question
|
questions chan question
|
||||||
reports chan waf.Report
|
reports chan wafprotocol.Report
|
||||||
}
|
}
|
||||||
|
|
||||||
// startWAF starts a stand-in for waf.
|
// startWAF starts a stand-in for waf.
|
||||||
@@ -184,11 +184,11 @@ func startWAF(t *testing.T, answer http.HandlerFunc) *WAF {
|
|||||||
|
|
||||||
s := &WAF{
|
s := &WAF{
|
||||||
questions: make(chan question, 10),
|
questions: make(chan question, 10),
|
||||||
reports: make(chan waf.Report, 10),
|
reports: make(chan wafprotocol.Report, 10),
|
||||||
}
|
}
|
||||||
s.Server = httptest.NewServer(http.HandlerFunc(
|
s.Server = httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(waf.ClientHeader) == "" {
|
if r.Header.Get(wafprotocol.ClientHeader) == "" {
|
||||||
s.takeReport(t, w, r)
|
s.takeReport(t, w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -217,11 +217,11 @@ func (s *WAF) takeReport(
|
|||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var report waf.Report
|
var report wafprotocol.Report
|
||||||
|
|
||||||
err := json.NewDecoder(r.Body).Decode(&report)
|
err := json.NewDecoder(r.Body).Decode(&report)
|
||||||
if err != nil || r.Method != http.MethodPost ||
|
if err != nil || r.Method != http.MethodPost ||
|
||||||
r.URL.Path != waf.ReportPath {
|
r.URL.Path != wafprotocol.ReportPath {
|
||||||
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
|
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,7 +242,7 @@ func (s *WAF) wantNoReport(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// report waits for a report.
|
// report waits for a report.
|
||||||
func (s *WAF) report(t *testing.T) waf.Report {
|
func (s *WAF) report(t *testing.T) wafprotocol.Report {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -251,7 +251,7 @@ func (s *WAF) report(t *testing.T) waf.Report {
|
|||||||
case <-time.After(waitLimit):
|
case <-time.After(waitLimit):
|
||||||
t.Fatal("no report")
|
t.Fatal("no report")
|
||||||
|
|
||||||
return waf.Report{}
|
return wafprotocol.Report{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -279,7 +279,7 @@ func startWAFAnsweringReports(
|
|||||||
|
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(waf.ClientHeader) != "" {
|
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||||
answer(w, r)
|
answer(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -356,6 +356,19 @@ func startServerLogging(
|
|||||||
) (string, *rproxy.Server) {
|
) (string, *rproxy.Server) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
||||||
|
|
||||||
|
return serve(t, server), server
|
||||||
|
}
|
||||||
|
|
||||||
|
// newServer returns rproxy, not yet serving, with its request lines going to
|
||||||
|
// requestLog and its process lines to processLog.
|
||||||
|
func newServer(
|
||||||
|
t *testing.T, backend, appURL string, env map[string]string,
|
||||||
|
requestLog, processLog io.Writer,
|
||||||
|
) *rproxy.Server {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
wafBackendTimeout: "60000",
|
wafBackendTimeout: "60000",
|
||||||
@@ -367,15 +380,20 @@ func startServerLogging(
|
|||||||
t.Fatalf("settings %v: %v", settings, err)
|
t.Fatalf("settings %v: %v", settings, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
return rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
processLog,
|
Output: processLog,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve has server serve until the test ends, and returns its address.
|
||||||
|
func serve(t *testing.T, server *rproxy.Server) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
t.Context(),
|
t.Context(),
|
||||||
@@ -396,7 +414,7 @@ func startServerLogging(
|
|||||||
server.WaitForReports()
|
server.WaitForReports()
|
||||||
})
|
})
|
||||||
|
|
||||||
return listener.Addr().String(), server
|
return listener.Addr().String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// answered is the visitor's answer.
|
// answered is the visitor's answer.
|
||||||
@@ -811,7 +829,7 @@ func TestAnAnswerRProxyHoldsWholeIsSentBeforeItsLogLine(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
|
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
|
||||||
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
|
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
|
||||||
{"the health check", "GET " + waf.HealthPath + host, "ok\n", http.StatusOK},
|
{"the health check", "GET " + wafprotocol.HealthPath + host, "ok\n", http.StatusOK},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1434,14 +1452,18 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|||||||
|
|
||||||
greet(w, r)
|
greet(w, r)
|
||||||
})
|
})
|
||||||
addr, out := startRProxy(
|
out := &output{}
|
||||||
t,
|
server := newServer(
|
||||||
standIn.URL,
|
t, standIn.URL, app.URL,
|
||||||
app.URL,
|
map[string]string{wafBackendTimeout: "200"},
|
||||||
map[string]string{
|
out, out,
|
||||||
wafBackendTimeout: "200",
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
// The breaker's clock stands still, as in
|
||||||
|
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
|
||||||
|
// failures could take more than five seconds, and the next request could
|
||||||
|
// come more than a second later.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
// An ask can time out before it reaches the stand-in, so these
|
||||||
// may leave fewer than five questions, or leave some late.
|
// may leave fewer than five questions, or leave some late.
|
||||||
@@ -1532,9 +1554,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
|||||||
tooLarge(w, r)
|
tooLarge(w, r)
|
||||||
})
|
})
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
out := &output{}
|
||||||
wafBackendTimeout: "60000",
|
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
||||||
})
|
// The breaker's clock stands still. On a slow host, the five failures could
|
||||||
|
// otherwise take more than the five seconds that open the breaker. The next
|
||||||
|
// request could come more than a second later, when it may ask waf again.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
for range 5 {
|
for range 5 {
|
||||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||||
@@ -1576,19 +1602,19 @@ func TestHealthCheckIsAnsweredWhileWAFIsDown(t *testing.T) {
|
|||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, nowhere, app.URL, nil)
|
addr, out := startRProxy(t, nowhere, app.URL, nil)
|
||||||
|
|
||||||
got := get(t, addr, waf.HealthPath)
|
got := get(t, addr, wafprotocol.HealthPath)
|
||||||
if got.status != http.StatusOK || got.body != "ok\n" {
|
if got.status != http.StatusOK || got.body != "ok\n" {
|
||||||
t.Errorf("answered %+v, want ok", got)
|
t.Errorf("answered %+v, want ok", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["path"] != waf.HealthPath || line["disposition"] != nil {
|
if line["path"] != wafprotocol.HealthPath || line["disposition"] != nil {
|
||||||
t.Errorf("line %v, want one with no disposition", line)
|
t.Errorf("line %v, want one with no disposition", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
// With waf up too, it is not asked.
|
// With waf up too, it is not asked.
|
||||||
addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
|
addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
|
||||||
get(t, addr, waf.HealthPath)
|
get(t, addr, wafprotocol.HealthPath)
|
||||||
|
|
||||||
if len(standIn.questions) != 0 {
|
if len(standIn.questions) != 0 {
|
||||||
t.Error("waf was asked about the health check")
|
t.Error("waf was asked about the health check")
|
||||||
@@ -1749,7 +1775,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(waf.ClientHeader) != "" {
|
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||||
allow(w, r)
|
allow(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -1809,11 +1835,15 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// The app's answer to the upgrade arrives well within these limits, even on a
|
||||||
|
// busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, echoAfterUpgrade)
|
app := startApp(t, echoAfterUpgrade)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
maxResponseTime: "2000",
|
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1838,7 +1868,10 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Past both time limits, the connection still carries what is sent.
|
// Past both time limits, the connection still carries what is sent.
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
|
// dial's deadline ran from before the upgrade and this wait.
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||||
|
|
||||||
const message = "still here\n"
|
const message = "still here\n"
|
||||||
|
|
||||||
|
|||||||
+36
-85
@@ -1,18 +1,16 @@
|
|||||||
package rproxy
|
package rproxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"maps"
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// answerBodyMaxBytes is the most of the body of waf's answer that is read.
|
// answerBodyMaxBytes is the most of the body of waf's answer that is read.
|
||||||
@@ -41,15 +39,12 @@ type wafAnswer struct {
|
|||||||
// notes nothing, and no report is due for it: waf holds no decision to report
|
// notes nothing, and no report is due for it: waf holds no decision to report
|
||||||
// on.
|
// on.
|
||||||
func (rq *request) noteAnswer(answer *wafAnswer) {
|
func (rq *request) noteAnswer(answer *wafAnswer) {
|
||||||
disposition := answer.header.Get(waf.DispositionHeader)
|
read, valid := wafprotocol.ReadAnswer(answer.header)
|
||||||
|
if valid {
|
||||||
switch disposition {
|
rq.disposition = read.Disposition
|
||||||
case waf.DispositionAllow, waf.DispositionDeny,
|
|
||||||
waf.DispositionClose:
|
|
||||||
rq.disposition = disposition
|
|
||||||
rq.answered = true
|
rq.answered = true
|
||||||
rq.asn = answer.header.Get(asnHeader)
|
rq.asn = read.ASN
|
||||||
rq.country = answer.header.Get(countryHeader)
|
rq.country = read.Country
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -80,7 +75,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
|
|||||||
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
|
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
|
||||||
rq.wafNoAnswer = true
|
rq.wafNoAnswer = true
|
||||||
|
|
||||||
if rq.disposition == waf.DispositionAllow {
|
if rq.disposition == wafprotocol.DispositionAllow {
|
||||||
rq.passToApp(ctx)
|
rq.passToApp(ctx)
|
||||||
} else {
|
} else {
|
||||||
rq.answer(http.StatusServiceUnavailable)
|
rq.answer(http.StatusServiceUnavailable)
|
||||||
@@ -97,7 +92,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
) (*wafAnswer, error) {
|
) (*wafAnswer, error) {
|
||||||
b := &rq.h.breaker
|
b := &rq.h.breaker
|
||||||
if !b.mayAsk(time.Now()) {
|
if !b.mayAsk(rq.h.breakerClock()) {
|
||||||
return nil, errBreakerOpen
|
return nil, errBreakerOpen
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,7 +102,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
case err == nil:
|
case err == nil:
|
||||||
b.noteAnswer()
|
b.noteAnswer()
|
||||||
case ctx.Err() == nil:
|
case ctx.Err() == nil:
|
||||||
b.noteFailure(time.Now())
|
b.noteFailure(rq.h.breakerClock())
|
||||||
}
|
}
|
||||||
|
|
||||||
return answer, err
|
return answer, err
|
||||||
@@ -120,7 +115,21 @@ func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
|
|||||||
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
|
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
question, err := rq.newQuestion(ctx)
|
var held []byte
|
||||||
|
if rq.body != nil {
|
||||||
|
held = rq.body.held
|
||||||
|
}
|
||||||
|
|
||||||
|
question, err := wafprotocol.NewQuestion(ctx, wafprotocol.QuestionParams{
|
||||||
|
Visitor: rq.in,
|
||||||
|
PathAsSent: rules.Path(rq.in),
|
||||||
|
Body: held,
|
||||||
|
WAFBackend: rq.h.config.WAFBackend,
|
||||||
|
Client: rq.client,
|
||||||
|
Peer: rq.peer,
|
||||||
|
RequestID: rq.id,
|
||||||
|
Scheme: scheme(rq.in, rq.peerTrusted),
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -177,69 +186,13 @@ func canBeAsked(r *http.Request) bool {
|
|||||||
len(r.Header.Values("User-Agent")) <= 1
|
len(r.Header.Values("User-Agent")) <= 1
|
||||||
}
|
}
|
||||||
|
|
||||||
// newQuestion returns the question about the request, as "The question" in
|
|
||||||
// SPEC.md gives it: the visitor's request as the visitor sent it, with the part
|
|
||||||
// of the body it holds and rproxy's headers.
|
|
||||||
func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) {
|
|
||||||
in := rq.in
|
|
||||||
|
|
||||||
var held []byte
|
|
||||||
if rq.body != nil {
|
|
||||||
held = rq.body.held
|
|
||||||
}
|
|
||||||
|
|
||||||
question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
|
||||||
ctx,
|
|
||||||
in.Method,
|
|
||||||
rq.h.config.WAFBackend.String(),
|
|
||||||
bytes.NewReader(held),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("make the question: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
backend := question.URL
|
|
||||||
target := *in.URL
|
|
||||||
target.Scheme = backend.Scheme
|
|
||||||
target.Host = backend.Host
|
|
||||||
setPathAsSent(&target, in)
|
|
||||||
question.URL = &target
|
|
||||||
question.Host = in.Host
|
|
||||||
|
|
||||||
// The length NewRequestWithContext set is that of what is held. This line makes
|
|
||||||
// a body whose length the visitor did not announce go chunked, as "The question"
|
|
||||||
// in SPEC.md requires.
|
|
||||||
question.TransferEncoding = in.TransferEncoding
|
|
||||||
|
|
||||||
question.Header = in.Header.Clone()
|
|
||||||
question.Header.Set(waf.ClientHeader, rq.client.String())
|
|
||||||
question.Header.Set(waf.PeerHeader, rq.peer.String())
|
|
||||||
question.Header.Del(waf.ContentLengthHeader)
|
|
||||||
|
|
||||||
announced := in.Header.Get("Content-Length")
|
|
||||||
if announced != "" {
|
|
||||||
question.Header.Set(waf.ContentLengthHeader, announced)
|
|
||||||
}
|
|
||||||
|
|
||||||
question.Header.Set(requestIDHeader, rq.id)
|
|
||||||
question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted))
|
|
||||||
|
|
||||||
// Go's client adds a User-Agent of its own to a request without the key.
|
|
||||||
_, sent := question.Header["User-Agent"]
|
|
||||||
if !sent {
|
|
||||||
question.Header["User-Agent"] = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return question, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerWith gives the visitor waf's answer, less its disposition.
|
// answerWith gives the visitor waf's answer, less its disposition.
|
||||||
func (rq *request) answerWith(answer *wafAnswer) {
|
func (rq *request) answerWith(answer *wafAnswer) {
|
||||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
rq.prepareOwnAnswer(rq.requestDeadline())
|
||||||
|
|
||||||
header := rq.out.Header()
|
header := rq.out.Header()
|
||||||
maps.Copy(header, answer.header)
|
maps.Copy(header, answer.header)
|
||||||
header.Del(waf.DispositionHeader)
|
header.Del(wafprotocol.DispositionHeader)
|
||||||
|
|
||||||
// Go's server adds a Content-Type it guesses from the body to a response whose
|
// Go's server adds a Content-Type it guesses from the body to a response whose
|
||||||
// headers have no Content-Type key. The key with no value sends none.
|
// headers have no Content-Type key. The key with no value sends none.
|
||||||
@@ -253,7 +206,10 @@ func (rq *request) answerWith(answer *wafAnswer) {
|
|||||||
// sendReport sends waf report on a goroutine of its own, so that the
|
// sendReport sends waf report on a goroutine of its own, so that the
|
||||||
// visitor does not wait for it. The report goes on after ctx, the request's own
|
// visitor does not wait for it. The report goes on after ctx, the request's own
|
||||||
// context, has ended.
|
// context, has ended.
|
||||||
func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
func (h *handler) sendReport(
|
||||||
|
ctx context.Context,
|
||||||
|
report *wafprotocol.Report,
|
||||||
|
) {
|
||||||
h.reportsMu.Lock()
|
h.reportsMu.Lock()
|
||||||
defer h.reportsMu.Unlock()
|
defer h.reportsMu.Unlock()
|
||||||
|
|
||||||
@@ -271,23 +227,18 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
|||||||
|
|
||||||
// postReport posts report to waf, and waits for its answer for up to
|
// postReport posts report to waf, and waits for its answer for up to
|
||||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
||||||
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
|
func (h *handler) postReport(
|
||||||
body, err := json.Marshal(report)
|
ctx context.Context,
|
||||||
if err != nil {
|
report *wafprotocol.Report,
|
||||||
return fmt.Errorf("encode the report: %w", err)
|
) error {
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
|
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
req, err := wafprotocol.NewReportRequest(ctx, h.reportURL, report)
|
||||||
ctx, http.MethodPost, h.reportURL, bytes.NewReader(body))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("make the request: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
res, err := h.wafTransport.RoundTrip(req)
|
res, err := h.wafTransport.RoundTrip(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("send the report: %w", err)
|
return fmt.Errorf("send the report: %w", err)
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
"github.com/fsnotify/fsnotify"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
|
|||||||
rule.Target,
|
rule.Target,
|
||||||
errNotTarget,
|
errNotTarget,
|
||||||
)
|
)
|
||||||
case isHeader && !config.IsHeaderName(headerName):
|
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
|
||||||
return Rule{}, false, fmt.Errorf(
|
return Rule{}, false, fmt.Errorf(
|
||||||
"the target %q %w",
|
"the target %q %w",
|
||||||
rule.Target,
|
rule.Target,
|
||||||
|
|||||||
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
|
|||||||
|
|
||||||
// A second ban, made while the first waits to be written, puts the
|
// A second ban, made while the first waits to be written, puts the
|
||||||
// write off no further, and is written with it.
|
// write off no further, and is written with it.
|
||||||
first, _ := params.Ledger.BanForLimit(
|
first, _ := params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
second, _ := params.Ledger.BanForLimit(
|
second, _ := params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5*time.Second - time.Nanosecond)
|
time.Sleep(5*time.Second - time.Nanosecond)
|
||||||
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
|
|||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// A ban, and bans.json written with it.
|
// A ban, and bans.json written with it.
|
||||||
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
|
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(), bans.Notes{})
|
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
time.Sleep(params.BansWriteDelay)
|
time.Sleep(params.BansWriteDelay)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
|
|
||||||
// A second ban is to be written BansWriteDelay later. Just before
|
// A second ban is to be written BansWriteDelay later. Just before
|
||||||
// then, an admin saves bans.json with the first ban lifted and
|
// then, an admin saves bans.json with the first ban lifted and
|
||||||
// another added.
|
// another added.
|
||||||
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
|
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(), bans.Notes{})
|
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
||||||
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
edit(t, dir, bansJSON, permanentBansJSON)
|
edit(t, dir, bansJSON, permanentBansJSON)
|
||||||
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.BanForLimit(
|
params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
params.Limiter.Count(
|
params.Limiter.Count(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.BanForLimit(
|
params.Ledger.Ban(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
|
|
||||||
err = files.WriteAll()
|
err = files.WriteAll()
|
||||||
@@ -2135,7 +2135,11 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
|||||||
|
|
||||||
// The ban that liftedBansJSON lifts, before it is lifted.
|
// The ban that liftedBansJSON lifts, before it is lifted.
|
||||||
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
||||||
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
params.Ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
|
|
||||||
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
||||||
if !banned {
|
if !banned {
|
||||||
@@ -2448,7 +2452,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
@@ -2510,11 +2514,15 @@ func fill(params state.Params) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
params.Ledger.Load([]bans.Ban{permanentBan()})
|
params.Ledger.Load([]bans.Ban{permanentBan()})
|
||||||
params.Ledger.BanForLimit(client, now, bans.Notes{
|
params.Ledger.Ban(client, now, bans.Grounds{
|
||||||
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
|
Cause: bans.CauseLimit,
|
||||||
|
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
|
||||||
})
|
})
|
||||||
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
|
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
|
||||||
bans.Notes{RuleID: "env-file", Target: "path"})
|
bans.Grounds{
|
||||||
|
Cause: bans.CauseAttack,
|
||||||
|
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
|
||||||
|
})
|
||||||
|
|
||||||
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
||||||
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
||||||
@@ -2700,7 +2708,11 @@ func wantLiftedBanKept(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Were the lifted ban counted, the next would last three hours.
|
// Were the lifted ban counted, the next would last three hours.
|
||||||
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
|
ban, _ := ledger.Ban(
|
||||||
|
netblock,
|
||||||
|
afterLifting(),
|
||||||
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour {
|
if ban.Expires.Sub(ban.Start) != time.Hour {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban lasts %s, want 1h",
|
"the next ban lasts %s, want 1h",
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// healthCheckTimeout bounds the whole health check.
|
// healthCheckTimeout bounds the whole health check.
|
||||||
@@ -116,7 +116,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
|
|||||||
health := "http://" + net.JoinHostPort(
|
health := "http://" + net.JoinHostPort(
|
||||||
"127.0.0.1",
|
"127.0.0.1",
|
||||||
port,
|
port,
|
||||||
) + waf.HealthPath
|
) + wafprotocol.HealthPath
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
params.Stdout,
|
Output: params.Stdout,
|
||||||
hostname,
|
InstanceName: hostname,
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
)
|
})
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+21
-24
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -164,7 +164,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
|
|||||||
|
|
||||||
// One request's three lines carry one id, rproxy's with waf's answer.
|
// One request's three lines carry one id, rproxy's with waf's answer.
|
||||||
line := firstOut.line(t, "path", bannedProbe)
|
line := firstOut.line(t, "path", bannedProbe)
|
||||||
if line["disposition"] != waf.DispositionDeny ||
|
if line["disposition"] != wafprotocol.DispositionDeny ||
|
||||||
line["client_ip"] != placed {
|
line["client_ip"] != placed {
|
||||||
t.Errorf("rproxy's line is %v", line)
|
t.Errorf("rproxy's line is %v", line)
|
||||||
}
|
}
|
||||||
@@ -249,7 +249,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
|
|||||||
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
||||||
|
|
||||||
line := out.line(t, "path", "/after-stop")
|
line := out.line(t, "path", "/after-stop")
|
||||||
if line["disposition"] != waf.DispositionAllow ||
|
if line["disposition"] != wafprotocol.DispositionAllow ||
|
||||||
line["waf_backend_no_answer"] != true {
|
line["waf_backend_no_answer"] != true {
|
||||||
t.Errorf("rproxy's line is %v", line)
|
t.Errorf("rproxy's line is %v", line)
|
||||||
}
|
}
|
||||||
@@ -437,7 +437,7 @@ func TestAChunkedBodyStaysABodyAtWAF(t *testing.T) {
|
|||||||
const smuggledID = "smuggled-request"
|
const smuggledID = "smuggled-request"
|
||||||
|
|
||||||
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
|
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
|
||||||
waf.ClientHeader + ": " + otherClient + "\r\n" +
|
wafprotocol.ClientHeader + ": " + otherClient + "\r\n" +
|
||||||
"X-Request-ID: " + smuggledID + "\r\n\r\n"
|
"X-Request-ID: " + smuggledID + "\r\n\r\n"
|
||||||
|
|
||||||
status := sendRaw(
|
status := sendRaw(
|
||||||
@@ -605,10 +605,11 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
|
|||||||
protocols := make(chan string, 10)
|
protocols := make(chan string, 10)
|
||||||
backend := httptest.NewUnstartedServer(http.HandlerFunc(
|
backend := httptest.NewUnstartedServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(waf.ClientHeader) != "" {
|
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||||
protocols <- r.Proto
|
protocols <- r.Proto
|
||||||
|
|
||||||
w.Header().Set(waf.DispositionHeader, waf.DispositionDeny)
|
w.Header().
|
||||||
|
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionDeny)
|
||||||
w.WriteHeader(http.StatusForbidden)
|
w.WriteHeader(http.StatusForbidden)
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
@@ -690,8 +691,9 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(waf.ClientHeader) != "" {
|
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||||
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
|
w.Header().
|
||||||
|
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -755,8 +757,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
||||||
// read from it ends with an error once waitLimit is up.
|
|
||||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -769,11 +770,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
})
|
})
|
||||||
|
|
||||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set the deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
_, err = io.WriteString(conn, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send: %v", err)
|
t.Fatalf("send: %v", err)
|
||||||
@@ -785,10 +781,7 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const cutOffAfter = 5 * time.Second
|
||||||
cutOffAfter = 5 * time.Second
|
|
||||||
upstreamSetting = "SWW_RPROXY_UPSTREAM_URL"
|
|
||||||
)
|
|
||||||
|
|
||||||
asked := make(chan struct{})
|
asked := make(chan struct{})
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
@@ -802,7 +795,8 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
|
w.Header().
|
||||||
|
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
|
||||||
}))
|
}))
|
||||||
t.Cleanup(backend.Close)
|
t.Cleanup(backend.Close)
|
||||||
|
|
||||||
@@ -810,9 +804,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
rproxyListenAddr: localhost + ":0",
|
rproxyListenAddr: localhost + ":0",
|
||||||
upstreamSetting: app.URL,
|
rproxyUpstreamURL: app.URL,
|
||||||
wafBackend: backend.URL,
|
wafBackend: backend.URL,
|
||||||
}
|
}
|
||||||
out := &output{}
|
out := &output{}
|
||||||
exited := make(chan int, 1)
|
exited := make(chan int, 1)
|
||||||
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
|
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
||||||
|
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
text, err := io.ReadAll(conn)
|
||||||
if err != nil || len(text) != 0 {
|
if err != nil || len(text) != 0 {
|
||||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||||
@@ -935,7 +932,7 @@ func TestMainRunsRProxy(t *testing.T) {
|
|||||||
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
|
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||||
"http://"+addr+waf.HealthPath, http.NoBody)
|
"http://"+addr+wafprotocol.HealthPath, http.NoBody)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("new request: %v", err)
|
t.Fatalf("new request: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -1813,7 +1813,7 @@ func statusFrom(t *testing.T, url, from string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("X-Forwarded-For", from)
|
req.Header.Set("X-Forwarded-For", from)
|
||||||
req.Header.Set(waf.ClientHeader, from)
|
req.Header.Set(wafprotocol.ClientHeader, from)
|
||||||
req.Header.Set("X-Request-ID", rand.Text())
|
req.Header.Set("X-Request-ID", rand.Text())
|
||||||
|
|
||||||
transport := &http.Transport{}
|
transport := &http.Transport{}
|
||||||
|
|||||||
+58
-28
@@ -19,10 +19,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout,
|
processLog := processlog.New(processlog.Params{
|
||||||
config.InstanceName(params.Viper), slog.LevelError)
|
Output: params.Stdout,
|
||||||
|
InstanceName: config.InstanceName(params.Viper),
|
||||||
|
Level: slog.LevelError,
|
||||||
|
})
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = requestlog.NewProcessLogger(
|
processLog = processlog.New(processlog.Params{
|
||||||
stdout,
|
Output: stdout,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -118,7 +121,8 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
// The last write follows the stop of every hook, and so the end of every
|
// The last write follows the stop of every hook, and so the end of every
|
||||||
// loop that changes the state or raises an alert. A hook of the state
|
// loop that changes the state or raises an alert, and of every ask of
|
||||||
|
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
||||||
// files would stop before the loops of the parts built ahead of them.
|
// files would stop before the loops of the parts built ahead of them.
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
err := files.WriteAll()
|
err := files.WriteAll()
|
||||||
@@ -238,14 +242,17 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Limits{
|
return ratelimit.New(ratelimit.Params{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
Limits: ratelimit.Limits{
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
}, cfg.MaxTrackedClients)
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
@@ -297,11 +304,13 @@ func newLists(
|
|||||||
return lists
|
return lists
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
||||||
|
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
||||||
func newDNSBL(
|
func newDNSBL(
|
||||||
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
) *reputation.DNSBL {
|
) *reputation.DNSBL {
|
||||||
return reputation.NewDNSBL(reputation.DNSBLParams{
|
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
Zones: cfg.DNSBLZones,
|
Zones: cfg.DNSBLZones,
|
||||||
Resolver: cfg.DNSBLResolver,
|
Resolver: cfg.DNSBLResolver,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
@@ -310,12 +319,17 @@ func newDNSBL(
|
|||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
Alerts: alertQueue,
|
Alerts: alertQueue,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
||||||
|
|
||||||
|
return dnsbl
|
||||||
}
|
}
|
||||||
|
|
||||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||||
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
||||||
|
// under way, as reputation.AbuseIPDB.StopChecking says.
|
||||||
func newAbuseIPDB(
|
func newAbuseIPDB(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||||
@@ -341,24 +355,30 @@ func newAbuseIPDB(
|
|||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||||
|
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
||||||
|
|
||||||
return abuseIPDB
|
return abuseIPDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset.
|
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
||||||
|
// way, as routewatch.Client.StopAsking says.
|
||||||
func newRouteWatch(
|
func newRouteWatch(
|
||||||
cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
) *routewatch.Client {
|
) *routewatch.Client {
|
||||||
if cfg.RouteWatchURL == "" {
|
if cfg.RouteWatchURL == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return routewatch.New(routewatch.Params{
|
routeWatch := routewatch.New(routewatch.Params{
|
||||||
URL: cfg.RouteWatchURL,
|
URL: cfg.RouteWatchURL,
|
||||||
Now: utcNow,
|
Now: utcNow,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
||||||
|
|
||||||
|
return routeWatch
|
||||||
}
|
}
|
||||||
|
|
||||||
// newMetrics returns the metrics, with those read from the other parts as
|
// newMetrics returns the metrics, with those read from the other parts as
|
||||||
@@ -396,12 +416,18 @@ func newMetrics(
|
|||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||||
// failure is handled as waf.GeoJSFailed says.
|
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
||||||
|
// under way, as geojs.GeoJS.StopAsking says.
|
||||||
func newGeoJS(
|
func newGeoJS(
|
||||||
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
lc fx.Lifecycle,
|
||||||
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
cfg *config.WAF,
|
||||||
|
m *metrics.Metrics,
|
||||||
|
limiter *ratelimit.Limiter,
|
||||||
|
ledger *bans.Ledger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
processLog *slog.Logger,
|
||||||
) *geojs.GeoJS {
|
) *geojs.GeoJS {
|
||||||
return geojs.New(geojs.Params{
|
geoJS := geojs.New(geojs.Params{
|
||||||
URL: geojs.URL,
|
URL: geojs.URL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
@@ -413,6 +439,10 @@ func newGeoJS(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(geoJS.StopAsking))
|
||||||
|
|
||||||
|
return geoJS
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWAFServer returns the server waf runs, with its parts and the time
|
// newWAFServer returns the server waf runs, with its parts and the time
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import (
|
|||||||
_ "time/tzdata"
|
_ "time/tzdata"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// commandForMain, while set in the environment, has this test binary run Main
|
// commandForMain, while set in the environment, has this test binary run Main
|
||||||
@@ -106,7 +106,7 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
|
|
||||||
starting := out.lineBeforeExit(t, "starting", exited)
|
starting := out.lineBeforeExit(t, "starting", exited)
|
||||||
addr, _ := starting["address"].(string)
|
addr, _ := starting["address"].(string)
|
||||||
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
|
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
|
|
||||||
// The files are written as sww stops.
|
// The files are written as sww stops.
|
||||||
err = cmd.Process.Signal(syscall.SIGTERM)
|
err = cmd.Process.Signal(syscall.SIGTERM)
|
||||||
@@ -217,7 +217,7 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
|
|||||||
t.Fatalf("new request: %v", err)
|
t.Fatalf("new request: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set(waf.ClientHeader, placed)
|
req.Header.Set(wafprotocol.ClientHeader, placed)
|
||||||
req.Header.Set("X-Request-ID", rand.Text())
|
req.Header.Set("X-Request-ID", rand.Text())
|
||||||
|
|
||||||
transport := &http.Transport{}
|
transport := &http.Transport{}
|
||||||
@@ -232,11 +232,11 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
|
|||||||
_ = res.Body.Close()
|
_ = res.Body.Close()
|
||||||
|
|
||||||
if res.StatusCode != status ||
|
if res.StatusCode != status ||
|
||||||
res.Header.Get(waf.DispositionHeader) != disposition {
|
res.Header.Get(wafprotocol.DispositionHeader) != disposition {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"answered %d with disposition %q, want %d with %q",
|
"answered %d with disposition %q, want %d with %q",
|
||||||
res.StatusCode,
|
res.StatusCode,
|
||||||
res.Header.Get(waf.DispositionHeader),
|
res.Header.Get(wafprotocol.DispositionHeader),
|
||||||
status,
|
status,
|
||||||
disposition,
|
disposition,
|
||||||
)
|
)
|
||||||
@@ -287,12 +287,18 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
runWAF(t, env, func(addr string) {
|
runWAF(t, env, func(addr string) {
|
||||||
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
|
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
// The default rule file bans a probe for this.
|
// The default rule file bans a probe for this.
|
||||||
ask(t, addr, "/.env", http.StatusForbidden, waf.DispositionDeny)
|
ask(
|
||||||
|
t,
|
||||||
|
addr,
|
||||||
|
"/.env",
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
runWAF(t, env, func(addr string) {
|
runWAF(t, env, func(addr string) {
|
||||||
ask(t, addr, "/", http.StatusForbidden, waf.DispositionDeny)
|
ask(t, addr, "/", http.StatusForbidden, wafprotocol.DispositionDeny)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -22,31 +22,42 @@ import (
|
|||||||
// an offence, and the request does not wait for the answer. The score is then
|
// an offence, and the request does not wait for the answer. The score is then
|
||||||
// used for each address of the client. Its figures are the score and
|
// used for each address of the client. Its figures are the score and
|
||||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
||||||
type abuseIPDB struct{}
|
type abuseIPDB struct {
|
||||||
|
abuseIPDB *reputation.AbuseIPDB
|
||||||
|
// limiter is the table of clients the rate limits count, whose histories
|
||||||
|
// say which clients are offenders.
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
|
||||||
|
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
|
||||||
|
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
hitNoter reputationHitNoter
|
||||||
|
}
|
||||||
|
|
||||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||||
|
|
||||||
func (abuseIPDB) check(
|
func (a abuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
if rq.h.config.AbuseIPDBKey == "" {
|
if !a.keySet {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(a.ipv6GroupPrefix, rq.client)
|
||||||
held, _ := rq.h.limiter.Client(client)
|
held, _ := a.limiter.Client(client)
|
||||||
offender := held.History.Offences != ratelimit.Offences{}
|
offender := held.History.Offences != ratelimit.Offences{}
|
||||||
|
|
||||||
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||||
if !hit {
|
if !hit {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.abuseIPDBHit = true
|
rq.abuseIPDBHit = true
|
||||||
rq.addTag(config.AbuseIPDBTag)
|
rq.addTag(config.AbuseIPDBTag)
|
||||||
rq.noteHit(
|
a.hitNoter.noteHit(
|
||||||
|
rq,
|
||||||
bans.ReputationHit{
|
bans.ReputationHit{
|
||||||
Source: reputation.AbuseIPDBSource,
|
Source: reputation.AbuseIPDBSource,
|
||||||
Score: &score,
|
Score: &score,
|
||||||
@@ -54,14 +65,14 @@ func (abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
if a.action != config.ReputationActionDeny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
|
figureScore: score, "min_score": a.minScore,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,9 @@ func BanAbuseIPDBBlacklist(
|
|||||||
scores := make(map[netip.Prefix]int64, len(listed))
|
scores := make(map[netip.Prefix]int64, len(listed))
|
||||||
|
|
||||||
for _, client := range listed {
|
for _, client := range listed {
|
||||||
netblock := banNetblock(cfg, client.Addr)
|
netblock := banNetblock(
|
||||||
|
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
|
||||||
|
)
|
||||||
scores[netblock] = max(scores[netblock], client.Score)
|
scores[netblock] = max(scores[netblock], client.Score)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -224,7 +224,11 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
|
|||||||
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banNetblock(h.config, addr), nil
|
return banNetblock(
|
||||||
|
h.config.BanScopeV4Prefix,
|
||||||
|
h.config.IPv6GroupPrefix,
|
||||||
|
addr,
|
||||||
|
), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// expiry returns when a ban made at now for duration ends: duration
|
// expiry returns when a ban made at now for duration ends: duration
|
||||||
@@ -295,7 +299,9 @@ func (rq *request) showClient() {
|
|||||||
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
||||||
}
|
}
|
||||||
|
|
||||||
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
|
client, seen := rq.h.limiter.Client(
|
||||||
|
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
|
||||||
|
)
|
||||||
if seen {
|
if seen {
|
||||||
answer.Client = &client
|
answer.Client = &client
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
|||||||
"",
|
"",
|
||||||
map[string]string{metricsToken: token},
|
map[string]string{metricsToken: token},
|
||||||
)
|
)
|
||||||
server.Ledger.BanForLimit(
|
server.Ledger.Ban(
|
||||||
netip.MustParsePrefix(otherClient+"/32"),
|
netip.MustParsePrefix(otherClient+"/32"),
|
||||||
clk.Now(),
|
clk.Now(),
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
before := server.Ledger.Snapshot()
|
before := server.Ledger.Snapshot()
|
||||||
|
|
||||||
|
|||||||
@@ -130,11 +130,9 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
|
|||||||
// A ban for a clear sign of attack, which a request under it would make
|
// A ban for a clear sign of attack, which a request under it would make
|
||||||
// permanent.
|
// permanent.
|
||||||
group := netip.MustParsePrefix(ipv6Group)
|
group := netip.MustParsePrefix(ipv6Group)
|
||||||
attackBan, _ := server.Ledger.BanForAttack(
|
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
|
||||||
group,
|
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
|
||||||
start,
|
})
|
||||||
bans.Notes{RuleID: "probe"},
|
|
||||||
)
|
|
||||||
|
|
||||||
// The third request breaks the limit, and so does the fourth, within the
|
// The third request breaks the limit, and so does the fourth, within the
|
||||||
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
// cooldown, which raises nothing. The probe is a clear sign of attack.
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -84,7 +85,7 @@ func TestSixteenThousandCookiesAreRefusedBeforeTheCoreRuleSetReadsThem(
|
|||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
wantDisposition(t, got, tooLarge, wafprotocol.DispositionDeny)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
|
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
|
||||||
@@ -164,9 +165,14 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck(
|
|||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
|
|
||||||
if tc.status == http.StatusOK {
|
if tc.status == http.StatusOK {
|
||||||
wantDisposition(t, got, tc.status, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
tc.status,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
wantDisposition(t, got, tc.status, waf.DispositionDeny)
|
wantDisposition(t, got, tc.status, wafprotocol.DispositionDeny)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantLine(t, line, tc.status, tc.action)
|
wantLine(t, line, tc.status, tc.action)
|
||||||
@@ -219,9 +225,14 @@ func TestAHeaderLineOverTheLimitIsRefusedWith431(t *testing.T) {
|
|||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
if tc.refused {
|
if tc.refused {
|
||||||
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
tooLarge,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -244,10 +255,7 @@ func TestRequestLineRefusalNamesItsHeuristicLengthAndLimit(t *testing.T) {
|
|||||||
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
repeated := make([]string, headerFieldLimit)
|
repeated := slices.Repeat([]string{"1"}, headerFieldLimit)
|
||||||
for i := range repeated {
|
|
||||||
repeated[i] = "1"
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -294,7 +302,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
|||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
|
|
||||||
if tc.refused {
|
if tc.refused {
|
||||||
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
tooLarge,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
wantLine(
|
wantLine(
|
||||||
t,
|
t,
|
||||||
line,
|
line,
|
||||||
@@ -305,7 +318,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
if line.DurationCoreRuleSetMS == nil {
|
if line.DurationCoreRuleSetMS == nil {
|
||||||
@@ -332,7 +350,12 @@ func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) {
|
|||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
tooLarge,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -348,7 +371,12 @@ func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) {
|
|||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
||||||
@@ -364,7 +392,7 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
|||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
|
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
|
||||||
@@ -386,14 +414,14 @@ func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
|
|||||||
for range 2 {
|
for range 2 {
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
wantDisposition(t, do(t, req), tooLarge, wafprotocol.DispositionDeny)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
s.ask(client, "/"),
|
s.ask(client, "/"),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
waf.DispositionAllow,
|
wafprotocol.DispositionAllow,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -466,7 +494,12 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
|||||||
req := s.question(siteA, client, tc.path)
|
req := s.question(siteA, client, tc.path)
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
tooLarge,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
wantLine(
|
wantLine(
|
||||||
t,
|
t,
|
||||||
s.out.requestLine(t),
|
s.out.requestLine(t),
|
||||||
@@ -503,11 +536,16 @@ func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
|
|||||||
|
|
||||||
over := s.question(siteA, client, "/")
|
over := s.question(siteA, client, "/")
|
||||||
over.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
over.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny)
|
wantDisposition(t, do(t, over), tooLarge, wafprotocol.DispositionDeny)
|
||||||
|
|
||||||
within := s.question(siteA, client, "/")
|
within := s.question(siteA, client, "/")
|
||||||
within.Header.Set(cookieHeader, cookies(cookieLimit))
|
within.Header.Set(cookieHeader, cookies(cookieLimit))
|
||||||
wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, within),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
|
|
||||||
@@ -564,7 +602,7 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
s.ask(fromDE, "/"),
|
s.ask(fromDE, "/"),
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
waf.DispositionDeny,
|
wafprotocol.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
@@ -601,7 +639,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
|
|||||||
t,
|
t,
|
||||||
s.ask(client, "/"),
|
s.ask(client, "/"),
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
waf.DispositionDeny,
|
wafprotocol.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
@@ -630,7 +668,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
|
|||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
|
|
||||||
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
|
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -672,7 +710,7 @@ func askDirectly(
|
|||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
|
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
|
||||||
req.Header.Set(waf.ClientHeader, client)
|
req.Header.Set(wafprotocol.ClientHeader, client)
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
req.Header.Set(requestIDHeader, "the-id")
|
||||||
change(req)
|
change(req)
|
||||||
|
|
||||||
@@ -686,10 +724,10 @@ func askDirectly(
|
|||||||
|
|
||||||
// banClient bans client at now.
|
// banClient bans client at now.
|
||||||
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||||
server.Ledger.BanForLimit(
|
server.Ledger.Ban(
|
||||||
netip.MustParsePrefix(client+"/32"),
|
netip.MustParsePrefix(client+"/32"),
|
||||||
now,
|
now,
|
||||||
bans.Notes{},
|
bans.Grounds{Cause: bans.CauseLimit},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+157
-94
@@ -1,7 +1,7 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
// The heuristic active_ban, and the code that makes bans, which several
|
// The heuristic active_ban, and banMaker, which makes the bans of several
|
||||||
// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
@@ -25,18 +26,22 @@ func (rq *request) banResponse(action string) *refusal {
|
|||||||
// in, in force at now. It notes for the log line when that ban ends. A request
|
// in, in force at now. It notes for the log line when that ban ends. A request
|
||||||
// that makes the ban permanent, or in observe mode would have, raises the alert
|
// that makes the ban permanent, or in observe mode would have, raises the alert
|
||||||
// for it. Its figures are the ban's netblock and when it ends.
|
// for it. Its figures are the ban's netblock and when it ends.
|
||||||
type activeBan struct{}
|
type activeBan struct {
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (activeBan) name() string { return "active_ban" }
|
func (activeBan) name() string { return "active_ban" }
|
||||||
|
|
||||||
func (activeBan) check(
|
func (a activeBan) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
check := rq.h.ledger.Check
|
ledger := a.banMaker.ledger
|
||||||
if rq.h.config.Observe {
|
|
||||||
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
|
check := ledger.Check
|
||||||
|
if a.banMaker.observe {
|
||||||
|
check = ledger.Find // the ban refuses nothing, and stays as it is
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, banned, madePermanent := check(rq.client, now)
|
ban, banned, madePermanent := check(rq.client, now)
|
||||||
@@ -46,7 +51,7 @@ func (activeBan) check(
|
|||||||
|
|
||||||
if madePermanent {
|
if madePermanent {
|
||||||
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
||||||
rq.alertBan(ban)
|
a.banMaker.alertBan(rq, ban)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !banned {
|
if !banned {
|
||||||
@@ -81,11 +86,12 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
|
|||||||
|
|
||||||
// countedBytes returns the request's bytes, once it has ended, as the
|
// countedBytes returns the request's bytes, once it has ended, as the
|
||||||
// byte limits and the anomaly thresholds count them: the response's body
|
// byte limits and the anomaly thresholds count them: the response's body
|
||||||
// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says.
|
// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED,
|
||||||
func (rq *request) countedBytes() int64 {
|
// says.
|
||||||
|
func (rq *request) countedBytes(bytesCounted config.BytesCounted) int64 {
|
||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes
|
||||||
|
|
||||||
switch rq.h.config.BytesCounted {
|
switch bytesCounted {
|
||||||
case config.BytesCountedResponse:
|
case config.BytesCountedResponse:
|
||||||
return response
|
return response
|
||||||
case config.BytesCountedRequest:
|
case config.BytesCountedRequest:
|
||||||
@@ -96,17 +102,39 @@ func (rq *request) countedBytes() int64 {
|
|||||||
return response + request
|
return response + request
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banMaker makes the bans of the heuristics that ban a client, in the ban
|
||||||
|
// ledger, and raises the alert for each. In observe mode it makes no ban, and
|
||||||
|
// raises the alert for the ban it would have made.
|
||||||
|
type banMaker struct {
|
||||||
|
ledger *bans.Ledger
|
||||||
|
// limiter is the table of clients the rate limits count. A ban for a limit
|
||||||
|
// sets the client's counters back to zero, and a ban's notes give how many
|
||||||
|
// requests its netblock has sent.
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
alerts *alerts.Queue
|
||||||
|
// observe is true in observe mode (SWW_WAF_MODE).
|
||||||
|
observe bool
|
||||||
|
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
|
||||||
|
// clear sign of attack give.
|
||||||
|
banResponse int
|
||||||
|
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
|
||||||
|
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
|
||||||
|
banScopeV4Prefix int
|
||||||
|
ipv6GroupPrefix int
|
||||||
|
}
|
||||||
|
|
||||||
|
// banForLimit bans the netblock of rq's client at now for a broken limit, the
|
||||||
// one hit names, notes the offence for the log line and counts the hit in
|
// one hit names, notes the offence for the log line and counts the hit in
|
||||||
// the metrics. status is what the client was sent, or is sent:
|
// the metrics. status is what the client was sent, or is sent:
|
||||||
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
|
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
|
||||||
// one whose bytes broke a byte limit, the refusal for one that broke the
|
// one whose bytes broke a byte limit, the refusal for one that broke the
|
||||||
// refusal limit. The ban's notes give the client's limit percentage for a
|
// refusal limit. The ban's notes give the client's limit percentage for a
|
||||||
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
|
||||||
// the client's counters back to zero. In observe mode it makes no ban and
|
// the client's counters back to zero. In observe mode, banClient says what
|
||||||
// sets nothing back, and raises the alert for the ban it would have made,
|
// happens instead.
|
||||||
// if that alert would be sent.
|
func (b banMaker) banForLimit(
|
||||||
func (rq *request) banForLimit(
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
hit ratelimit.Hit,
|
hit ratelimit.Hit,
|
||||||
status int,
|
status int,
|
||||||
@@ -121,25 +149,14 @@ func (rq *request) banForLimit(
|
|||||||
}
|
}
|
||||||
|
|
||||||
rq.line.Offence = requestlog.OffenceLimit
|
rq.line.Offence = requestlog.OffenceLimit
|
||||||
rq.h.metrics.LimitHit(hit)
|
|
||||||
|
|
||||||
netblock := banNetblock(rq.h.config, rq.client)
|
b.metrics.LimitHit(hit)
|
||||||
if rq.h.config.Observe &&
|
|
||||||
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
notes := bans.Notes{
|
notes := bans.Notes{
|
||||||
ASN: rq.line.ASN,
|
Kind: hit.Kind,
|
||||||
ASName: rq.line.ASName,
|
Limit: hit.Limit,
|
||||||
Country: rq.line.Country,
|
Window: hit.Window,
|
||||||
Kind: hit.Kind,
|
Count: hit.Count,
|
||||||
Limit: hit.Limit,
|
|
||||||
Window: hit.Window,
|
|
||||||
Count: hit.Count,
|
|
||||||
Reputation: rq.reputation,
|
|
||||||
Request: rq.noted(now, status),
|
|
||||||
Requests: rq.netblockRequests(netblock),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
switch hit.Kind {
|
switch hit.Kind {
|
||||||
@@ -149,86 +166,130 @@ func (rq *request) banForLimit(
|
|||||||
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.h.config.Observe {
|
b.banClient(
|
||||||
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
|
rq,
|
||||||
if wouldBan {
|
now,
|
||||||
rq.alertBan(ban)
|
status,
|
||||||
}
|
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
|
||||||
|
)
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
|
|
||||||
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
|
||||||
rq.line.BanExpires = banExpires(ban)
|
|
||||||
|
|
||||||
if made {
|
|
||||||
rq.alertBan(ban)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForAttack bans the client's netblock at now for a clear sign of
|
// banForAttack bans the netblock of rq's client at now for a clear sign of
|
||||||
// attack, which notes name: the ban rule that matched, or the trap path
|
// attack, which notes name: the ban rule that matched, or the trap path
|
||||||
// asked for. It fills in the rest of the notes. In observe mode it makes
|
// asked for. It returns the action for that ban. In observe mode, banClient
|
||||||
// no ban, and raises the alert for the ban it would have made, if that
|
// says what happens instead.
|
||||||
// alert would be sent. It returns the action for that ban.
|
func (b banMaker) banForAttack(
|
||||||
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
rq *request, now time.Time, notes bans.Notes,
|
||||||
netblock := banNetblock(rq.h.config, rq.client)
|
) string {
|
||||||
if rq.h.config.Observe &&
|
return b.banClient(
|
||||||
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
rq,
|
||||||
return banAction(
|
now,
|
||||||
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
|
b.banResponse,
|
||||||
|
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// banClient bans the netblock of rq's client at now on grounds, and returns
|
||||||
|
// the action for that ban. It adds to the notes what the request tells of the
|
||||||
|
// client, with status, what the client was sent. A ban for a broken limit
|
||||||
|
// sets the client's counters back to zero.
|
||||||
|
//
|
||||||
|
// In observe mode it makes no ban and sets nothing back. It raises the
|
||||||
|
// alert for the ban it would have made, if that alert would be sent.
|
||||||
|
func (b banMaker) banClient(
|
||||||
|
rq *request,
|
||||||
|
now time.Time,
|
||||||
|
status int,
|
||||||
|
grounds bans.Grounds,
|
||||||
|
) string {
|
||||||
|
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
|
||||||
|
|
||||||
|
if b.observe {
|
||||||
|
if !b.wouldAlertBan(netblock, now, grounds.Cause) {
|
||||||
|
return banAction(
|
||||||
|
b.ledger.WouldBePermanent(netblock, now, grounds.Cause),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
grounds.Notes = b.withClientNotes(
|
||||||
|
rq,
|
||||||
|
grounds.Notes,
|
||||||
|
now,
|
||||||
|
status,
|
||||||
|
netblock,
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
notes.ASN = rq.line.ASN
|
ban, wouldBan := b.ledger.WouldBan(netblock, now, grounds)
|
||||||
notes.ASName = rq.line.ASName
|
|
||||||
notes.Country = rq.line.Country
|
|
||||||
notes.Reputation = rq.reputation
|
|
||||||
notes.Request = rq.noted(now, rq.h.config.BanResponse)
|
|
||||||
notes.Requests = rq.netblockRequests(netblock)
|
|
||||||
|
|
||||||
if rq.h.config.Observe {
|
|
||||||
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
|
|
||||||
if wouldBan {
|
if wouldBan {
|
||||||
rq.alertBan(ban)
|
b.alertBan(rq, ban)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
|
grounds.Notes = b.withClientNotes(
|
||||||
|
rq,
|
||||||
|
grounds.Notes,
|
||||||
|
now,
|
||||||
|
status,
|
||||||
|
netblock,
|
||||||
|
)
|
||||||
|
|
||||||
|
ban, made := b.ledger.Ban(netblock, now, grounds)
|
||||||
|
if grounds.Cause == bans.CauseLimit {
|
||||||
|
b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client))
|
||||||
|
}
|
||||||
|
|
||||||
rq.line.BanExpires = banExpires(ban)
|
rq.line.BanExpires = banExpires(ban)
|
||||||
|
|
||||||
if made {
|
if made {
|
||||||
rq.alertBan(ban)
|
b.alertBan(rq, ban)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banAction(ban.Permanent())
|
return banAction(ban.Permanent())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withClientNotes is notes with what rq tells of its client added: its AS,
|
||||||
|
// country and reputation, the request at now with status, and how many
|
||||||
|
// requests netblock, the client's, has sent.
|
||||||
|
func (b banMaker) withClientNotes(
|
||||||
|
rq *request,
|
||||||
|
notes bans.Notes,
|
||||||
|
now time.Time,
|
||||||
|
status int,
|
||||||
|
netblock netip.Prefix,
|
||||||
|
) bans.Notes {
|
||||||
|
notes.ASN = rq.line.ASN
|
||||||
|
notes.ASName = rq.line.ASName
|
||||||
|
notes.Country = rq.line.Country
|
||||||
|
notes.Reputation = rq.reputation
|
||||||
|
notes.Request = rq.noted(now, status)
|
||||||
|
notes.Requests = b.netblockRequests(netblock)
|
||||||
|
|
||||||
|
return notes
|
||||||
|
}
|
||||||
|
|
||||||
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
||||||
// made at now would be sent. In observe mode the ban the request would
|
// made at now would be sent. In observe mode the ban the request would
|
||||||
// have made is worked out only then, at most once per
|
// have made is worked out only then, at most once per
|
||||||
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
|
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
|
||||||
// netblock's requests, which can mean going through every client.
|
// netblock's requests, which can mean going through every client.
|
||||||
func (rq *request) wouldAlertBan(
|
func (b banMaker) wouldAlertBan(
|
||||||
netblock netip.Prefix, now time.Time, cause string,
|
netblock netip.Prefix, now time.Time, cause string,
|
||||||
) bool {
|
) bool {
|
||||||
event := alerts.EventBan
|
event := alerts.EventBan
|
||||||
if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
|
if b.ledger.WouldBePermanent(netblock, now, cause) {
|
||||||
event = alerts.EventPermanentBan
|
event = alerts.EventPermanentBan
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.h.alerts.WouldSend(event, netblock)
|
return b.alerts.WouldSend(event, netblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
// alertBan raises the alert for ban, which the request made, or made
|
// alertBan raises the alert for ban, which rq made, or made permanent:
|
||||||
// permanent: permanent_ban for a permanent ban, ban for another. Its
|
// permanent_ban for a permanent ban, ban for another. Its detail gives the
|
||||||
// detail gives the ban's cause, when it ends, and its notes, and in
|
// ban's cause, when it ends, and its notes, and in observe mode, where ban is
|
||||||
// observe mode, where ban is the ban that would have been made, or made
|
// the ban that would have been made, or made permanent, mode, observe.
|
||||||
// permanent, mode, observe.
|
func (b banMaker) alertBan(rq *request, ban bans.Ban) {
|
||||||
func (rq *request) alertBan(ban bans.Ban) {
|
|
||||||
event := alerts.EventBan
|
event := alerts.EventBan
|
||||||
if ban.Permanent() {
|
if ban.Permanent() {
|
||||||
event = alerts.EventPermanentBan
|
event = alerts.EventPermanentBan
|
||||||
@@ -237,11 +298,11 @@ func (rq *request) alertBan(ban bans.Ban) {
|
|||||||
detail := map[string]any{
|
detail := map[string]any{
|
||||||
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
|
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
|
||||||
}
|
}
|
||||||
if rq.h.config.Observe {
|
if b.observe {
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
b.alerts.Raise(alerts.Alert{
|
||||||
Event: event,
|
Event: event,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
@@ -254,6 +315,13 @@ func (rq *request) alertBan(ban bans.Ban) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// netblockRequests is how many requests netblock has sent since it was
|
||||||
|
// first seen, the request being banned for included: the histories count it
|
||||||
|
// only once it has ended.
|
||||||
|
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
|
||||||
|
return b.limiter.Requests(netblock) + 1
|
||||||
|
}
|
||||||
|
|
||||||
// noted is the request, at now, with status, what the client was sent, or
|
// noted is the request, at now, with status, what the client was sent, or
|
||||||
// in observe mode would have been, as the notes of the ban it makes keep
|
// in observe mode would have been, as the notes of the ban it makes keep
|
||||||
// it.
|
// it.
|
||||||
@@ -268,23 +336,18 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// netblockRequests is how many requests netblock has sent since it was
|
|
||||||
// first seen, this one included: the histories count it only once it has
|
|
||||||
// ended.
|
|
||||||
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
|
|
||||||
return rq.h.limiter.Requests(netblock) + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// banNetblock is the netblock a ban on client covers: its IPv4 address,
|
// banNetblock is the netblock a ban on client covers: its IPv4 address,
|
||||||
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
|
// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group
|
||||||
// counts it in.
|
// clientGroup counts it in, of length ipv6GroupPrefix.
|
||||||
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
|
func banNetblock(
|
||||||
|
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
|
||||||
|
) netip.Prefix {
|
||||||
addr := client.Unmap()
|
addr := client.Unmap()
|
||||||
if addr.Is4() {
|
if addr.Is4() {
|
||||||
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
|
return netip.PrefixFrom(addr, banScopeV4Prefix).Masked()
|
||||||
}
|
}
|
||||||
|
|
||||||
return clientGroup(cfg, addr)
|
return clientGroup(ipv6GroupPrefix, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// banExpires is when ban ends, as the log line gives it: a time, or
|
// banExpires is when ban ends, as the log line gives it: a time, or
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
|
|||||||
) *verdict {
|
) *verdict {
|
||||||
var running atomic.Pointer[string]
|
var running atomic.Pointer[string]
|
||||||
|
|
||||||
running.Store(new(trapPaths{}.name()))
|
running.Store(new(rq.h.trapPaths.name()))
|
||||||
|
|
||||||
found := make(chan *besideLookupFindings, 1)
|
found := make(chan *besideLookupFindings, 1)
|
||||||
|
|
||||||
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
|
|||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
|
return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// findBesideLookup does the work of the heuristics of besideLookup on the
|
// findBesideLookup does the work of the heuristics of besideLookup on the
|
||||||
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
|
|||||||
) *besideLookupFindings {
|
) *besideLookupFindings {
|
||||||
findings := &besideLookupFindings{}
|
findings := &besideLookupFindings{}
|
||||||
|
|
||||||
findings.trapPath = rq.trapPath()
|
findings.trapPath = rq.h.trapPaths.trapPath(rq)
|
||||||
if findings.trapPath != "" {
|
if findings.trapPath != "" {
|
||||||
findings.refuses = true
|
findings.refuses = true
|
||||||
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
running.Store(new(ruleFiles{}.name()))
|
running.Store(new(rq.h.ruleFiles.name()))
|
||||||
|
|
||||||
findings.rules = rq.h.rules.Match(rq.in)
|
findings.rules = rq.h.ruleFiles.rules.Match(rq.in)
|
||||||
if refusedByRule(findings.rules) {
|
if refusedByRule(findings.rules) {
|
||||||
findings.refuses = true
|
findings.refuses = true
|
||||||
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
running.Store(new(coreRuleSet{}.name()))
|
coreRuleSet := rq.h.coreRuleSet
|
||||||
|
running.Store(new(coreRuleSet.name()))
|
||||||
|
|
||||||
cfg := rq.h.config
|
if coreRuleSet.mode == config.CoreRuleSetModeOff ||
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
|
pathExempt(rq.in, coreRuleSet.exemptPaths) {
|
||||||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
// A question's body has been read into memory, so that reading it here
|
// A question's body has been read into memory, so that reading it here
|
||||||
// cannot fail.
|
// cannot fail.
|
||||||
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
|
findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect(
|
||||||
rq.in, rq.client, rq.in.Body,
|
rq.in, rq.client, rq.in.Body,
|
||||||
)
|
)
|
||||||
findings.inspectionTime = time.Since(start)
|
findings.inspectionTime = time.Since(start)
|
||||||
findings.inspected = true
|
findings.inspected = true
|
||||||
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
|
findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection)
|
||||||
|
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
|
|||||||
) *verdict {
|
) *verdict {
|
||||||
rq.findings = findings
|
rq.findings = findings
|
||||||
|
|
||||||
return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
|
return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// timedOut refuses the request for the heuristics of besideLookup not having
|
// timedOut refuses the request for the heuristics of besideLookup not having
|
||||||
|
|||||||
+43
-22
@@ -2,6 +2,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// whole is the percentage of each limit a client gets when no biased
|
// whole is the percentage of each limit a client gets when no biased
|
||||||
@@ -29,6 +30,25 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
|
|||||||
cfg.ASNLimitPercentURL != ""
|
cfg.ASNLimitPercentURL != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// biasedThresholds are the settings and the lists by which limitPercentages
|
||||||
|
// lowers a client's limits.
|
||||||
|
type biasedThresholds struct {
|
||||||
|
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
|
||||||
|
lists *reputation.Lists
|
||||||
|
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
|
||||||
|
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
|
||||||
|
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
|
||||||
|
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
|
||||||
|
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
|
||||||
|
// and its percentage.
|
||||||
|
blocklistAction config.ReputationAction
|
||||||
|
blocklistLimitPercent int64
|
||||||
|
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
|
||||||
|
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
|
||||||
|
dnsblAbuseIPDBAction config.ReputationAction
|
||||||
|
dnsblAbuseIPDBLimitPercent int64
|
||||||
|
}
|
||||||
|
|
||||||
// limitPercentages returns the client's limit percentages, for the rate
|
// limitPercentages returns the client's limit percentages, for the rate
|
||||||
// limits and for the byte limits, by its tags, its AS number and country
|
// limits and for the byte limits, by its tags, its AS number and country
|
||||||
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
||||||
@@ -45,49 +65,50 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
|
|||||||
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
|
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
|
||||||
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
|
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
|
||||||
// an AS number or a country they list.
|
// an AS number or a country they list.
|
||||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
func (b biasedThresholds) limitPercentages(
|
||||||
cfg := rq.h.config
|
rq *request,
|
||||||
|
) (percentage, percentage) {
|
||||||
asn, country := rq.line.ASN, rq.line.Country
|
asn, country := rq.line.ASN, rq.line.Country
|
||||||
|
|
||||||
unknown := percentage{percent: whole}
|
unknown := percentage{percent: whole}
|
||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
unknown = percentage{
|
unknown = percentage{
|
||||||
cfg.UnknownCountryLimitPercent,
|
b.unknownCountryLimitPercent,
|
||||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
config.UnknownCountryLimitPercentSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched := percentage{percent: whole}
|
fetched := percentage{percent: whole}
|
||||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
if percent, listed := b.lists.ASNLimitPercent(asn); listed {
|
||||||
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
if rq.blocklisted &&
|
if rq.blocklisted &&
|
||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
b.blocklistAction == config.ReputationActionLimit {
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
b.blocklistLimitPercent,
|
||||||
"SWW_WAF_BLOCKLIST_ACTION",
|
config.BlocklistActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
b.dnsblAbuseIPDBAction == config.ReputationActionLimit {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
b.dnsblAbuseIPDBLimitPercent,
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
config.DNSBLAbuseIPDBActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
const tagSetting = config.TagLimitPercentSetting
|
||||||
|
|
||||||
asnRequests := lowest(
|
asnRequests := lowest(
|
||||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
given(b.tagLimitPercent, asnTag(asn), tagSetting),
|
||||||
fetched,
|
fetched,
|
||||||
)
|
)
|
||||||
countryRequests := given(
|
countryRequests := given(
|
||||||
cfg.TagLimitPercent,
|
b.tagLimitPercent,
|
||||||
countryTag(country),
|
countryTag(country),
|
||||||
tagSetting,
|
tagSetting,
|
||||||
)
|
)
|
||||||
@@ -98,25 +119,25 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
if tag != asnTag(asn) && tag != countryTag(country) {
|
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||||
otherTags = lowest(
|
otherTags = lowest(
|
||||||
otherTags,
|
otherTags,
|
||||||
given(cfg.TagLimitPercent, tag, tagSetting),
|
given(b.tagLimitPercent, tag, tagSetting),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
asnBytes, countryBytes := asnRequests, countryRequests
|
asnBytes, countryBytes := asnRequests, countryRequests
|
||||||
if _, listed := cfg.ASNBytesLimitPercent[asn]; listed {
|
if _, listed := b.asnBytesLimitPercent[asn]; listed {
|
||||||
asnBytes = given(
|
asnBytes = given(
|
||||||
cfg.ASNBytesLimitPercent,
|
b.asnBytesLimitPercent,
|
||||||
asn,
|
asn,
|
||||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
config.ASNBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, listed := cfg.CountryBytesLimitPercent[country]; listed {
|
if _, listed := b.countryBytesLimitPercent[country]; listed {
|
||||||
countryBytes = given(
|
countryBytes = given(
|
||||||
cfg.CountryBytesLimitPercent,
|
b.countryBytesLimitPercent,
|
||||||
country,
|
country,
|
||||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
config.CountryBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The biased thresholds.
|
// The biased thresholds.
|
||||||
@@ -589,7 +590,7 @@ func serveFromDE(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
||||||
req.Header.Set(waf.ClientHeader, fromDE)
|
req.Header.Set(wafprotocol.ClientHeader, fromDE)
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
req.Header.Set(requestIDHeader, "the-id")
|
||||||
|
|
||||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,20 +16,24 @@ import (
|
|||||||
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
|
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
|
||||||
// lowers the client's limits instead (see limitPercentages), and being log, it
|
// lowers the client's limits instead (see limitPercentages), and being log, it
|
||||||
// does nothing more. Its figure is the blocklists that list the client.
|
// does nothing more. Its figure is the blocklists that list the client.
|
||||||
type blocklists struct{}
|
type blocklists struct {
|
||||||
|
lists *reputation.Lists
|
||||||
|
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
|
||||||
|
hitNoter reputationHitNoter
|
||||||
|
}
|
||||||
|
|
||||||
func (blocklists) name() string { return "blocklists" }
|
func (blocklists) name() string { return "blocklists" }
|
||||||
|
|
||||||
func (blocklists) check(
|
func (b blocklists) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
listedBy := rq.h.lists.ListedBy(rq.client)
|
listedBy := b.lists.ListedBy(rq.client)
|
||||||
rq.blocklisted = len(listedBy) > 0
|
rq.blocklisted = len(listedBy) > 0
|
||||||
rq.noteListed(listedBy, "listed by a blocklist")
|
b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist")
|
||||||
|
|
||||||
if !rq.refusedByBlocklists(listedBy) {
|
if !b.refusedByBlocklists(listedBy) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,7 +46,6 @@ func (blocklists) check(
|
|||||||
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
func (b blocklists) refusedByBlocklists(listedBy []string) bool {
|
||||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
return len(listedBy) > 0 && b.action == config.ReputationActionDeny
|
||||||
config.ReputationActionDeny
|
|
||||||
}
|
}
|
||||||
@@ -6,6 +6,9 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// byteLimits counts the request's bytes, as countedBytes gives them, for the
|
// byteLimits counts the request's bytes, as countedBytes gives them, for the
|
||||||
@@ -16,11 +19,16 @@ import (
|
|||||||
// the client over a byte limit, as its limit percentage for the byte limits
|
// the client over a byte limit, as its limit percentage for the byte limits
|
||||||
// lowers it, break it, which bans the client; the response was passed on whole.
|
// lowers it, break it, which bans the client; the response was passed on whole.
|
||||||
// Its figures are the window, the count and the limit.
|
// Its figures are the window, the count and the limit.
|
||||||
type byteLimits struct{}
|
type byteLimits struct {
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
bytesCounted config.BytesCounted // SWW_WAF_BYTES_COUNTED
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (byteLimits) name() string { return "byte_limits" }
|
func (byteLimits) name() string { return "byte_limits" }
|
||||||
|
|
||||||
func (byteLimits) check(
|
func (b byteLimits) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -29,10 +37,10 @@ func (byteLimits) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
counts, hit, over := rq.h.limiter.CountBytes(
|
counts, hit, over := b.limiter.CountBytes(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(b.ipv6GroupPrefix, rq.client),
|
||||||
now,
|
now,
|
||||||
rq.countedBytes(),
|
rq.countedBytes(b.bytesCounted),
|
||||||
rq.bytesLimitPercent.percent,
|
rq.bytesLimitPercent.percent,
|
||||||
)
|
)
|
||||||
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
||||||
@@ -43,7 +51,7 @@ func (byteLimits) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banForLimit(now, hit, rq.out.status)
|
b.banMaker.banForLimit(rq, now, hit, rq.out.status)
|
||||||
|
|
||||||
return &verdict{figures: limitFigures(hit)}
|
return &verdict{figures: limitFigures(hit)}
|
||||||
}
|
}
|
||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer.
|
// peerAddress is the address of the request's TCP peer.
|
||||||
@@ -19,14 +19,11 @@ func peerAddress(r *http.Request) netip.Addr {
|
|||||||
return addrPort.Addr().Unmap()
|
return addrPort.Addr().Unmap()
|
||||||
}
|
}
|
||||||
|
|
||||||
// requestIDHeader carries the request's id, from rproxy.
|
|
||||||
const requestIDHeader = "X-Request-ID"
|
|
||||||
|
|
||||||
// requestID is the request's id: the one a question gives, or for any other
|
// requestID is the request's id: the one a question gives, or for any other
|
||||||
// request a new random one.
|
// request a new random one.
|
||||||
func requestID(r *http.Request, isQuestion bool) string {
|
func requestID(r *http.Request, isQuestion bool) string {
|
||||||
if isQuestion {
|
if isQuestion {
|
||||||
return r.Header.Get(requestIDHeader)
|
return r.Header.Get(wafprotocol.RequestIDHeader)
|
||||||
}
|
}
|
||||||
|
|
||||||
return rand.Text()
|
return rand.Text()
|
||||||
@@ -45,12 +42,13 @@ func scheme(r *http.Request, isQuestion bool) string {
|
|||||||
|
|
||||||
// clientGroup is the client a request is counted toward: its IPv4
|
// clientGroup is the client a request is counted toward: its IPv4
|
||||||
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
||||||
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
|
// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since
|
||||||
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
|
// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as
|
||||||
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
|
// IPv4.
|
||||||
|
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
|
||||||
addr = addr.Unmap()
|
addr = addr.Unmap()
|
||||||
if addr.Is6() {
|
if addr.Is6() {
|
||||||
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
|
return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked()
|
||||||
}
|
}
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen())
|
return netip.PrefixFrom(addr, addr.BitLen())
|
||||||
|
|||||||
+30
-18
@@ -10,6 +10,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,11 +23,22 @@ import (
|
|||||||
// exploitsignature, raises the core_rule_set_match alert, and in block mode
|
// exploitsignature, raises the core_rule_set_match alert, and in block mode
|
||||||
// refuses the request with 403, which is an offence its client's history
|
// refuses the request with 403, which is an offence its client's history
|
||||||
// counts. Its figures are the score and the threshold.
|
// counts. Its figures are the score and the threshold.
|
||||||
type coreRuleSet struct{}
|
type coreRuleSet struct {
|
||||||
|
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
|
||||||
|
// off.
|
||||||
|
inspector CoreRuleSetInspector
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
alerts *alerts.Queue
|
||||||
|
mode config.CoreRuleSetMode // SWW_WAF_CORE_RULE_SET_MODE
|
||||||
|
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
|
||||||
|
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
|
||||||
|
observe bool // SWW_WAF_MODE is observe
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
}
|
||||||
|
|
||||||
func (coreRuleSet) name() string { return "core_rule_set" }
|
func (coreRuleSet) name() string { return "core_rule_set" }
|
||||||
|
|
||||||
func (coreRuleSet) check(
|
func (c coreRuleSet) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
@@ -36,7 +48,6 @@ func (coreRuleSet) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := rq.h.config
|
|
||||||
result := findings.inspection
|
result := findings.inspection
|
||||||
rq.requestBytes = int64(len(findings.read))
|
rq.requestBytes = int64(len(findings.read))
|
||||||
|
|
||||||
@@ -47,18 +58,18 @@ func (coreRuleSet) check(
|
|||||||
rq.line.CoreRuleSetScore = &result.Score
|
rq.line.CoreRuleSetScore = &result.Score
|
||||||
|
|
||||||
for _, id := range result.RuleIDs {
|
for _, id := range result.RuleIDs {
|
||||||
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
c.metrics.CoreRuleSetMatched(string(c.mode), id)
|
||||||
}
|
}
|
||||||
|
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
threshold := c.scoreThreshold
|
||||||
if threshold == 0 || result.Score < threshold {
|
if threshold == 0 || result.Score < threshold {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.addTag(config.ExploitSignatureTag)
|
rq.addTag(config.ExploitSignatureTag)
|
||||||
rq.alertCoreRuleSetMatch(result)
|
c.alertCoreRuleSetMatch(rq, result)
|
||||||
|
|
||||||
if !rq.refusedByCoreRuleSet(result) {
|
if !c.refusedByCoreRuleSet(result) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,20 +87,21 @@ func (coreRuleSet) check(
|
|||||||
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
|
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
|
||||||
// Rule Set found result of: whether the score is at or over
|
// Rule Set found result of: whether the score is at or over
|
||||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
|
||||||
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
|
func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool {
|
||||||
cfg := rq.h.config
|
threshold := c.scoreThreshold
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
|
||||||
|
|
||||||
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
|
return c.mode == config.CoreRuleSetModeBlock &&
|
||||||
threshold != 0 && result.Score >= threshold
|
threshold != 0 && result.Score >= threshold
|
||||||
}
|
}
|
||||||
|
|
||||||
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
|
// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the
|
||||||
// which the Core Rule Set scored at result, at or over
|
// Core Rule Set scored at result, at or over
|
||||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
|
||||||
// score, the method and the path with the query, and, for a request that is
|
// score, the method and the path with the query, and, for a request that is
|
||||||
// not refused for it, the mode: detect, or observe in observe mode.
|
// not refused for it, the mode: detect, or observe in observe mode.
|
||||||
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
func (c coreRuleSet) alertCoreRuleSetMatch(
|
||||||
|
rq *request, result coreruleset.Result,
|
||||||
|
) {
|
||||||
detail := map[string]any{
|
detail := map[string]any{
|
||||||
"rule_ids": result.RuleIDs,
|
"rule_ids": result.RuleIDs,
|
||||||
"score": result.Score,
|
"score": result.Score,
|
||||||
@@ -98,17 +110,17 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
case c.mode == config.CoreRuleSetModeDetect:
|
||||||
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
||||||
case rq.h.config.Observe:
|
case c.observe:
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
c.alerts.Raise(alerts.Alert{
|
||||||
Event: alerts.EventCoreRuleSetMatch,
|
Event: alerts.EventCoreRuleSetMatch,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: clientGroup(rq.h.config, rq.client),
|
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.line.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.line.Country,
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
|
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
|
||||||
@@ -395,9 +395,9 @@ func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
wantDisposition(t, do(t, s.question(siteA, client, fileName)),
|
wantDisposition(t, do(t, s.question(siteA, client, fileName)),
|
||||||
http.StatusOK, waf.DispositionAllow)
|
http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
wantDisposition(t, do(t, s.question(siteB, client, fileName)),
|
wantDisposition(t, do(t, s.question(siteB, client, fileName)),
|
||||||
http.StatusForbidden, waf.DispositionDeny)
|
http.StatusForbidden, wafprotocol.DispositionDeny)
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
|
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -14,16 +15,18 @@ import (
|
|||||||
// does not hold the client's country as it was looked up. A client without a
|
// does not hold the client's country as it was looked up. A client without a
|
||||||
// country, or whose country cannot be found, is refused. Its figures are the
|
// country, or whose country cannot be found, is refused. Its figures are the
|
||||||
// country and the setting that refused it.
|
// country and the setting that refused it.
|
||||||
type countries struct{}
|
type countries struct {
|
||||||
|
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
|
||||||
|
}
|
||||||
|
|
||||||
func (countries) name() string { return "countries" }
|
func (countries) name() string { return "countries" }
|
||||||
|
|
||||||
func (countries) check(
|
func (c countries) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
allowed := c.allowed
|
||||||
country := rq.line.Country
|
country := rq.line.Country
|
||||||
|
|
||||||
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||||
@@ -33,7 +36,8 @@ func (countries) check(
|
|||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionCountryDenied,
|
action: requestlog.ActionCountryDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
"country": country,
|
||||||
|
"setting": config.ExclusivelyAllowedCountriesSetting,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"net/netip"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -11,12 +12,18 @@ import (
|
|||||||
|
|
||||||
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
|
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
|
||||||
// the setting the client is in.
|
// the setting the client is in.
|
||||||
type denyNets struct{}
|
type denyNets struct {
|
||||||
|
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
|
||||||
|
}
|
||||||
|
|
||||||
func (denyNets) name() string { return "deny_nets" }
|
func (denyNets) name() string { return "deny_nets" }
|
||||||
|
|
||||||
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (d denyNets) check(
|
||||||
for _, netblock := range rq.h.config.DenyNets {
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
|
for _, netblock := range d.netblocks {
|
||||||
if netblock.Contains(rq.client) {
|
if netblock.Contains(rq.client) {
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package waf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
heuristic := denyNets{
|
||||||
|
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
|
||||||
|
}
|
||||||
|
|
||||||
|
// The request has no handler: the heuristic reads only what it holds.
|
||||||
|
inside := &request{
|
||||||
|
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
|
||||||
|
}
|
||||||
|
|
||||||
|
v := heuristic.check(t.Context(), inside, time.Time{})
|
||||||
|
if v == nil {
|
||||||
|
t.Fatal("a client inside the netblock was let through")
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.action != requestlog.ActionClientDenied {
|
||||||
|
t.Errorf(
|
||||||
|
"action = %q, want %q",
|
||||||
|
v.action,
|
||||||
|
requestlog.ActionClientDenied,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
|
||||||
|
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
outside := &request{
|
||||||
|
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
|
||||||
|
t.Errorf("a client outside the netblock was refused: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
-6
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -19,21 +20,28 @@ import (
|
|||||||
// being log, it does nothing more. A zone without a verdict on the client is
|
// being log, it does nothing more. A zone without a verdict on the client is
|
||||||
// asked about it in the background, and the request does not wait for the
|
// asked about it in the background, and the request does not wait for the
|
||||||
// answer. Its figure is the zones that list the client, their keys masked.
|
// answer. Its figure is the zones that list the client, their keys masked.
|
||||||
type dnsbl struct{}
|
type dnsbl struct {
|
||||||
|
dnsbl *reputation.DNSBL
|
||||||
|
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||||
|
hitNoter reputationHitNoter
|
||||||
|
}
|
||||||
|
|
||||||
func (dnsbl) name() string { return "dnsbl" }
|
func (dnsbl) name() string { return "dnsbl" }
|
||||||
|
|
||||||
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
func (d dnsbl) check(
|
||||||
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
|
ctx context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
|
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
|
||||||
rq.dnsblListed = len(listedBy) > 0
|
rq.dnsblListed = len(listedBy) > 0
|
||||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone")
|
||||||
|
|
||||||
for _, zone := range listedBy {
|
for _, zone := range listedBy {
|
||||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed ||
|
if !rq.dnsblListed || d.action != config.ReputationActionDeny {
|
||||||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
||||||
@@ -25,8 +25,13 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for range healthChecks {
|
for range healthChecks {
|
||||||
got := s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "",
|
got := s.ownEndpointRequest(
|
||||||
http.StatusOK)
|
"",
|
||||||
|
http.MethodGet,
|
||||||
|
wafprotocol.HealthPath,
|
||||||
|
"",
|
||||||
|
http.StatusOK,
|
||||||
|
)
|
||||||
|
|
||||||
if string(got.body) != "ok\n" ||
|
if string(got.body) != "ok\n" ||
|
||||||
got.header.Get("Content-Type") != contentType {
|
got.header.Get("Content-Type") != contentType {
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// To add a heuristic, write one file with a type that implements heuristic,
|
// To add a heuristic, write one file with a struct that implements heuristic
|
||||||
// and add one line for it to one of the lists below. Its settings go in
|
// and holds the settings and parts it reads. Add one line for it to one of the
|
||||||
// config.WAF, and the comment at the top of its file names them.
|
// lists that newHandler builds, which hands it those settings and parts. Its
|
||||||
|
// settings go in config.WAF, and the comment at the top of its file names them.
|
||||||
|
|
||||||
// heuristic is one of the checks that decide what waf does with a
|
// heuristic is one of the checks that decide what waf does with a
|
||||||
// request.
|
// request.
|
||||||
@@ -36,80 +37,6 @@ const (
|
|||||||
figureNetblock = "netblock"
|
figureNetblock = "netblock"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The lists of heuristics, each in the order its heuristics run. A client in
|
|
||||||
// SWW_WAF_ALLOW_NETS skips them all.
|
|
||||||
|
|
||||||
// beforeLookup are the heuristics that run before the client's AS number and
|
|
||||||
// country are looked up, so that a client they refuse is not looked up.
|
|
||||||
// header_limits comes first: it only measures and counts, and a request it
|
|
||||||
// refuses causes no other work.
|
|
||||||
func beforeLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
headerLimits{},
|
|
||||||
tagLists{},
|
|
||||||
routeWatchTags{},
|
|
||||||
denyNets{},
|
|
||||||
activeBan{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterLookup are the heuristics that run once the lookup has been made.
|
|
||||||
func afterLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
countries{},
|
|
||||||
blocklists{},
|
|
||||||
dnsbl{},
|
|
||||||
abuseIPDB{},
|
|
||||||
tagDeny{},
|
|
||||||
rateLimits{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// besideLookup are the heuristics that only read the request. Their work runs
|
|
||||||
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
|
|
||||||
// of the request. Their check then applies what that work found, and runs only
|
|
||||||
// once waf takes it.
|
|
||||||
func besideLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
trapPaths{},
|
|
||||||
ruleFiles{},
|
|
||||||
coreRuleSet{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterBesideLookup are the heuristics that run once afterLookup and
|
|
||||||
// besideLookup have both let the request through. tag_deny runs a second time,
|
|
||||||
// on the tag rule_files and core_rule_set can add.
|
|
||||||
func afterBesideLookup() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
tagDeny{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// afterAnswer are the heuristics that run on a request once waf has
|
|
||||||
// answered it, before its decision line is written. They ban the client, and do
|
|
||||||
// not change what the request got: their verdict's action is not used.
|
|
||||||
func afterAnswer() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
refusalLimit{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// onReport are the heuristics that run on an allowed request when rproxy's
|
|
||||||
// report of it comes, before its report line is written. Like afterAnswer, they
|
|
||||||
// only ban.
|
|
||||||
func onReport() []heuristic {
|
|
||||||
return []heuristic{
|
|
||||||
byteLimits{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
|
||||||
// it is called.
|
|
||||||
func (rq *request) runAfterAnswer(ctx context.Context) {
|
|
||||||
rq.runBanningHeuristics(ctx, afterAnswer())
|
|
||||||
}
|
|
||||||
|
|
||||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||||
// and returns the first verdict, or nil when none gives one. The log line's
|
// and returns the first verdict, or nil when none gives one. The log line's
|
||||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
||||||
@@ -118,7 +119,7 @@ func TestOwnEndpointsAreNotInTheHistory(t *testing.T) {
|
|||||||
s.ownEndpointRequest(
|
s.ownEndpointRequest(
|
||||||
"",
|
"",
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
waf.HealthPath,
|
wafprotocol.HealthPath,
|
||||||
"",
|
"",
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
)
|
)
|
||||||
|
|||||||
+5
-25
@@ -3,7 +3,6 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -16,15 +15,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The headers in which an answer of allow gives rproxy the client's AS number
|
|
||||||
// and country, for the app, while SWW_WAF_ADD_LOOKUP_HEADERS is true. Go writes
|
|
||||||
// every header name in this form, as it sends it and as it receives it, so
|
|
||||||
// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive.
|
|
||||||
const (
|
|
||||||
asnHeader = "X-Client-Asn"
|
|
||||||
countryHeader = "X-Client-Country"
|
|
||||||
)
|
|
||||||
|
|
||||||
// lookupResult is what the lookup gave for a request: whether its client was
|
// lookupResult is what the lookup gave for a request: whether its client was
|
||||||
// looked up, the answer, and whether the request skipped the wait for GeoJS.
|
// looked up, the answer, and whether the request skipped the wait for GeoJS.
|
||||||
type lookupResult struct {
|
type lookupResult struct {
|
||||||
@@ -56,7 +46,7 @@ func (rq *request) startLookUp(
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == config.LookupSourceFile {
|
||||||
result <- lookupResult{
|
result <- lookupResult{
|
||||||
@@ -143,8 +133,10 @@ func AddLookup(
|
|||||||
) {
|
) {
|
||||||
limiter.AddLookup(answer.Client, answer.Answered,
|
limiter.AddLookup(answer.Client, answer.Answered,
|
||||||
answer.ASN, answer.ASName, answer.Country)
|
answer.ASN, answer.ASName, answer.Country)
|
||||||
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
|
netblock := banNetblock(
|
||||||
answer.ASN, answer.ASName, answer.Country)
|
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(),
|
||||||
|
)
|
||||||
|
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
|
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
|
||||||
@@ -167,18 +159,6 @@ func GeoJSFailed(
|
|||||||
m.GeoJSFailures.Inc()
|
m.GeoJSFailures.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// setLookupHeaders sets the headers that give the client's AS number and
|
|
||||||
// country, leaving out one that is unknown.
|
|
||||||
func setLookupHeaders(header http.Header, asn, country string) {
|
|
||||||
if asn != "" {
|
|
||||||
header.Set(asnHeader, asn)
|
|
||||||
}
|
|
||||||
|
|
||||||
if country != "" {
|
|
||||||
header.Set(countryHeader, country)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// canBePlaced reports whether a lookup can place addr: private, loopback
|
// canBePlaced reports whether a lookup can place addr: private, loopback
|
||||||
// and link-local addresses have no AS number or country.
|
// and link-local addresses have no AS number or country.
|
||||||
func canBePlaced(addr netip.Addr) bool {
|
func canBePlaced(addr netip.Addr) bool {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -66,7 +67,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
|||||||
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
|
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
|
||||||
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
|
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
|
||||||
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
|
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
|
||||||
{http.MethodPost, waf.HealthPath, "", http.StatusNotFound},
|
{http.MethodPost, wafprotocol.HealthPath, "", http.StatusNotFound},
|
||||||
} {
|
} {
|
||||||
got := s.ownEndpointRequest(
|
got := s.ownEndpointRequest(
|
||||||
tc.authorization,
|
tc.authorization,
|
||||||
|
|||||||
+168
-39
@@ -28,14 +28,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
|
|
||||||
// never sees a path under it.
|
|
||||||
const ownEndpointPrefix = "/_sww/"
|
|
||||||
|
|
||||||
// HealthPath is the health endpoint of waf and of rproxy, which the
|
|
||||||
// container's health check asks.
|
|
||||||
const HealthPath = "/_sww/healthz"
|
|
||||||
|
|
||||||
// MetricsPath is where the metrics are, for a request that carries
|
// MetricsPath is where the metrics are, for a request that carries
|
||||||
// SWW_WAF_METRICS_TOKEN.
|
// SWW_WAF_METRICS_TOKEN.
|
||||||
const MetricsPath = "/_sww/metrics"
|
const MetricsPath = "/_sww/metrics"
|
||||||
@@ -133,15 +125,19 @@ type Server struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newHandler returns the handler with what every request shares, the parts
|
// newHandler returns the handler with what every request shares, the parts
|
||||||
// in params among it.
|
// in params among it. It builds the heuristics, each with the settings and
|
||||||
|
// parts it reads, and the lists they run in.
|
||||||
|
//
|
||||||
|
//nolint:funlen // a few lines for each heuristic, a list that grows with them
|
||||||
func newHandler(params Params) *handler {
|
func newHandler(params Params) *handler {
|
||||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // New fails only for a size below one
|
panic(err) // New fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
return &handler{
|
cfg := params.Config
|
||||||
config: params.Config,
|
h := &handler{
|
||||||
|
config: cfg,
|
||||||
requestLog: params.RequestLog,
|
requestLog: params.RequestLog,
|
||||||
processLog: params.ProcessLog,
|
processLog: params.ProcessLog,
|
||||||
now: params.Now,
|
now: params.Now,
|
||||||
@@ -152,47 +148,142 @@ func newHandler(params Params) *handler {
|
|||||||
geojs: params.GeoJS,
|
geojs: params.GeoJS,
|
||||||
anomalies: params.Anomalies,
|
anomalies: params.Anomalies,
|
||||||
lookupFile: params.LookupFile,
|
lookupFile: params.LookupFile,
|
||||||
lists: params.Lists,
|
|
||||||
dnsbl: params.DNSBL,
|
|
||||||
abuseIPDB: params.AbuseIPDB,
|
|
||||||
routeWatch: params.RouteWatch,
|
|
||||||
rules: params.Rules,
|
|
||||||
coreRuleSet: params.CoreRuleSet,
|
|
||||||
alerts: params.Alerts,
|
|
||||||
// The country list, the headers, the tag settings that name an asn=
|
// The country list, the headers, the tag settings that name an asn=
|
||||||
// or country= tag and the biased thresholds act on the answer before
|
// or country= tag and the biased thresholds act on the answer before
|
||||||
// the request goes on.
|
// the request goes on.
|
||||||
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
|
||||||
params.Config.AddLookupHeaders ||
|
cfg.AddLookupHeaders ||
|
||||||
params.Config.TagsNeedLookup() ||
|
cfg.TagsNeedLookup() ||
|
||||||
biasedThresholdsSet(params.Config),
|
biasedThresholdsSet(cfg),
|
||||||
besideLookupLimit: cmp.Or(
|
besideLookupLimit: cmp.Or(
|
||||||
params.BesideLookupTimeLimit,
|
params.BesideLookupTimeLimit,
|
||||||
besideLookupTimeLimit,
|
besideLookupTimeLimit,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sharedBanMaker := banMaker{
|
||||||
|
ledger: params.Ledger,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
observe: cfg.Observe,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
banScopeV4Prefix: cfg.BanScopeV4Prefix,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
hitNoter := reputationHitNoter{
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.blocklists = blocklists{
|
||||||
|
lists: params.Lists,
|
||||||
|
action: cfg.BlocklistAction,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
}
|
||||||
|
h.rateLimits = rateLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
exemptNets: cfg.RateLimitExemptNets,
|
||||||
|
exemptPaths: cfg.RateLimitExemptPaths,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
biasedThresholds: biasedThresholds{
|
||||||
|
lists: params.Lists,
|
||||||
|
tagLimitPercent: cfg.TagLimitPercent,
|
||||||
|
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
||||||
|
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
||||||
|
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
||||||
|
blocklistAction: cfg.BlocklistAction,
|
||||||
|
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
||||||
|
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
|
},
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
}
|
||||||
|
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
||||||
|
h.ruleFiles = ruleFiles{
|
||||||
|
rules: params.Rules,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
}
|
||||||
|
h.coreRuleSet = coreRuleSet{
|
||||||
|
inspector: params.CoreRuleSet,
|
||||||
|
metrics: params.Metrics,
|
||||||
|
alerts: params.Alerts,
|
||||||
|
mode: cfg.CoreRuleSetMode,
|
||||||
|
exemptPaths: cfg.CoreRuleSetExemptPaths,
|
||||||
|
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
|
||||||
|
observe: cfg.Observe,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
}
|
||||||
|
|
||||||
|
h.beforeLookup = []heuristic{
|
||||||
|
headerLimits{},
|
||||||
|
tagLists{lists: params.Lists},
|
||||||
|
routeWatchTags{
|
||||||
|
routeWatch: params.RouteWatch,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
},
|
||||||
|
denyNets{netblocks: cfg.DenyNets},
|
||||||
|
activeBan{banMaker: sharedBanMaker},
|
||||||
|
}
|
||||||
|
h.afterLookup = []heuristic{
|
||||||
|
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
||||||
|
h.blocklists,
|
||||||
|
dnsbl{
|
||||||
|
dnsbl: params.DNSBL,
|
||||||
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
abuseIPDB{
|
||||||
|
abuseIPDB: params.AbuseIPDB,
|
||||||
|
limiter: params.Limiter,
|
||||||
|
keySet: cfg.AbuseIPDBKey != "",
|
||||||
|
minScore: cfg.AbuseIPDBMinScore,
|
||||||
|
action: cfg.DNSBLAbuseIPDBAction,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
hitNoter: hitNoter,
|
||||||
|
},
|
||||||
|
tagDeny{tags: cfg.TagDeny},
|
||||||
|
h.rateLimits,
|
||||||
|
}
|
||||||
|
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
||||||
|
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
||||||
|
h.afterAnswer = []heuristic{
|
||||||
|
refusalLimit{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
perMinute: cfg.RefusalsLimitPerMinute,
|
||||||
|
banResponse: cfg.BanResponse,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
h.onReport = []heuristic{
|
||||||
|
byteLimits{
|
||||||
|
limiter: params.Limiter,
|
||||||
|
bytesCounted: cfg.BytesCounted,
|
||||||
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||||
|
banMaker: sharedBanMaker,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
// belongs to one request is in a request.
|
// belongs to one request is in a request.
|
||||||
type handler struct {
|
type handler struct {
|
||||||
config *config.WAF
|
config *config.WAF
|
||||||
requestLog io.Writer
|
requestLog io.Writer
|
||||||
processLog *slog.Logger
|
processLog *slog.Logger
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
metrics *metrics.Metrics
|
metrics *metrics.Metrics
|
||||||
limiter *ratelimit.Limiter
|
limiter *ratelimit.Limiter
|
||||||
ledger *bans.Ledger
|
ledger *bans.Ledger
|
||||||
geojs *geojs.GeoJS
|
geojs *geojs.GeoJS
|
||||||
anomalies *anomaly.Counters
|
anomalies *anomaly.Counters
|
||||||
lookupFile *lookup.File
|
lookupFile *lookup.File
|
||||||
lists *reputation.Lists
|
|
||||||
dnsbl *reputation.DNSBL
|
|
||||||
abuseIPDB *reputation.AbuseIPDB
|
|
||||||
routeWatch *routewatch.Client
|
|
||||||
rules *rules.Files
|
|
||||||
coreRuleSet CoreRuleSetInspector
|
|
||||||
alerts *alerts.Queue
|
|
||||||
// waitForGeoJS is true when a setting needs each request's answer from
|
// waitForGeoJS is true when a setting needs each request's answer from
|
||||||
// GeoJS before the request goes on.
|
// GeoJS before the request goes on.
|
||||||
waitForGeoJS bool
|
waitForGeoJS bool
|
||||||
@@ -202,4 +293,42 @@ type handler struct {
|
|||||||
// awaitingReport holds the decision on each answered question by its id
|
// awaitingReport holds the decision on each answered question by its id
|
||||||
// until its report comes, and nil for a question still being handled.
|
// until its report comes, and nil for a question still being handled.
|
||||||
awaitingReport *lru.Cache[string, *decision]
|
awaitingReport *lru.Cache[string, *decision]
|
||||||
|
|
||||||
|
// The lists of heuristics, each in the order its heuristics run. A client
|
||||||
|
// in SWW_WAF_ALLOW_NETS skips them all.
|
||||||
|
|
||||||
|
// beforeLookup are the heuristics that run before the client's AS number
|
||||||
|
// and country are looked up, so that a client they refuse is not looked
|
||||||
|
// up. header_limits comes first: it only measures and counts, and a
|
||||||
|
// request it refuses causes no other work.
|
||||||
|
beforeLookup []heuristic
|
||||||
|
// afterLookup are the heuristics that run once the lookup has been made.
|
||||||
|
afterLookup []heuristic
|
||||||
|
// besideLookup are the heuristics that only read the request. Their work
|
||||||
|
// runs beside the lookup and afterLookup, in findBesideLookup, which
|
||||||
|
// changes nothing of the request. Their check then applies what that work
|
||||||
|
// found, and runs only once waf takes it.
|
||||||
|
besideLookup []heuristic
|
||||||
|
// afterBesideLookup are the heuristics that run once afterLookup and
|
||||||
|
// besideLookup have both let the request through. tag_deny runs a second
|
||||||
|
// time, on the tag rule_files and core_rule_set can add.
|
||||||
|
afterBesideLookup []heuristic
|
||||||
|
// afterAnswer are the heuristics that run on a request once waf has
|
||||||
|
// answered it, before its decision line is written. They ban the client,
|
||||||
|
// and do not change what the request got: their verdict's action is not
|
||||||
|
// used.
|
||||||
|
afterAnswer []heuristic
|
||||||
|
// onReport are the heuristics that run on an allowed request when
|
||||||
|
// rproxy's report of it comes, before its report line is written. Like
|
||||||
|
// afterAnswer, they only ban.
|
||||||
|
onReport []heuristic
|
||||||
|
|
||||||
|
// The heuristics of the lists that other steps of a request use too:
|
||||||
|
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
|
||||||
|
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
|
||||||
|
blocklists blocklists
|
||||||
|
rateLimits rateLimits
|
||||||
|
trapPaths trapPaths
|
||||||
|
ruleFiles ruleFiles
|
||||||
|
coreRuleSet coreRuleSet
|
||||||
}
|
}
|
||||||
+22
-18
@@ -24,6 +24,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -394,11 +395,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"rproxy",
|
InstanceName: "rproxy",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -467,11 +468,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
out,
|
Output: out,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -557,14 +558,17 @@ func addCounters(params *waf.Params) {
|
|||||||
cfg.MetricsBusiestCountriesAndASNs,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
)
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Limits{
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
Limits: ratelimit.Limits{
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
}, cfg.MaxTrackedClients)
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
|||||||
+34
-22
@@ -6,8 +6,10 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"net/netip"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,28 +20,37 @@ import (
|
|||||||
// refused, and bans the client. A request exempt from the rate limits is exempt
|
// refused, and bans the client. A request exempt from the rate limits is exempt
|
||||||
// from the byte limits too. Its figures are the window, the count and the
|
// from the byte limits too. Its figures are the window, the count and the
|
||||||
// limit.
|
// limit.
|
||||||
type rateLimits struct{}
|
type rateLimits struct {
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
|
||||||
|
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
|
||||||
|
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
biasedThresholds biasedThresholds
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (rateLimits) name() string { return "rate_limits" }
|
func (rateLimits) name() string { return "rate_limits" }
|
||||||
|
|
||||||
func (rateLimits) check(
|
func (r rateLimits) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
rq.counted = rq.countedByRateLimits()
|
rq.counted = r.countedByRateLimits(rq)
|
||||||
if !rq.counted {
|
if !rq.counted {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages()
|
rq.rateLimitPercent, rq.bytesLimitPercent =
|
||||||
|
r.biasedThresholds.limitPercentages(rq)
|
||||||
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
|
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
|
||||||
rq.rateLimitPercent.logged()
|
rq.rateLimitPercent.logged()
|
||||||
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
||||||
rq.bytesLimitPercent.logged()
|
rq.bytesLimitPercent.logged()
|
||||||
|
|
||||||
counts, hit, over := rq.h.limiter.Count(
|
counts, hit, over := r.limiter.Count(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
now,
|
now,
|
||||||
rq.rateLimitPercent.percent,
|
rq.rateLimitPercent.percent,
|
||||||
)
|
)
|
||||||
@@ -49,28 +60,29 @@ func (rateLimits) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banForLimit(now, hit, rq.h.config.BanResponse)
|
r.banMaker.banForLimit(rq, now, hit, r.banResponse)
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
|
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
|
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
|
||||||
// request at now, whatever its client's limit percentages: the rate limits
|
// at now, whatever its client's limit percentages: the rate limits count it,
|
||||||
// count it, and one more request from its client takes it over a whole rate
|
// and one more request from its client takes it over a whole rate limit. It
|
||||||
// limit. It counts nothing.
|
// counts nothing.
|
||||||
func (rq *request) overAWholeRateLimit(now time.Time) bool {
|
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
|
||||||
return rq.countedByRateLimits() &&
|
return r.countedByRateLimits(rq) &&
|
||||||
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
|
r.limiter.OverARateLimit(
|
||||||
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
|
now,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// countedByRateLimits reports whether the rate limits count the request: its
|
// countedByRateLimits reports whether the rate limits count rq: its client is
|
||||||
// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
|
// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under
|
||||||
// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
|
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
|
||||||
func (rq *request) countedByRateLimits() bool {
|
func (r rateLimits) countedByRateLimits(rq *request) bool {
|
||||||
cfg := rq.h.config
|
return !isInside(rq.client, r.exemptNets) &&
|
||||||
|
!pathExempt(rq.in, r.exemptPaths)
|
||||||
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
|
||||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,17 +19,22 @@ import (
|
|||||||
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
|
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
|
||||||
// heuristics skip, is not counted, and nothing is while the limit is off.
|
// heuristics skip, is not counted, and nothing is while the limit is off.
|
||||||
// Its figures are the window, the count and the limit.
|
// Its figures are the window, the count and the limit.
|
||||||
type refusalLimit struct{}
|
type refusalLimit struct {
|
||||||
|
limiter *ratelimit.Limiter
|
||||||
|
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
|
||||||
|
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (refusalLimit) name() string { return "refusal_limit" }
|
func (refusalLimit) name() string { return "refusal_limit" }
|
||||||
|
|
||||||
func (refusalLimit) check(
|
func (r refusalLimit) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
cfg := rq.h.config
|
if r.perMinute == 0 {
|
||||||
if cfg.RefusalsLimitPerMinute == 0 {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,10 +42,10 @@ func (refusalLimit) check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
hit, over := rq.h.limiter.CountRefusal(
|
hit, over := r.limiter.CountRefusal(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
now,
|
now,
|
||||||
cfg.RefusalsLimitPerMinute,
|
r.perMinute,
|
||||||
)
|
)
|
||||||
if !over {
|
if !over {
|
||||||
return nil
|
return nil
|
||||||
@@ -54,10 +60,10 @@ func (refusalLimit) check(
|
|||||||
status = http.StatusForbidden
|
status = http.StatusForbidden
|
||||||
case requestlog.ActionTemporarilyBanned,
|
case requestlog.ActionTemporarilyBanned,
|
||||||
requestlog.ActionPermanentlyBanned:
|
requestlog.ActionPermanentlyBanned:
|
||||||
status = cfg.BanResponse
|
status = r.banResponse
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banForLimit(now, hit, status)
|
r.banMaker.banForLimit(rq, now, hit, status)
|
||||||
|
|
||||||
return &verdict{figures: limitFigures(hit)}
|
return &verdict{figures: limitFigures(hit)}
|
||||||
}
|
}
|
||||||
@@ -5,22 +5,35 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// reputationHitNoter notes the reputation sources that list a request's
|
||||||
|
// client, for the reputation heuristics.
|
||||||
|
type reputationHitNoter struct {
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
alerts *alerts.Queue
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
}
|
||||||
|
|
||||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
// DNSBL zones, their keys masked, that list rq's client, as noteHit does,
|
||||||
// with reason.
|
// with reason.
|
||||||
func (rq *request) noteListed(sources []string, reason string) {
|
func (n reputationHitNoter) noteListed(
|
||||||
|
rq *request, sources []string, reason string,
|
||||||
|
) {
|
||||||
for _, source := range sources {
|
for _, source := range sources {
|
||||||
rq.noteHit(bans.ReputationHit{Source: source}, reason)
|
n.noteHit(rq, bans.ReputationHit{Source: source}, reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// noteHit adds hit's source, which lists the client, to the log line's
|
// noteHit adds hit's source, which lists rq's client, to the log line's
|
||||||
// reputation, and hit to the notes of a ban the request makes, counts the
|
// reputation, and hit to the notes of a ban the request makes, counts the
|
||||||
// source in the metrics, and raises a reputation_hit alert with reason,
|
// source in the metrics, and raises a reputation_hit alert with reason,
|
||||||
// whose detail gives hit's source and score.
|
// whose detail gives hit's source and score.
|
||||||
func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
func (n reputationHitNoter) noteHit(
|
||||||
|
rq *request, hit bans.ReputationHit, reason string,
|
||||||
|
) {
|
||||||
detail := map[string]any{"source": hit.Source}
|
detail := map[string]any{"source": hit.Source}
|
||||||
if hit.Score != nil {
|
if hit.Score != nil {
|
||||||
detail["score"] = *hit.Score
|
detail["score"] = *hit.Score
|
||||||
@@ -28,12 +41,12 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
|||||||
|
|
||||||
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
|
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
|
||||||
rq.reputation = append(rq.reputation, hit)
|
rq.reputation = append(rq.reputation, hit)
|
||||||
rq.h.metrics.ReputationHit(hit.Source)
|
n.metrics.ReputationHit(hit.Source)
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
n.alerts.Raise(alerts.Alert{
|
||||||
Event: alerts.EventReputationHit,
|
Event: alerts.EventReputationHit,
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: clientGroup(rq.h.config, rq.client),
|
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
ASName: rq.line.ASName,
|
ASName: rq.line.ASName,
|
||||||
Country: rq.line.Country,
|
Country: rq.line.Country,
|
||||||
|
|||||||
+23
-8
@@ -122,8 +122,11 @@ func (h *handler) startRequest(
|
|||||||
RequestID: requestID(r, isQuestion),
|
RequestID: requestID(r, isQuestion),
|
||||||
PeerIP: peer.String(),
|
PeerIP: peer.String(),
|
||||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||||
ClientGroup: clientGroup(h.config, client).String(),
|
ClientGroup: clientGroup(
|
||||||
ContentType: r.Header.Get("Content-Type"),
|
h.config.IPv6GroupPrefix,
|
||||||
|
client,
|
||||||
|
).String(),
|
||||||
|
ContentType: r.Header.Get("Content-Type"),
|
||||||
RequestHeaders: requestHeaders(
|
RequestHeaders: requestHeaders(
|
||||||
r,
|
r,
|
||||||
h.config.LogRequestHeaders,
|
h.config.LogRequestHeaders,
|
||||||
@@ -233,7 +236,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
|
|||||||
|
|
||||||
now := rq.h.now()
|
now := rq.h.now()
|
||||||
|
|
||||||
v := rq.runHeuristics(ctx, beforeLookup(), now)
|
v := rq.runHeuristics(ctx, rq.h.beforeLookup, now)
|
||||||
if v != nil {
|
if v != nil {
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
@@ -265,7 +268,7 @@ func (rq *request) runAfterLookup(
|
|||||||
// again: a request counted at the time it arrived could be counted
|
// again: a request counted at the time it arrived could be counted
|
||||||
// after a later one of its client's, which the rate limits would take
|
// after a later one of its client's, which the rate limits would take
|
||||||
// for the clock set back.
|
// for the clock set back.
|
||||||
return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
|
return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// pathExempt reports whether r is exempt under prefixes,
|
// pathExempt reports whether r is exempt under prefixes,
|
||||||
@@ -345,6 +348,16 @@ func timing(start, end time.Time) *float64 {
|
|||||||
return new(requestlog.Milliseconds(end.Sub(start)))
|
return new(requestlog.Milliseconds(end.Sub(start)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
|
||||||
|
// It then counts the request for the anomaly thresholds and adds it to its
|
||||||
|
// client's history. list runs first, so that a refusal limit it finds broken
|
||||||
|
// is in that history. ctx is the request's own context.
|
||||||
|
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
|
||||||
|
rq.runBanningHeuristics(ctx, list)
|
||||||
|
rq.countAnomalies()
|
||||||
|
rq.addToHistory()
|
||||||
|
}
|
||||||
|
|
||||||
// addToHistory adds the request, which has ended, to its client's
|
// addToHistory adds the request, which has ended, to its client's
|
||||||
// history, and counts its offences in the metrics, and then the lookup's
|
// history, and counts its offences in the metrics, and then the lookup's
|
||||||
// answer about the client, as answerAtTheEnd gives it, to that history and
|
// answer about the client, as answerAtTheEnd gives it, to that history and
|
||||||
@@ -365,7 +378,7 @@ func (rq *request) addToHistory() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rq.h.limiter.AddToHistory(
|
rq.h.limiter.AddToHistory(
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||||
rq.h.now(),
|
rq.h.now(),
|
||||||
request,
|
request,
|
||||||
)
|
)
|
||||||
@@ -394,11 +407,11 @@ func (rq *request) countAnomalies() {
|
|||||||
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
||||||
Host: rq.site,
|
Host: rq.site,
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
ClientGroup: clientGroup(rq.h.config, rq.client),
|
ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||||
ASN: answer.ASN,
|
ASN: answer.ASN,
|
||||||
ASName: answer.ASName,
|
ASName: answer.ASName,
|
||||||
Country: answer.Country,
|
Country: answer.Country,
|
||||||
Bytes: rq.countedBytes(),
|
Bytes: rq.countedBytes(rq.h.config.BytesCounted),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -425,7 +438,9 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
|||||||
return rq.lookupAnswer, true
|
return rq.lookupAnswer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
|
return rq.h.geojs.Kept(
|
||||||
|
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// responseWriter is waf's answer to a request. It notes the status and
|
// responseWriter is waf's answer to a request. It notes the status and
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -248,7 +248,7 @@ func TestTimingsAreInOrder(t *testing.T) {
|
|||||||
wantStatus(t, do(t, req), http.StatusForbidden)
|
wantStatus(t, do(t, req), http.StatusForbidden)
|
||||||
refused := out.endedLines(t, 2)[1]
|
refused := out.endedLines(t, 2)[1]
|
||||||
|
|
||||||
wantStatus(t, get(t, server.Addr, waf.HealthPath), http.StatusOK)
|
wantStatus(t, get(t, server.Addr, wafprotocol.HealthPath), http.StatusOK)
|
||||||
health := out.endedLines(t, 3)[2]
|
health := out.endedLines(t, 3)[2]
|
||||||
|
|
||||||
// A request passed to the app has every timing; one refused, none of
|
// A request passed to the app has every timing; one refused, none of
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
)
|
)
|
||||||
|
|
||||||
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
|
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
|
||||||
@@ -17,22 +18,26 @@ import (
|
|||||||
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
|
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
|
||||||
// asked for by the request's address, and a route without an origin AS gives only
|
// asked for by the request's address, and a route without an origin AS gives only
|
||||||
// the prefix= tag.
|
// the prefix= tag.
|
||||||
type routeWatchTags struct{}
|
type routeWatchTags struct {
|
||||||
|
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
|
routeWatch *routewatch.Client
|
||||||
|
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||||
|
}
|
||||||
|
|
||||||
func (routeWatchTags) name() string { return "routewatch_tags" }
|
func (routeWatchTags) name() string { return "routewatch_tags" }
|
||||||
|
|
||||||
func (routeWatchTags) check(
|
func (r routeWatchTags) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
if rq.h.routeWatch == nil {
|
if r.routeWatch == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
route, missing := rq.h.routeWatch.Route(
|
route, missing := r.routeWatch.Route(
|
||||||
ctx,
|
ctx,
|
||||||
clientGroup(rq.h.config, rq.client),
|
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||||
rq.client,
|
rq.client,
|
||||||
)
|
)
|
||||||
if missing != "" {
|
if missing != "" {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
@@ -22,11 +23,15 @@ import (
|
|||||||
// mode raises the alert for the ban
|
// mode raises the alert for the ban
|
||||||
// it would have made. Either rule's match is noted as an offence, for the
|
// it would have made. Either rule's match is noted as an offence, for the
|
||||||
// client's history. Its figures are the rule's id and action.
|
// client's history. Its figures are the rule's id and action.
|
||||||
type ruleFiles struct{}
|
type ruleFiles struct {
|
||||||
|
rules *rules.Files
|
||||||
|
metrics *metrics.Metrics
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (ruleFiles) name() string { return "rule_files" }
|
func (ruleFiles) name() string { return "rule_files" }
|
||||||
|
|
||||||
func (ruleFiles) check(
|
func (r ruleFiles) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -35,7 +40,7 @@ func (ruleFiles) check(
|
|||||||
|
|
||||||
for _, rule := range matched {
|
for _, rule := range matched {
|
||||||
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
||||||
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
|
r.metrics.RuleMatched(rule.ID, rule.Action)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(matched) == 0 {
|
if len(matched) == 0 {
|
||||||
@@ -60,7 +65,8 @@ func (ruleFiles) check(
|
|||||||
rq.attack = true
|
rq.attack = true
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: rq.banForAttack(
|
action: r.banMaker.banForAttack(
|
||||||
|
rq,
|
||||||
now,
|
now,
|
||||||
bans.Notes{RuleID: last.ID, Target: last.Target},
|
bans.Notes{RuleID: last.ID, Target: last.Target},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -12,13 +12,19 @@ import (
|
|||||||
|
|
||||||
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
|
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
|
||||||
// the first such tag, in sorted order.
|
// the first such tag, in sorted order.
|
||||||
type tagDeny struct{}
|
type tagDeny struct {
|
||||||
|
tags []string // SWW_WAF_TAG_DENY
|
||||||
|
}
|
||||||
|
|
||||||
func (tagDeny) name() string { return "tag_deny" }
|
func (tagDeny) name() string { return "tag_deny" }
|
||||||
|
|
||||||
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (t tagDeny) check(
|
||||||
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
for _, tag := range rq.line.Tags {
|
for _, tag := range rq.line.Tags {
|
||||||
if slices.Contains(rq.h.config.TagDeny, tag) {
|
if slices.Contains(t.tags, tag) {
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
figures: map[string]any{"tag": tag},
|
figures: map[string]any{"tag": tag},
|
||||||
|
|||||||
@@ -6,17 +6,25 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
|
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
|
||||||
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
||||||
// first so that every refusal's log line carries these tags.
|
// first so that every refusal's log line carries these tags.
|
||||||
type tagLists struct{}
|
type tagLists struct {
|
||||||
|
lists *reputation.Lists
|
||||||
|
}
|
||||||
|
|
||||||
func (tagLists) name() string { return "tag_lists" }
|
func (tagLists) name() string { return "tag_lists" }
|
||||||
|
|
||||||
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (t tagLists) check(
|
||||||
for _, name := range rq.h.lists.TagsOf(rq.client) {
|
_ context.Context,
|
||||||
|
rq *request,
|
||||||
|
_ time.Time,
|
||||||
|
) *verdict {
|
||||||
|
for _, name := range t.lists.TagsOf(rq.client) {
|
||||||
rq.addTag(name)
|
rq.addTag(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,11 +16,14 @@ import (
|
|||||||
// it bans the client's netblock, or in observe mode raises the alert for the
|
// it bans the client's netblock, or in observe mode raises the alert for the
|
||||||
// ban it would have made. The ban's notes give the path as sent, and so does
|
// ban it would have made. The ban's notes give the path as sent, and so does
|
||||||
// its figure.
|
// its figure.
|
||||||
type trapPaths struct{}
|
type trapPaths struct {
|
||||||
|
paths []string // SWW_WAF_TRAP_PATHS
|
||||||
|
banMaker banMaker
|
||||||
|
}
|
||||||
|
|
||||||
func (trapPaths) name() string { return "trap_paths" }
|
func (trapPaths) name() string { return "trap_paths" }
|
||||||
|
|
||||||
func (trapPaths) check(
|
func (t trapPaths) check(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -33,18 +36,22 @@ func (trapPaths) check(
|
|||||||
rq.attack = true
|
rq.attack = true
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: rq.banForAttack(now, bans.Notes{TrapPath: path}),
|
action: t.banMaker.banForAttack(
|
||||||
|
rq,
|
||||||
|
now,
|
||||||
|
bans.Notes{TrapPath: path},
|
||||||
|
),
|
||||||
figures: map[string]any{"path": path},
|
figures: map[string]any{"path": path},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// trapPath returns the request's path as sent, when its path as a path rule
|
// trapPath returns rq's path as sent, when its path as a path rule sees it,
|
||||||
// sees it, without the query, as sent or once percent-decoded, is in
|
// without the query, as sent or once percent-decoded, is in
|
||||||
// SWW_WAF_TRAP_PATHS, and "" otherwise.
|
// SWW_WAF_TRAP_PATHS, and "" otherwise.
|
||||||
func (rq *request) trapPath() string {
|
func (t trapPaths) trapPath(rq *request) string {
|
||||||
path := rules.Path(rq.in)
|
path := rules.Path(rq.in)
|
||||||
if !slices.Contains(rq.h.config.TrapPaths, path) &&
|
if !slices.Contains(t.paths, path) &&
|
||||||
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
|
!slices.Contains(t.paths, rules.DecodeOnce(path)) {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+82
-259
@@ -1,75 +1,21 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The headers of rproxy's question and of waf's answer, as
|
|
||||||
// "waf" in SPEC.md gives them.
|
|
||||||
const (
|
|
||||||
// ClientHeader marks a request as rproxy's question, and gives the
|
|
||||||
// visitor's address.
|
|
||||||
ClientHeader = "X-Sww-Client"
|
|
||||||
// PeerHeader gives the address of rproxy's own TCP peer.
|
|
||||||
PeerHeader = "X-Sww-Peer"
|
|
||||||
// ContentLengthHeader gives the body length the visitor announced.
|
|
||||||
ContentLengthHeader = "X-Sww-Content-Length"
|
|
||||||
// DispositionHeader gives waf's answer.
|
|
||||||
DispositionHeader = "X-Sww-Disposition"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The values of DispositionHeader.
|
|
||||||
const (
|
|
||||||
// DispositionAllow tells rproxy to pass the request to the app.
|
|
||||||
DispositionAllow = "allow"
|
|
||||||
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
|
||||||
// headers and body.
|
|
||||||
DispositionDeny = "deny"
|
|
||||||
// DispositionClose tells rproxy to close the visitor's connection
|
|
||||||
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
|
|
||||||
// waf closing the question's connection instead would read as
|
|
||||||
// waf being down, and rproxy would then let the request through.
|
|
||||||
DispositionClose = "close"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ReportPath is where rproxy sends the report of each request waf
|
|
||||||
// answered.
|
|
||||||
const ReportPath = "/_sww/report"
|
|
||||||
|
|
||||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
|
||||||
const QuestionBodyMaxBytes = 64 << 10
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// maxAwaitingReport is the most answered requests held until their
|
// maxAwaitingReport is the most answered requests held until their
|
||||||
// reports come. Past it, the request answered longest ago is dropped.
|
// reports come. Past it, the request answered longest ago is dropped.
|
||||||
maxAwaitingReport = 10000
|
maxAwaitingReport = 10000
|
||||||
// maxRequestIDBytes is the longest id a question may give.
|
|
||||||
maxRequestIDBytes = 128
|
|
||||||
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
|
||||||
// far above any report's.
|
|
||||||
reportBodyMaxBytes = 1 << 20
|
|
||||||
// maxHTTPStatus is the highest status a report may give.
|
|
||||||
maxHTTPStatus = 599
|
|
||||||
// maxReportedBytes is the most bytes a report may give for each way, 1
|
|
||||||
// PiB, so that adding them up cannot overflow.
|
|
||||||
maxReportedBytes = 1 << 50
|
|
||||||
// maxReportedMilliseconds is the longest duration a report may give,
|
|
||||||
// about 31 years, so that it fits a time.Duration.
|
|
||||||
maxReportedMilliseconds = 1e12
|
|
||||||
// wafTimeout bounds how long a request to waf takes to
|
// wafTimeout bounds how long a request to waf takes to
|
||||||
// arrive, and its answer to leave, and wafIdleTimeout how long a
|
// arrive, and its answer to leave, and wafIdleTimeout how long a
|
||||||
// kept-open connection waits for its next request.
|
// kept-open connection waits for its next request.
|
||||||
@@ -77,26 +23,6 @@ const (
|
|||||||
wafIdleTimeout = 120 * time.Second
|
wafIdleTimeout = 120 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
errReportFieldMissing = errors.New("a field is missing")
|
|
||||||
errReportValue = errors.New("a value is out of range")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
|
||||||
// answer has ended, its fields named as the request log's of the same
|
|
||||||
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
|
||||||
// if nothing. The durations are in milliseconds.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
|
||||||
type Report struct {
|
|
||||||
RequestID string `json:"request_id"`
|
|
||||||
Status int `json:"status"`
|
|
||||||
RequestBytes int64 `json:"request_bytes"`
|
|
||||||
ResponseBytes int64 `json:"response_bytes"`
|
|
||||||
DurationTotalMS float64 `json:"duration_total_ms"`
|
|
||||||
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns the server `sww waf` runs. It answers
|
// New returns the server `sww waf` runs. It answers
|
||||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
||||||
// passes nothing to an app. rproxy applies the size and time limits to the
|
// passes nothing to an app. rproxy applies the size and time limits to the
|
||||||
@@ -123,19 +49,19 @@ func New(params Params) *Server {
|
|||||||
GeoJS: h.geojs,
|
GeoJS: h.geojs,
|
||||||
Anomalies: h.anomalies,
|
Anomalies: h.anomalies,
|
||||||
LookupFile: h.lookupFile,
|
LookupFile: h.lookupFile,
|
||||||
Lists: h.lists,
|
Lists: params.Lists,
|
||||||
DNSBL: h.dnsbl,
|
DNSBL: params.DNSBL,
|
||||||
AbuseIPDB: h.abuseIPDB,
|
AbuseIPDB: params.AbuseIPDB,
|
||||||
RouteWatch: h.routeWatch,
|
RouteWatch: params.RouteWatch,
|
||||||
Metrics: h.metrics,
|
Metrics: h.metrics,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServeHTTP handles one request to waf: a request carrying
|
// ServeHTTP handles one request to waf: a request carrying
|
||||||
// ClientHeader is rproxy's question, and any other is for waf
|
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
|
||||||
// itself.
|
// itself.
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
if len(r.Header.Values(ClientHeader)) > 0 {
|
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
||||||
h.answerQuestion(w, r)
|
h.answerQuestion(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -154,112 +80,86 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
|
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
|
||||||
rq.overAWholeRateLimit(now)
|
|
||||||
|
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
|
||||||
|
rq.h.rateLimits.overAWholeRateLimit(rq, now)
|
||||||
}
|
}
|
||||||
|
|
||||||
// answerQuestion runs the checks on the visitor's request a question
|
// answerQuestion runs the checks on the visitor's request a question
|
||||||
// carries, and answers rproxy what to do with it. A denied request's counting
|
// carries, and answers rproxy what to do with it. A denied request's counting
|
||||||
// is finished here, and an allowed one's by its report.
|
// is finished here, and an allowed one's by its report.
|
||||||
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||||
client, body, ok := h.readQuestion(w, r)
|
question, ok := h.readQuestion(w, r)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// One that cannot be read leaves the log line's peer_ip invalid.
|
|
||||||
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
|
||||||
|
|
||||||
rq := h.startRequest(
|
rq := h.startRequest(
|
||||||
w,
|
w,
|
||||||
asTheVisitorSentIt(r, body),
|
question.Visitor,
|
||||||
client.Unmap(),
|
question.Client,
|
||||||
peer.Unmap(),
|
question.Peer,
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
// The last deferred runs first, so that what is held has the finished log
|
// These run even when a check panics. finish runs first, so that what is
|
||||||
// line. Go's server sends the answer only once this function has
|
// held has the finished log line. Go's server sends the answer only once
|
||||||
// returned, so the report cannot come before it is held.
|
// this function has returned, so the report cannot come before it is held.
|
||||||
defer rq.holdForReport()
|
defer func() {
|
||||||
defer rq.finish()
|
rq.finish()
|
||||||
|
rq.holdForReport()
|
||||||
|
}()
|
||||||
|
|
||||||
rq.site = r.Host
|
rq.site = r.Host
|
||||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||||
refused := rq.check(r.Context())
|
refused := rq.check(r.Context())
|
||||||
rq.checked = time.Now()
|
rq.checked = time.Now()
|
||||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
underOwnPrefix := strings.HasPrefix(
|
||||||
|
r.URL.Path,
|
||||||
|
wafprotocol.OwnEndpointPrefix,
|
||||||
|
)
|
||||||
|
|
||||||
if refused == nil && !underOwnPrefix {
|
if refused == nil && !underOwnPrefix {
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
|
||||||
defer rq.runAfterAnswer(r.Context())
|
|
||||||
|
|
||||||
rq.allow()
|
rq.allow()
|
||||||
|
// In observe mode, what enforce mode would have refused counts here.
|
||||||
|
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
|
||||||
// a broken refusal limit is in the client's history.
|
|
||||||
defer rq.addToHistory()
|
|
||||||
defer rq.countAnomalies()
|
|
||||||
defer rq.runAfterAnswer(r.Context())
|
|
||||||
|
|
||||||
if refused != nil {
|
if refused != nil {
|
||||||
rq.deny(*refused)
|
rq.deny(*refused)
|
||||||
|
} else {
|
||||||
return
|
// The app never sees this prefix. waf's own endpoints are at its
|
||||||
|
// own address, so this answers as a path with no endpoint does.
|
||||||
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
|
wafprotocol.SetAnswerHeaders(
|
||||||
|
rq.out.Header(),
|
||||||
|
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
||||||
|
)
|
||||||
|
http.Error(
|
||||||
|
rq.out,
|
||||||
|
http.StatusText(http.StatusNotFound),
|
||||||
|
http.StatusNotFound,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The app never sees this prefix. waf's own endpoints are at its
|
rq.banAndCount(r.Context(), rq.h.afterAnswer)
|
||||||
// own address, so this answers as a path with no endpoint does.
|
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
http.StatusText(http.StatusNotFound),
|
|
||||||
http.StatusNotFound,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// readQuestion returns the visitor's address and the body a question carries,
|
// readQuestion reads a question, and reports whether waf takes it. It answers
|
||||||
// and whether waf takes the question. It answers a question it does not
|
// a question it does not take with 400 and no disposition. rproxy passes that
|
||||||
// take with 400 and no disposition. rproxy passes that 400 to the visitor,
|
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
|
||||||
// and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does not apply. The id of
|
// not apply. The id of a question waf takes is held from here on, without a
|
||||||
// a question waf takes is held from here on, without a decision yet, so
|
// decision yet, so that another question with that id is refused while this
|
||||||
// that another question with that id is refused while this one is being
|
// one is being handled.
|
||||||
// handled.
|
|
||||||
//
|
//
|
||||||
// The body is read before any check, since Go's server does not send an answer
|
// A question whose body stops arriving gets no answer, and is not looked at.
|
||||||
// until it has read the body. A question whose body stops arriving gets no
|
|
||||||
// answer, and is not looked at.
|
|
||||||
func (h *handler) readQuestion(
|
func (h *handler) readQuestion(
|
||||||
w http.ResponseWriter, r *http.Request,
|
w http.ResponseWriter, r *http.Request,
|
||||||
) (netip.Addr, []byte, bool) {
|
) (wafprotocol.Question, bool) {
|
||||||
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
question, err := wafprotocol.ReadQuestion(w, r)
|
||||||
if err != nil || client.Zone() != "" {
|
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
|
||||||
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
id := r.Header.Get(requestIDHeader)
|
|
||||||
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
|
||||||
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(
|
|
||||||
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
|
|
||||||
)
|
|
||||||
|
|
||||||
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
|
||||||
h.refuseQuestion(w, "the body is over 64 KiB")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
h.processLog.Warn(
|
h.processLog.Warn(
|
||||||
"a question's body stopped arriving",
|
"a question's body stopped arriving",
|
||||||
"error",
|
"error",
|
||||||
@@ -270,17 +170,23 @@ func (h *handler) readQuestion(
|
|||||||
panic(http.ErrAbortHandler)
|
panic(http.ErrAbortHandler)
|
||||||
}
|
}
|
||||||
|
|
||||||
held, _ := h.awaitingReport.ContainsOrAdd(id, nil)
|
if err != nil {
|
||||||
|
h.refuseQuestion(w, err.Error())
|
||||||
|
|
||||||
|
return question, false
|
||||||
|
}
|
||||||
|
|
||||||
|
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
|
||||||
if held {
|
if held {
|
||||||
h.refuseQuestion(
|
h.refuseQuestion(
|
||||||
w,
|
w,
|
||||||
requestIDHeader+" is the id of a request waf holds",
|
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
|
||||||
)
|
)
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
return question, false
|
||||||
}
|
}
|
||||||
|
|
||||||
return client, body, true
|
return question, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// refuseQuestion answers a question waf does not take, for reason.
|
// refuseQuestion answers a question waf does not take, for reason.
|
||||||
@@ -289,48 +195,18 @@ func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
|||||||
http.Error(w, reason, http.StatusBadRequest)
|
http.Error(w, reason, http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
// allow answers a question with wafprotocol.DispositionAllow and, while
|
||||||
// log line are to see it: without the headers rproxy adds for waf,
|
|
||||||
// with body, which has been read of r, and with the length the visitor
|
|
||||||
// announced, as -1 when it announced none. The question's own length covers
|
|
||||||
// only the part of the body it carries.
|
|
||||||
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
|
||||||
sent := r.Clone(r.Context())
|
|
||||||
announced := sent.Header.Get(ContentLengthHeader)
|
|
||||||
|
|
||||||
sent.Header.Del(ClientHeader)
|
|
||||||
sent.Header.Del(PeerHeader)
|
|
||||||
sent.Header.Del(ContentLengthHeader)
|
|
||||||
sent.Header.Del("Content-Length")
|
|
||||||
|
|
||||||
sent.ContentLength = -1
|
|
||||||
|
|
||||||
length, err := strconv.ParseInt(announced, 10, 64)
|
|
||||||
if err == nil && length >= 0 {
|
|
||||||
sent.ContentLength = length
|
|
||||||
sent.Header.Set("Content-Length", announced)
|
|
||||||
}
|
|
||||||
|
|
||||||
sent.Body = http.NoBody
|
|
||||||
if len(body) > 0 {
|
|
||||||
sent.Body = io.NopCloser(bytes.NewReader(body))
|
|
||||||
}
|
|
||||||
|
|
||||||
return sent
|
|
||||||
}
|
|
||||||
|
|
||||||
// allow answers a question with DispositionAllow and, while
|
|
||||||
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
||||||
func (rq *request) allow() {
|
func (rq *request) allow() {
|
||||||
rq.allowed = true
|
rq.allowed = true
|
||||||
|
|
||||||
header := rq.out.Header()
|
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
||||||
header.Set(DispositionHeader, DispositionAllow)
|
|
||||||
|
|
||||||
if rq.h.config.AddLookupHeaders {
|
if rq.h.config.AddLookupHeaders {
|
||||||
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
|
answer.ASN = rq.line.ASN
|
||||||
|
answer.Country = rq.line.Country
|
||||||
}
|
}
|
||||||
|
|
||||||
|
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
||||||
rq.out.WriteHeader(http.StatusOK)
|
rq.out.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -348,13 +224,19 @@ func (rq *request) deny(r refusal) {
|
|||||||
if r.status == 0 {
|
if r.status == 0 {
|
||||||
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
||||||
// status stays 0: the visitor is sent nothing.
|
// status stays 0: the visitor is sent nothing.
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionClose)
|
wafprotocol.SetAnswerHeaders(
|
||||||
|
rq.out.Header(),
|
||||||
|
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
|
||||||
|
)
|
||||||
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
wafprotocol.SetAnswerHeaders(
|
||||||
|
rq.out.Header(),
|
||||||
|
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
||||||
|
)
|
||||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,7 +244,7 @@ func (rq *request) deny(r refusal) {
|
|||||||
// runs on it: the address is on the private network, where the peers are
|
// runs on it: the address is on the private network, where the peers are
|
||||||
// rproxy and the admin's tools, not visitors.
|
// rproxy and the admin's tools, not visitors.
|
||||||
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
|
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
|
||||||
h.takeReport(w, r)
|
h.takeReport(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -373,7 +255,7 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
|||||||
rq := h.startRequest(w, r, peer, peer, false)
|
rq := h.startRequest(w, r, peer, peer, false)
|
||||||
defer rq.finish()
|
defer rq.finish()
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
||||||
rq.answerHealth()
|
rq.answerHealth()
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -382,11 +264,11 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
|||||||
rq.answerAdmin()
|
rq.answerAdmin()
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeReport answers POST ReportPath: it finishes the request the report
|
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
|
||||||
// names, if it holds that request's decision, and ignores the report
|
// report names, if it holds that request's decision, and ignores the report
|
||||||
// otherwise.
|
// otherwise.
|
||||||
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||||
report, err := readReport(w, r)
|
report, err := wafprotocol.ReadReport(w, r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
||||||
@@ -404,63 +286,6 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readReport reads a report from r's body, which must be one JSON object with
|
|
||||||
// all of Report's fields, in the ranges SPEC.md gives.
|
|
||||||
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
|
||||||
var report Report
|
|
||||||
|
|
||||||
body, err := io.ReadAll(
|
|
||||||
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("read the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var fields map[string]json.RawMessage
|
|
||||||
|
|
||||||
err = json.Unmarshal(body, &fields)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("decode the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"request_id", "status", "request_bytes", "response_bytes",
|
|
||||||
"duration_total_ms", "duration_upstream_total_ms",
|
|
||||||
} {
|
|
||||||
value, there := fields[name]
|
|
||||||
if !there || string(value) == "null" {
|
|
||||||
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(body, &report)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("decode the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return report, report.checkRanges()
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRanges returns an error for a value of the report that cannot be true.
|
|
||||||
func (report *Report) checkRanges() error {
|
|
||||||
switch {
|
|
||||||
case report.Status != 0 &&
|
|
||||||
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
|
|
||||||
return fmt.Errorf("%w: status", errReportValue)
|
|
||||||
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
|
||||||
return fmt.Errorf("%w: request_bytes", errReportValue)
|
|
||||||
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
|
||||||
return fmt.Errorf("%w: response_bytes", errReportValue)
|
|
||||||
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
|
|
||||||
return fmt.Errorf("%w: duration_total_ms", errReportValue)
|
|
||||||
case report.DurationUpstreamTotalMS < 0 ||
|
|
||||||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
|
|
||||||
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// finishWith finishes a request with what its report says, and writes the
|
// finishWith finishes a request with what its report says, and writes the
|
||||||
// request's last log line, which holds its decision's fields and the report's.
|
// request's last log line, which holds its decision's fields and the report's.
|
||||||
// For an allowed request it first runs the heuristics of onReport, and adds the
|
// For an allowed request it first runs the heuristics of onReport, and adds the
|
||||||
@@ -470,7 +295,7 @@ func (report *Report) checkRanges() error {
|
|||||||
func (h *handler) finishWith(
|
func (h *handler) finishWith(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
held *decision,
|
held *decision,
|
||||||
report Report,
|
report wafprotocol.Report,
|
||||||
) {
|
) {
|
||||||
rq := &request{
|
rq := &request{
|
||||||
decision: *held,
|
decision: *held,
|
||||||
@@ -491,9 +316,7 @@ func (h *handler) finishWith(
|
|||||||
if rq.allowed {
|
if rq.allowed {
|
||||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||||
|
|
||||||
rq.runBanningHeuristics(ctx, onReport())
|
rq.banAndCount(ctx, rq.h.onReport)
|
||||||
rq.countAnomalies()
|
|
||||||
rq.addToHistory()
|
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(
|
rq.h.metrics.RequestReported(
|
||||||
&rq.line,
|
&rq.line,
|
||||||
|
|||||||
+195
-86
@@ -22,6 +22,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -106,8 +107,8 @@ func (s *WAF) questionWith(
|
|||||||
|
|
||||||
req := newRequest(s.t, method, s.addr, path, body)
|
req := newRequest(s.t, method, s.addr, path, body)
|
||||||
req.Host = host
|
req.Host = host
|
||||||
req.Header.Set(waf.ClientHeader, from)
|
req.Header.Set(wafprotocol.ClientHeader, from)
|
||||||
req.Header.Set(waf.PeerHeader, rproxyPeer)
|
req.Header.Set(wafprotocol.PeerHeader, rproxyPeer)
|
||||||
req.Header.Set(requestIDHeader, rand.Text())
|
req.Header.Set(requestIDHeader, rand.Text())
|
||||||
|
|
||||||
return req
|
return req
|
||||||
@@ -134,13 +135,18 @@ func (s *WAF) allowed(from string) string {
|
|||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
req := s.question(siteA, from, "/")
|
req := s.question(siteA, from, "/")
|
||||||
wantDisposition(s.t, do(s.t, req), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
s.t,
|
||||||
|
do(s.t, req),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
|
|
||||||
return req.Header.Get(requestIDHeader)
|
return req.Header.Get(requestIDHeader)
|
||||||
}
|
}
|
||||||
|
|
||||||
// report sends rproxy's report and returns the answer.
|
// report sends rproxy's report and returns the answer.
|
||||||
func (s *WAF) report(report waf.Report) answer {
|
func (s *WAF) report(report wafprotocol.Report) answer {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
body, err := json.Marshal(report)
|
body, err := json.Marshal(report)
|
||||||
@@ -155,8 +161,11 @@ func (s *WAF) report(report waf.Report) answer {
|
|||||||
func (s *WAF) reportBody(body string) answer {
|
func (s *WAF) reportBody(body string) answer {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
return do(s.t, newRequest(s.t, http.MethodPost, s.addr, waf.ReportPath,
|
return do(
|
||||||
strings.NewReader(body)))
|
s.t,
|
||||||
|
newRequest(s.t, http.MethodPost, s.addr, wafprotocol.ReportPath,
|
||||||
|
strings.NewReader(body)),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// passedOn asks about a request from the visitor at from on siteA, and reports
|
// passedOn asks about a request from the visitor at from on siteA, and reports
|
||||||
@@ -164,7 +173,7 @@ func (s *WAF) reportBody(body string) answer {
|
|||||||
func (s *WAF) passedOn(from string) {
|
func (s *WAF) passedOn(from string) {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: s.allowed(from),
|
RequestID: s.allowed(from),
|
||||||
Status: http.StatusOK,
|
Status: http.StatusOK,
|
||||||
RequestBytes: bodyBytes,
|
RequestBytes: bodyBytes,
|
||||||
@@ -210,7 +219,7 @@ func wantDisposition(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
wantStatus(t, got, status)
|
wantStatus(t, got, status)
|
||||||
|
|
||||||
if answered := got.header.Get(waf.DispositionHeader); answered !=
|
if answered := got.header.Get(wafprotocol.DispositionHeader); answered !=
|
||||||
disposition {
|
disposition {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"answered with disposition %q, want %q",
|
"answered with disposition %q, want %q",
|
||||||
@@ -272,10 +281,10 @@ func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) {
|
|||||||
req := s.question(siteA, client, "/page?q=1&token=visible")
|
req := s.question(siteA, client, "/page?q=1&token=visible")
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
req.Header.Set(requestIDHeader, "the-id")
|
||||||
req.Header.Set("X-Forwarded-Proto", "https")
|
req.Header.Set("X-Forwarded-Proto", "https")
|
||||||
req.Header.Set(waf.ContentLengthHeader, "1234")
|
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
|
||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
wantBody(t, got, "")
|
wantBody(t, got, "")
|
||||||
|
|
||||||
if id := got.header.Get(requestIDHeader); id != "" {
|
if id := got.header.Get(requestIDHeader); id != "" {
|
||||||
@@ -308,7 +317,12 @@ func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) {
|
|||||||
s.allowed("2001:db8::1")
|
s.allowed("2001:db8::1")
|
||||||
|
|
||||||
got := s.ask("2001:db8::2", "/")
|
got := s.ask("2001:db8::2", "/")
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
||||||
@@ -317,9 +331,14 @@ func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
|||||||
s := startWAF(t, "", map[string]string{denyNets: client})
|
s := startWAF(t, "", map[string]string{denyNets: client})
|
||||||
|
|
||||||
req := s.question(siteA, "::ffff:"+client, "/")
|
req := s.question(siteA, "::ffff:"+client, "/")
|
||||||
req.Header.Set(waf.PeerHeader, "::ffff:"+rproxyPeer)
|
req.Header.Set(wafprotocol.PeerHeader, "::ffff:"+rproxyPeer)
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.ClientIP != client || line.PeerIP != rproxyPeer {
|
if line.ClientIP != client || line.PeerIP != rproxyPeer {
|
||||||
@@ -339,13 +358,13 @@ func TestMalformedQuestionGets400AndAWarningAndNoLine(t *testing.T) {
|
|||||||
change func(*http.Request)
|
change func(*http.Request)
|
||||||
}{
|
}{
|
||||||
{"client not an address", func(req *http.Request) {
|
{"client not an address", func(req *http.Request) {
|
||||||
req.Header.Set(waf.ClientHeader, "not an address")
|
req.Header.Set(wafprotocol.ClientHeader, "not an address")
|
||||||
}},
|
}},
|
||||||
{"client empty", func(req *http.Request) {
|
{"client empty", func(req *http.Request) {
|
||||||
req.Header.Set(waf.ClientHeader, "")
|
req.Header.Set(wafprotocol.ClientHeader, "")
|
||||||
}},
|
}},
|
||||||
{"client address with a zone", func(req *http.Request) {
|
{"client address with a zone", func(req *http.Request) {
|
||||||
req.Header.Set(waf.ClientHeader, "2001:db8::1%eth0")
|
req.Header.Set(wafprotocol.ClientHeader, "2001:db8::1%eth0")
|
||||||
}},
|
}},
|
||||||
{"id absent", func(req *http.Request) {
|
{"id absent", func(req *http.Request) {
|
||||||
req.Header.Del(requestIDHeader)
|
req.Header.Del(requestIDHeader)
|
||||||
@@ -390,7 +409,12 @@ func TestLongestValidIDIsTaken(t *testing.T) {
|
|||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
|
req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
||||||
@@ -399,16 +423,18 @@ func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
|||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
over := strings.NewReader(
|
over := strings.NewReader(
|
||||||
strings.Repeat("x", waf.QuestionBodyMaxBytes+1),
|
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes+1),
|
||||||
)
|
)
|
||||||
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
|
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
|
||||||
wantDisposition(t, got, http.StatusBadRequest, "")
|
wantDisposition(t, got, http.StatusBadRequest, "")
|
||||||
wantProcessMessages(t, s, "a question is refused", 1)
|
wantProcessMessages(t, s, "a question is refused", 1)
|
||||||
wantNoRequestLines(t, s)
|
wantNoRequestLines(t, s)
|
||||||
|
|
||||||
whole := strings.NewReader(strings.Repeat("x", waf.QuestionBodyMaxBytes))
|
whole := strings.NewReader(
|
||||||
|
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes),
|
||||||
|
)
|
||||||
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
|
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
||||||
@@ -424,12 +450,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
|||||||
wantProcessMessages(t, s, "a question is refused", 1)
|
wantProcessMessages(t, s, "a question is refused", 1)
|
||||||
|
|
||||||
// A denied request's id is held as well.
|
// A denied request's id is held as well.
|
||||||
denied := s.question(siteA, otherClient, waf.HealthPath)
|
denied := s.question(siteA, otherClient, wafprotocol.HealthPath)
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
do(t, denied),
|
do(t, denied),
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
waf.DispositionDeny,
|
wafprotocol.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
again = s.question(siteA, client, "/")
|
again = s.question(siteA, client, "/")
|
||||||
@@ -437,7 +463,9 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
|||||||
wantDisposition(t, do(t, again), http.StatusBadRequest, "")
|
wantDisposition(t, do(t, again), http.StatusBadRequest, "")
|
||||||
|
|
||||||
// The report ends the holding, and the refused questions changed nothing.
|
// The report ends the holding, and the refused questions changed nothing.
|
||||||
got := s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
got := s.report(
|
||||||
|
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
||||||
|
)
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
if history := s.history(); history.ResponseBytes != answerBytes {
|
if history := s.history(); history.ResponseBytes != answerBytes {
|
||||||
@@ -446,7 +474,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
|||||||
|
|
||||||
again = s.question(siteA, client, "/")
|
again = s.question(siteA, client, "/")
|
||||||
again.Header.Set(requestIDHeader, id)
|
again.Header.Set(requestIDHeader, id)
|
||||||
wantDisposition(t, do(t, again), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, again),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||||
@@ -481,7 +514,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
|
|
||||||
// A report for a question not yet answered is ignored, and that question's
|
// A report for a question not yet answered is ignored, and that question's
|
||||||
// id is taken.
|
// id is taken.
|
||||||
got := s.report(waf.Report{RequestID: id, ResponseBytes: 1000})
|
got := s.report(wafprotocol.Report{RequestID: id, ResponseBytes: 1000})
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
// This would wait on the same lookup were the id not taken.
|
// This would wait on the same lookup were the id not taken.
|
||||||
@@ -496,7 +529,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
|
|
||||||
select {
|
select {
|
||||||
case got := <-answered:
|
case got := <-answered:
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
case <-time.After(waitLimit):
|
case <-time.After(waitLimit):
|
||||||
t.Fatal("the question got no answer")
|
t.Fatal("the question got no answer")
|
||||||
}
|
}
|
||||||
@@ -515,7 +548,9 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The true report comes after, and is the one that counts.
|
// The true report comes after, and is the one that counts.
|
||||||
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
got = s.report(
|
||||||
|
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
||||||
|
)
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
if history := s.history(); history.ResponseBytes != answerBytes {
|
if history := s.history(); history.ResponseBytes != answerBytes {
|
||||||
@@ -533,17 +568,17 @@ func TestPeerAbsentOrUnreadableIsAnsweredAndLoggedInvalid(t *testing.T) {
|
|||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Del(waf.PeerHeader)
|
req.Header.Del(wafprotocol.PeerHeader)
|
||||||
|
|
||||||
if peer != "" {
|
if peer != "" {
|
||||||
req.Header.Set(waf.PeerHeader, peer)
|
req.Header.Set(wafprotocol.PeerHeader, peer)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
do(t, req),
|
do(t, req),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
waf.DispositionAllow,
|
wafprotocol.DispositionAllow,
|
||||||
)
|
)
|
||||||
|
|
||||||
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
|
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
|
||||||
@@ -571,12 +606,12 @@ func TestContentLengthUnreadableOrNegativeCountsAsAbsent(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(waf.ContentLengthHeader, announced)
|
req.Header.Set(wafprotocol.ContentLengthHeader, announced)
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
do(t, req),
|
do(t, req),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
waf.DispositionAllow,
|
wafprotocol.DispositionAllow,
|
||||||
)
|
)
|
||||||
|
|
||||||
if line := s.out.requestLine(t); line.ContentLength != 0 {
|
if line := s.out.requestLine(t); line.ContentLength != 0 {
|
||||||
@@ -595,13 +630,18 @@ func TestHeaderSentTwiceUsesItsFirstValue(t *testing.T) {
|
|||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Add(waf.ClientHeader, otherClient)
|
req.Header.Add(wafprotocol.ClientHeader, otherClient)
|
||||||
req.Header.Add(waf.PeerHeader, "172.18.0.3")
|
req.Header.Add(wafprotocol.PeerHeader, "172.18.0.3")
|
||||||
req.Header.Set(requestIDHeader, "first")
|
req.Header.Set(requestIDHeader, "first")
|
||||||
req.Header.Add(requestIDHeader, "second")
|
req.Header.Add(requestIDHeader, "second")
|
||||||
req.Header.Set(waf.ContentLengthHeader, "1234")
|
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
|
||||||
req.Header.Add(waf.ContentLengthHeader, "5678")
|
req.Header.Add(wafprotocol.ContentLengthHeader, "5678")
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.ClientIP != client || line.PeerIP != rproxyPeer ||
|
if line.ClientIP != client || line.PeerIP != rproxyPeer ||
|
||||||
@@ -630,7 +670,7 @@ func sendQuestionWithPartOfItsBody(t *testing.T, conn net.Conn) {
|
|||||||
_, err := fmt.Fprintf(conn,
|
_, err := fmt.Fprintf(conn,
|
||||||
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
|
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
|
||||||
"Content-Length: 100\r\n\r\npart",
|
"Content-Length: 100\r\n\r\npart",
|
||||||
siteA, waf.ClientHeader, client, requestIDHeader)
|
siteA, wafprotocol.ClientHeader, client, requestIDHeader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send the question: %v", err)
|
t.Fatalf("send the question: %v", err)
|
||||||
}
|
}
|
||||||
@@ -699,7 +739,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
do(t, again),
|
do(t, again),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
waf.DispositionAllow,
|
wafprotocol.DispositionAllow,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -717,7 +757,7 @@ func TestQuestionDeniedGetsTheBanResponse(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(client, "/")
|
got := s.ask(client, "/")
|
||||||
wantDisposition(t, got, status, waf.DispositionDeny)
|
wantDisposition(t, got, status, wafprotocol.DispositionDeny)
|
||||||
wantBody(t, got, http.StatusText(status)+"\n")
|
wantBody(t, got, http.StatusText(status)+"\n")
|
||||||
wantLine(
|
wantLine(
|
||||||
t,
|
t,
|
||||||
@@ -737,7 +777,7 @@ func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(client, "/")
|
got := s.ask(client, "/")
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionClose)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionClose)
|
||||||
wantBody(t, got, "")
|
wantBody(t, got, "")
|
||||||
// The visitor is sent nothing.
|
// The visitor is sent nothing.
|
||||||
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
|
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
|
||||||
@@ -761,12 +801,22 @@ func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse(
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(client, "/blocked")
|
got := s.ask(client, "/blocked")
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
wantLine(t, s.out.requestLine(t), http.StatusForbidden,
|
wantLine(t, s.out.requestLine(t), http.StatusForbidden,
|
||||||
requestlog.ActionBlockedByBlockRule)
|
requestlog.ActionBlockedByBlockRule)
|
||||||
|
|
||||||
got = s.ask(client, "/.env")
|
got = s.ask(client, "/.env")
|
||||||
wantDisposition(t, got, http.StatusTooManyRequests, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusTooManyRequests,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
|
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
|
||||||
requestlog.ActionTemporarilyBanned)
|
requestlog.ActionTemporarilyBanned)
|
||||||
}
|
}
|
||||||
@@ -781,10 +831,15 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
|
|||||||
})
|
})
|
||||||
|
|
||||||
for _, path := range []string{
|
for _, path := range []string{
|
||||||
waf.HealthPath, waf.MetricsPath, waf.ReportPath,
|
wafprotocol.HealthPath, waf.MetricsPath, wafprotocol.ReportPath,
|
||||||
} {
|
} {
|
||||||
got := s.ask(client, path)
|
got := s.ask(client, path)
|
||||||
wantDisposition(t, got, http.StatusNotFound, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusNotFound,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantLine(
|
wantLine(
|
||||||
@@ -795,7 +850,12 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
|
|||||||
)
|
)
|
||||||
|
|
||||||
got := s.ask(otherClient, waf.MetricsPath)
|
got := s.ask(otherClient, waf.MetricsPath)
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
||||||
@@ -804,7 +864,7 @@ func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
|||||||
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
|
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
|
||||||
|
|
||||||
got := s.ask(client, "/")
|
got := s.ask(client, "/")
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||||
@@ -822,7 +882,7 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheRefusalLimit(
|
|||||||
|
|
||||||
for range 2 {
|
for range 2 {
|
||||||
got := s.ask(client, "/blocked")
|
got := s.ask(client, "/blocked")
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
}
|
}
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
@@ -846,7 +906,12 @@ func TestDeniedQuestionCountsForTheRefusalLimit(t *testing.T) {
|
|||||||
|
|
||||||
for range 2 {
|
for range 2 {
|
||||||
got := s.ask(client, "/blocked")
|
got := s.ask(client, "/blocked")
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
@@ -882,7 +947,12 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(fromDE, "/")
|
got := s.ask(fromDE, "/")
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
|
|
||||||
asn, country := got.header.Get(
|
asn, country := got.header.Get(
|
||||||
"X-Client-Asn",
|
"X-Client-Asn",
|
||||||
@@ -916,7 +986,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 100 bytes are over the limit of 99.
|
// 100 bytes are over the limit of 99.
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: id, Status: http.StatusBadGateway,
|
RequestID: id, Status: http.StatusBadGateway,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
})
|
})
|
||||||
@@ -940,7 +1010,12 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
got = s.ask(client, "/")
|
got = s.ask(client, "/")
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
||||||
@@ -958,12 +1033,12 @@ func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
s.ask(fromKP, "/"),
|
s.ask(fromKP, "/"),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
waf.DispositionAllow,
|
wafprotocol.DispositionAllow,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.passedOn(fromDE)
|
s.passedOn(fromDE)
|
||||||
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
|
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
|
||||||
waf.DispositionDeny)
|
wafprotocol.DispositionDeny)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
||||||
@@ -989,7 +1064,7 @@ func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: id, Status: http.StatusBadGateway,
|
RequestID: id, Status: http.StatusBadGateway,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
})
|
})
|
||||||
@@ -1021,7 +1096,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
do(t, denied),
|
do(t, denied),
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
waf.DispositionDeny,
|
wafprotocol.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
counters := s.server.Anomalies.Snapshot()
|
counters := s.server.Anomalies.Snapshot()
|
||||||
@@ -1039,7 +1114,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
|
|||||||
|
|
||||||
wantMetric(t, s.metrics(), series, 1)
|
wantMetric(t, s.metrics(), series, 1)
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: denied.Header.Get(
|
RequestID: denied.Header.Get(
|
||||||
requestIDHeader,
|
requestIDHeader,
|
||||||
), Status: http.StatusForbidden,
|
), Status: http.StatusForbidden,
|
||||||
@@ -1082,7 +1157,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
|
|||||||
1,
|
1,
|
||||||
) // the scrape itself
|
) // the scrape itself
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: id, Status: http.StatusBadGateway,
|
RequestID: id, Status: http.StatusBadGateway,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
|
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
|
||||||
@@ -1100,7 +1175,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
|
|||||||
// check needs a server that has not been scraped yet.
|
// check needs a server that has not been scraped yet.
|
||||||
unscraped := startWAF(t, "", map[string]string{metricsToken: token})
|
unscraped := startWAF(t, "", map[string]string{metricsToken: token})
|
||||||
|
|
||||||
got = unscraped.report(waf.Report{
|
got = unscraped.report(wafprotocol.Report{
|
||||||
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
|
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
|
||||||
DurationTotalMS: 5,
|
DurationTotalMS: 5,
|
||||||
})
|
})
|
||||||
@@ -1118,7 +1193,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
|
|||||||
|
|
||||||
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: "no-such-request", ResponseBytes: 1000,
|
RequestID: "no-such-request", ResponseBytes: 1000,
|
||||||
})
|
})
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
@@ -1137,7 +1212,9 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
|
|||||||
// A second report of the same request does not count it twice.
|
// A second report of the same request does not count it twice.
|
||||||
id := s.allowed(client)
|
id := s.allowed(client)
|
||||||
for range 2 {
|
for range 2 {
|
||||||
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
got = s.report(
|
||||||
|
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
||||||
|
)
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1347,7 +1424,7 @@ func TestBytesAtTheirMostEachWayBanRatherThanOverflow(t *testing.T) {
|
|||||||
|
|
||||||
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: s.allowed(client), Status: http.StatusOK,
|
RequestID: s.allowed(client), Status: http.StatusOK,
|
||||||
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
|
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
|
||||||
})
|
})
|
||||||
@@ -1441,9 +1518,14 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields(
|
|||||||
req.Header.Set("X-Forwarded-Proto", "https")
|
req.Header.Set("X-Forwarded-Proto", "https")
|
||||||
id := req.Header.Get(requestIDHeader)
|
id := req.Header.Get(requestIDHeader)
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
do(t, req),
|
||||||
|
http.StatusOK,
|
||||||
|
wafprotocol.DispositionAllow,
|
||||||
|
)
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: id, Status: http.StatusCreated,
|
RequestID: id, Status: http.StatusCreated,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
|
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
|
||||||
@@ -1490,7 +1572,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
|
|||||||
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
|
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
|
||||||
0, requestlog.ActionClientDenied,
|
0, requestlog.ActionClientDenied,
|
||||||
},
|
},
|
||||||
{"own prefix", nil, waf.HealthPath, http.StatusNotFound,
|
{"own prefix", nil, wafprotocol.HealthPath, http.StatusNotFound,
|
||||||
requestlog.ActionOwnEndpoint},
|
requestlog.ActionOwnEndpoint},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
@@ -1502,7 +1584,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
|
|||||||
id := req.Header.Get(requestIDHeader)
|
id := req.Header.Get(requestIDHeader)
|
||||||
do(t, req)
|
do(t, req)
|
||||||
|
|
||||||
got := s.report(waf.Report{
|
got := s.report(wafprotocol.Report{
|
||||||
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
|
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
|
||||||
})
|
})
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
@@ -1542,13 +1624,13 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
readAnswer(res),
|
readAnswer(res),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
waf.DispositionAllow,
|
wafprotocol.DispositionAllow,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The table is full again with next and those, without oldest.
|
// The table is full again with next and those, without oldest.
|
||||||
got := s.report(
|
got := s.report(
|
||||||
waf.Report{RequestID: oldest, ResponseBytes: answerBytes},
|
wafprotocol.Report{RequestID: oldest, ResponseBytes: answerBytes},
|
||||||
)
|
)
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
@@ -1559,7 +1641,9 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
got = s.report(waf.Report{RequestID: next, ResponseBytes: answerBytes})
|
got = s.report(
|
||||||
|
wafprotocol.Report{RequestID: next, ResponseBytes: answerBytes},
|
||||||
|
)
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
if counted := s.history().Requests; counted != 1 {
|
if counted := s.history().Requests; counted != 1 {
|
||||||
@@ -1580,10 +1664,20 @@ func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
got := do(t, s.question(siteA, client, "/.env"))
|
got := do(t, s.question(siteA, client, "/.env"))
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
|
|
||||||
got = do(t, s.question(siteB, client, "/"))
|
got = do(t, s.question(siteB, client, "/"))
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
if lines[0].Host != siteA || lines[1].Host != siteB {
|
if lines[0].Host != siteA || lines[1].Host != siteB {
|
||||||
@@ -1621,7 +1715,12 @@ func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) {
|
|||||||
s.allowed(client)
|
s.allowed(client)
|
||||||
|
|
||||||
got := do(t, s.question(siteB, client, "/"))
|
got := do(t, s.question(siteB, client, "/"))
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
||||||
@@ -1636,13 +1735,13 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
|||||||
path, authorization string
|
path, authorization string
|
||||||
want int
|
want int
|
||||||
}{
|
}{
|
||||||
{waf.HealthPath, "", http.StatusOK},
|
{wafprotocol.HealthPath, "", http.StatusOK},
|
||||||
{waf.MetricsPath, bearer, http.StatusOK},
|
{waf.MetricsPath, bearer, http.StatusOK},
|
||||||
{waf.MetricsPath, "", http.StatusUnauthorized},
|
{waf.MetricsPath, "", http.StatusUnauthorized},
|
||||||
{waf.BansPath, adminBearer, http.StatusOK},
|
{waf.BansPath, adminBearer, http.StatusOK},
|
||||||
{waf.BansPath, "", http.StatusUnauthorized},
|
{waf.BansPath, "", http.StatusUnauthorized},
|
||||||
{waf.ClientsPath + client, adminBearer, http.StatusOK},
|
{waf.ClientsPath + client, adminBearer, http.StatusOK},
|
||||||
{waf.ReportPath, "", http.StatusNotFound},
|
{wafprotocol.ReportPath, "", http.StatusNotFound},
|
||||||
{"/_sww/other", "", http.StatusNotFound},
|
{"/_sww/other", "", http.StatusNotFound},
|
||||||
{"/", "", http.StatusNotFound},
|
{"/", "", http.StatusNotFound},
|
||||||
} {
|
} {
|
||||||
@@ -1659,10 +1758,10 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
do(t, req),
|
do(t, req),
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
waf.DispositionDeny,
|
wafprotocol.DispositionDeny,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, do(t, req), http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1672,11 +1771,11 @@ func TestOwnEndpointsLinesDoNotSayWhichOfTheTwoTheyAre(t *testing.T) {
|
|||||||
|
|
||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
wantStatus(t, get(t, s.addr, waf.HealthPath), http.StatusOK)
|
wantStatus(t, get(t, s.addr, wafprotocol.HealthPath), http.StatusOK)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if _, there := line.fields["waf_line"]; there ||
|
if _, there := line.fields["waf_line"]; there ||
|
||||||
line.Path != waf.HealthPath {
|
line.Path != wafprotocol.HealthPath {
|
||||||
t.Errorf("the health check's log line is %+v", line.fields)
|
t.Errorf("the health check's log line is %+v", line.fields)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1688,7 +1787,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
|
|||||||
|
|
||||||
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
||||||
strings.NewReader("a body"))
|
strings.NewReader("a body"))
|
||||||
req.Header.Set(waf.ContentLengthHeader, "1073741824")
|
req.Header.Set(wafprotocol.ContentLengthHeader, "1073741824")
|
||||||
req.Header.Set(forwardedFor, otherClient)
|
req.Header.Set(forwardedFor, otherClient)
|
||||||
|
|
||||||
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
|
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
|
||||||
@@ -1698,7 +1797,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.ClientIP != client || line.ContentLength != 1<<30 {
|
if line.ClientIP != client || line.ContentLength != 1<<30 {
|
||||||
@@ -1776,7 +1875,7 @@ carried header:Content-Length block ^6$
|
|||||||
strings.NewReader("a body"))
|
strings.NewReader("a body"))
|
||||||
|
|
||||||
if tc.announced != "" {
|
if tc.announced != "" {
|
||||||
req.Header.Set(waf.ContentLengthHeader, tc.announced)
|
req.Header.Set(wafprotocol.ContentLengthHeader, tc.announced)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantStatus(t, do(t, req), tc.want)
|
wantStatus(t, do(t, req), tc.want)
|
||||||
@@ -1794,19 +1893,24 @@ func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) {
|
|||||||
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
||||||
strings.NewReader(body))
|
strings.NewReader(body))
|
||||||
req.Header.Set("Content-Type", contentType)
|
req.Header.Set("Content-Type", contentType)
|
||||||
req.Header.Set(waf.ContentLengthHeader, "100000")
|
req.Header.Set(wafprotocol.ContentLengthHeader, "100000")
|
||||||
|
|
||||||
return do(t, req)
|
return do(t, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := ask(formData, "id=1'%20OR%20'1'='1")
|
got := ask(formData, "id=1'%20OR%20'1'='1")
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
|
|
||||||
// The first 8 KiB of a longer JSON body cannot be parsed, which is no
|
// The first 8 KiB of a longer JSON body cannot be parsed, which is no
|
||||||
// attack, so the Core Rule Set does not read it.
|
// attack, so the Core Rule Set does not read it.
|
||||||
cut := `{"text":"` + strings.Repeat("x", 8<<10)
|
cut := `{"text":"` + strings.Repeat("x", 8<<10)
|
||||||
got = ask("application/json", cut)
|
got = ask("application/json", cut)
|
||||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
||||||
@@ -1815,7 +1919,12 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
|||||||
s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
|
s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
|
||||||
|
|
||||||
got := s.ask(client, sqlInjection)
|
got := s.ask(client, sqlInjection)
|
||||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
wantDisposition(
|
||||||
|
t,
|
||||||
|
got,
|
||||||
|
http.StatusForbidden,
|
||||||
|
wafprotocol.DispositionDeny,
|
||||||
|
)
|
||||||
|
|
||||||
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||||
if len(waiting) != 1 ||
|
if len(waiting) != 1 ||
|
||||||
@@ -1836,7 +1945,7 @@ func TestReputationHitAlertNamesTheSite(t *testing.T) {
|
|||||||
loadLists(t, s.server, map[string][]string{dropURL: {client}})
|
loadLists(t, s.server, map[string][]string{dropURL: {client}})
|
||||||
|
|
||||||
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
|
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
|
||||||
waf.DispositionDeny)
|
wafprotocol.DispositionDeny)
|
||||||
|
|
||||||
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||||
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
|
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package wafprotocol
|
||||||
|
|
||||||
|
import "net/http"
|
||||||
|
|
||||||
|
// Answer is what waf's answer to a question tells rproxy in its headers.
|
||||||
|
// The status, the other headers and the body of an answer of deny are what
|
||||||
|
// the visitor gets.
|
||||||
|
type Answer struct {
|
||||||
|
// Disposition is what rproxy does with the request: DispositionAllow,
|
||||||
|
// DispositionDeny or DispositionClose.
|
||||||
|
Disposition string
|
||||||
|
// ASN and Country are the client's AS number and country, each empty
|
||||||
|
// when the answer does not give it.
|
||||||
|
ASN, Country string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetAnswerHeaders sets the headers of waf's answer that tell rproxy answer,
|
||||||
|
// leaving out an ASN or Country that is empty.
|
||||||
|
func SetAnswerHeaders(header http.Header, answer Answer) {
|
||||||
|
header.Set(DispositionHeader, answer.Disposition)
|
||||||
|
|
||||||
|
if answer.ASN != "" {
|
||||||
|
header.Set(ASNHeader, answer.ASN)
|
||||||
|
}
|
||||||
|
|
||||||
|
if answer.Country != "" {
|
||||||
|
header.Set(CountryHeader, answer.Country)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAnswer returns what the headers of waf's answer tell rproxy, and
|
||||||
|
// whether they give a valid disposition.
|
||||||
|
func ReadAnswer(header http.Header) (Answer, bool) {
|
||||||
|
disposition := header.Get(DispositionHeader)
|
||||||
|
|
||||||
|
switch disposition {
|
||||||
|
case DispositionAllow, DispositionDeny, DispositionClose:
|
||||||
|
return Answer{
|
||||||
|
Disposition: disposition,
|
||||||
|
ASN: header.Get(ASNHeader),
|
||||||
|
Country: header.Get(CountryHeader),
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
return Answer{}, false
|
||||||
|
}
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package wafprotocol
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// maxRequestIDBytes is the longest id a question may give.
|
||||||
|
const maxRequestIDBytes = 128
|
||||||
|
|
||||||
|
// ErrBodyStoppedArriving is the error ReadQuestion wraps when the question's
|
||||||
|
// body stops arriving. Every other error it returns is why waf refuses the
|
||||||
|
// question.
|
||||||
|
var ErrBodyStoppedArriving = errors.New("the body stopped arriving")
|
||||||
|
|
||||||
|
// Why waf refuses a question. Each is also the body of the refusal.
|
||||||
|
var (
|
||||||
|
errClientNotAnAddress = errors.New(
|
||||||
|
ClientHeader + " is not an address without a zone",
|
||||||
|
)
|
||||||
|
errRequestIDNotValid = errors.New(RequestIDHeader + " is not a valid id")
|
||||||
|
errBodyTooLarge = errors.New("the body is over 64 KiB")
|
||||||
|
)
|
||||||
|
|
||||||
|
// QuestionParams are what NewQuestion needs.
|
||||||
|
type QuestionParams struct {
|
||||||
|
// Visitor is the visitor's request.
|
||||||
|
Visitor *http.Request
|
||||||
|
// PathAsSent is Visitor's path as the visitor sent it, which the question
|
||||||
|
// sends unchanged.
|
||||||
|
PathAsSent string
|
||||||
|
// Body is the part of Visitor's body the question carries, at most
|
||||||
|
// QuestionBodyMaxBytes.
|
||||||
|
Body []byte
|
||||||
|
// WAFBackend is waf's address. The question takes its scheme and host.
|
||||||
|
WAFBackend *url.URL
|
||||||
|
// Client is the visitor's address, and Peer that of rproxy's own TCP
|
||||||
|
// peer.
|
||||||
|
Client, Peer netip.Addr
|
||||||
|
// RequestID is the request's id.
|
||||||
|
RequestID string
|
||||||
|
// Scheme is how the visitor reached rproxy, or the trusted proxy in front
|
||||||
|
// of it.
|
||||||
|
Scheme string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Question is a question as waf reads it.
|
||||||
|
type Question struct {
|
||||||
|
// Client is the visitor's address.
|
||||||
|
Client netip.Addr
|
||||||
|
// Peer is the address of rproxy's own TCP peer. It is not valid when the
|
||||||
|
// question gives none that can be read.
|
||||||
|
Peer netip.Addr
|
||||||
|
// RequestID is the request's id.
|
||||||
|
RequestID string
|
||||||
|
// Visitor is the visitor's request, as asTheVisitorSentIt gives it.
|
||||||
|
Visitor *http.Request
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewQuestion returns the question about a visitor's request, as "The
|
||||||
|
// question" in SPEC.md gives it: the request as the visitor sent it, with the
|
||||||
|
// part of the body it carries and rproxy's headers.
|
||||||
|
func NewQuestion(
|
||||||
|
ctx context.Context,
|
||||||
|
params QuestionParams,
|
||||||
|
) (*http.Request, error) {
|
||||||
|
in := params.Visitor
|
||||||
|
|
||||||
|
question, err := http.NewRequestWithContext(
|
||||||
|
ctx,
|
||||||
|
in.Method,
|
||||||
|
params.WAFBackend.String(),
|
||||||
|
bytes.NewReader(params.Body),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("make the question: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
backend := question.URL
|
||||||
|
target := *in.URL
|
||||||
|
target.Scheme = backend.Scheme
|
||||||
|
target.Host = backend.Host
|
||||||
|
target.Opaque = params.PathAsSent
|
||||||
|
question.URL = &target
|
||||||
|
question.Host = in.Host
|
||||||
|
|
||||||
|
// The length NewRequestWithContext set is that of the body carried. This
|
||||||
|
// line makes a body whose length the visitor did not announce go chunked,
|
||||||
|
// as "The question" in SPEC.md requires.
|
||||||
|
question.TransferEncoding = in.TransferEncoding
|
||||||
|
|
||||||
|
question.Header = in.Header.Clone()
|
||||||
|
question.Header.Set(ClientHeader, params.Client.String())
|
||||||
|
question.Header.Set(PeerHeader, params.Peer.String())
|
||||||
|
question.Header.Del(ContentLengthHeader)
|
||||||
|
|
||||||
|
announced := in.Header.Get("Content-Length")
|
||||||
|
if announced != "" {
|
||||||
|
question.Header.Set(ContentLengthHeader, announced)
|
||||||
|
}
|
||||||
|
|
||||||
|
question.Header.Set(RequestIDHeader, params.RequestID)
|
||||||
|
question.Header.Set("X-Forwarded-Proto", params.Scheme)
|
||||||
|
|
||||||
|
// Go's client adds a User-Agent of its own to a request without the key.
|
||||||
|
_, sent := question.Header["User-Agent"]
|
||||||
|
if !sent {
|
||||||
|
question.Header["User-Agent"] = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return question, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadQuestion reads the question r, with the body it carries. An error
|
||||||
|
// wrapping ErrBodyStoppedArriving means the body stopped arriving. Any other
|
||||||
|
// error is why waf refuses the question, and the question's body is then not
|
||||||
|
// read. The body is read whole, since Go's server does not send an answer
|
||||||
|
// until it has read the body.
|
||||||
|
func ReadQuestion(w http.ResponseWriter, r *http.Request) (Question, error) {
|
||||||
|
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
||||||
|
if err != nil || client.Zone() != "" {
|
||||||
|
return Question{}, errClientNotAnAddress
|
||||||
|
}
|
||||||
|
|
||||||
|
id := r.Header.Get(RequestIDHeader)
|
||||||
|
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
||||||
|
return Question{}, errRequestIDNotValid
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(
|
||||||
|
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
|
||||||
|
)
|
||||||
|
|
||||||
|
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
||||||
|
return Question{}, errBodyTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return Question{}, fmt.Errorf("%w: %w", ErrBodyStoppedArriving, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One that cannot be read leaves the log line's peer_ip invalid.
|
||||||
|
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
||||||
|
|
||||||
|
return Question{
|
||||||
|
Client: client.Unmap(),
|
||||||
|
Peer: peer.Unmap(),
|
||||||
|
RequestID: id,
|
||||||
|
Visitor: asTheVisitorSentIt(r, body),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
||||||
|
// log line are to see it: without the headers rproxy adds for waf,
|
||||||
|
// with body, which has been read of r, and with the length the visitor
|
||||||
|
// announced, as -1 when it announced none. The question's own length covers
|
||||||
|
// only the part of the body it carries.
|
||||||
|
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
||||||
|
sent := r.Clone(r.Context())
|
||||||
|
announced := sent.Header.Get(ContentLengthHeader)
|
||||||
|
|
||||||
|
sent.Header.Del(ClientHeader)
|
||||||
|
sent.Header.Del(PeerHeader)
|
||||||
|
sent.Header.Del(ContentLengthHeader)
|
||||||
|
sent.Header.Del("Content-Length")
|
||||||
|
|
||||||
|
sent.ContentLength = -1
|
||||||
|
|
||||||
|
length, err := strconv.ParseInt(announced, 10, 64)
|
||||||
|
if err == nil && length >= 0 {
|
||||||
|
sent.ContentLength = length
|
||||||
|
sent.Header.Set("Content-Length", announced)
|
||||||
|
}
|
||||||
|
|
||||||
|
sent.Body = http.NoBody
|
||||||
|
if len(body) > 0 {
|
||||||
|
sent.Body = io.NopCloser(bytes.NewReader(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
return sent
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package wafprotocol
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
||||||
|
// far above any report's.
|
||||||
|
reportBodyMaxBytes = 1 << 20
|
||||||
|
// maxReportedBytes is the most bytes a report may give for each way, 1
|
||||||
|
// PiB, so that adding them up cannot overflow.
|
||||||
|
maxReportedBytes = 1 << 50
|
||||||
|
// maxReportedMilliseconds is the longest duration a report may give,
|
||||||
|
// about 31 years, so that it fits a time.Duration.
|
||||||
|
maxReportedMilliseconds = 1e12
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
errReportFieldMissing = errors.New("a field is missing")
|
||||||
|
errReportValue = errors.New("a value is out of range")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
||||||
|
// answer has ended, its fields named as the request log's of the same
|
||||||
|
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
||||||
|
// if nothing. The durations are in milliseconds.
|
||||||
|
//
|
||||||
|
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||||
|
type Report struct {
|
||||||
|
RequestID string `json:"request_id"`
|
||||||
|
Status int `json:"status"`
|
||||||
|
RequestBytes int64 `json:"request_bytes"`
|
||||||
|
ResponseBytes int64 `json:"response_bytes"`
|
||||||
|
DurationTotalMS float64 `json:"duration_total_ms"`
|
||||||
|
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewReportRequest returns the request that posts report to reportURL, which
|
||||||
|
// is waf's ReportPath.
|
||||||
|
func NewReportRequest(
|
||||||
|
ctx context.Context,
|
||||||
|
reportURL string,
|
||||||
|
report *Report,
|
||||||
|
) (*http.Request, error) {
|
||||||
|
body, err := json.Marshal(report)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("encode the report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(
|
||||||
|
ctx, http.MethodPost, reportURL, bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("make the request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadReport reads a report from r's body, which must be one JSON object with
|
||||||
|
// all of Report's fields, in the ranges SPEC.md gives.
|
||||||
|
func ReadReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
||||||
|
var report Report
|
||||||
|
|
||||||
|
body, err := io.ReadAll(
|
||||||
|
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return report, fmt.Errorf("read the body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
|
||||||
|
err = json.Unmarshal(body, &fields)
|
||||||
|
if err != nil {
|
||||||
|
return report, fmt.Errorf("decode the body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{
|
||||||
|
"request_id", "status", "request_bytes", "response_bytes",
|
||||||
|
"duration_total_ms", "duration_upstream_total_ms",
|
||||||
|
} {
|
||||||
|
value, there := fields[name]
|
||||||
|
if !there || string(value) == "null" {
|
||||||
|
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.Unmarshal(body, &report)
|
||||||
|
if err != nil {
|
||||||
|
return report, fmt.Errorf("decode the body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return report, report.checkRanges()
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkRanges returns an error for a value of the report that cannot be true.
|
||||||
|
func (report *Report) checkRanges() error {
|
||||||
|
switch {
|
||||||
|
case report.Status != 0 &&
|
||||||
|
(report.Status < http.StatusContinue || report.Status > ReportStatusMax):
|
||||||
|
return fmt.Errorf("%w: status", errReportValue)
|
||||||
|
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
||||||
|
return fmt.Errorf("%w: request_bytes", errReportValue)
|
||||||
|
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
||||||
|
return fmt.Errorf("%w: response_bytes", errReportValue)
|
||||||
|
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
|
||||||
|
return fmt.Errorf("%w: duration_total_ms", errReportValue)
|
||||||
|
case report.DurationUpstreamTotalMS < 0 ||
|
||||||
|
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
|
||||||
|
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// Package wafprotocol holds the three messages rproxy and waf send each
|
||||||
|
// other, as "`waf`" in SPEC.md gives them. rproxy asks waf a question about
|
||||||
|
// each visitor's request, and waf answers it. Once the visitor's answer has
|
||||||
|
// ended, rproxy sends waf a report. This package writes and reads each
|
||||||
|
// message, and declares every name and limit both sides must agree on.
|
||||||
|
package wafprotocol
|
||||||
|
|
||||||
|
// The headers of the question and of the answer. Go writes every header name
|
||||||
|
// in this form, as it sends it and as it receives it, so X-Client-ASN arrives
|
||||||
|
// as X-Client-Asn; header names are not case-sensitive.
|
||||||
|
const (
|
||||||
|
// ClientHeader marks a request as rproxy's question, and gives the
|
||||||
|
// visitor's address.
|
||||||
|
ClientHeader = "X-Sww-Client"
|
||||||
|
// PeerHeader gives the address of rproxy's own TCP peer.
|
||||||
|
PeerHeader = "X-Sww-Peer"
|
||||||
|
// ContentLengthHeader gives the body length the visitor announced.
|
||||||
|
ContentLengthHeader = "X-Sww-Content-Length"
|
||||||
|
// RequestIDHeader gives the request's id, which rproxy makes. rproxy
|
||||||
|
// also sets it on the request it passes to the app.
|
||||||
|
RequestIDHeader = "X-Request-ID"
|
||||||
|
// DispositionHeader gives waf's answer.
|
||||||
|
DispositionHeader = "X-Sww-Disposition"
|
||||||
|
// ASNHeader and CountryHeader give the client's AS number and country in
|
||||||
|
// an answer of allow. rproxy passes them to the app.
|
||||||
|
ASNHeader = "X-Client-Asn"
|
||||||
|
CountryHeader = "X-Client-Country"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The values of DispositionHeader.
|
||||||
|
const (
|
||||||
|
// DispositionAllow tells rproxy to pass the request to the app.
|
||||||
|
DispositionAllow = "allow"
|
||||||
|
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
||||||
|
// headers and body.
|
||||||
|
DispositionDeny = "deny"
|
||||||
|
// DispositionClose tells rproxy to close the visitor's connection
|
||||||
|
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
|
||||||
|
// waf closing the question's connection instead would read as
|
||||||
|
// waf being down, and rproxy would then let the request through.
|
||||||
|
DispositionClose = "close"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OwnEndpointPrefix starts the path of every request for sww itself: waf's
|
||||||
|
// own endpoints and rproxy's health endpoint. The app never sees a path under
|
||||||
|
// it.
|
||||||
|
const OwnEndpointPrefix = "/_sww/"
|
||||||
|
|
||||||
|
// HealthPath is the health endpoint of waf and of rproxy, which the
|
||||||
|
// container's health check asks.
|
||||||
|
const HealthPath = "/_sww/healthz"
|
||||||
|
|
||||||
|
// ReportPath is where rproxy sends the report of each request waf
|
||||||
|
// answered.
|
||||||
|
const ReportPath = "/_sww/report"
|
||||||
|
|
||||||
|
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
||||||
|
const QuestionBodyMaxBytes = 64 << 10
|
||||||
|
|
||||||
|
// ReportStatusMax is the highest status a report may give.
|
||||||
|
const ReportStatusMax = 599
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
package wafprotocol_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requestID is the id the tests' questions and reports give.
|
||||||
|
const requestID = "an-id"
|
||||||
|
|
||||||
|
// questionRead is what the server startQuestionReader starts read of a
|
||||||
|
// question: the question, and the body it carries.
|
||||||
|
type questionRead struct {
|
||||||
|
question wafprotocol.Question
|
||||||
|
body string
|
||||||
|
}
|
||||||
|
|
||||||
|
// startQuestionReader starts a server that reads each question it gets with
|
||||||
|
// ReadQuestion. It returns the server's URL and the channel what it reads goes
|
||||||
|
// to.
|
||||||
|
func startQuestionReader(t *testing.T) (*url.URL, <-chan questionRead) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
read := make(chan questionRead, 1)
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
question, err := wafprotocol.ReadQuestion(w, r)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("read the question: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var body []byte
|
||||||
|
if question.Visitor != nil {
|
||||||
|
body, _ = io.ReadAll(question.Visitor.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
read <- questionRead{question, string(body)}
|
||||||
|
},
|
||||||
|
))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
|
||||||
|
serverURL, err := url.Parse(server.URL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse the server's URL: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return serverURL, read
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestQuestionReadsAsWritten sends a question NewQuestion writes over HTTP,
|
||||||
|
// and checks that ReadQuestion gives back the visitor's request and what
|
||||||
|
// rproxy added.
|
||||||
|
func TestQuestionReadsAsWritten(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
target = "/a%2Fb?q=1"
|
||||||
|
body = "name=value"
|
||||||
|
held = "name"
|
||||||
|
)
|
||||||
|
|
||||||
|
client := netip.MustParseAddr("2001:db8::1")
|
||||||
|
peer := netip.MustParseAddr("192.0.2.1")
|
||||||
|
backend, read := startQuestionReader(t)
|
||||||
|
|
||||||
|
visitor := httptest.NewRequestWithContext(
|
||||||
|
t.Context(),
|
||||||
|
http.MethodPost,
|
||||||
|
"http://site.example.com"+target,
|
||||||
|
strings.NewReader(body),
|
||||||
|
)
|
||||||
|
visitor.Header.Set("Content-Length", "10")
|
||||||
|
visitor.Header.Set("User-Agent", "a visitor")
|
||||||
|
visitor.Header.Set(wafprotocol.PeerHeader, "the visitor's own")
|
||||||
|
|
||||||
|
question, err := wafprotocol.NewQuestion(
|
||||||
|
t.Context(),
|
||||||
|
wafprotocol.QuestionParams{
|
||||||
|
Visitor: visitor,
|
||||||
|
PathAsSent: "/a%2Fb",
|
||||||
|
Body: []byte(held),
|
||||||
|
WAFBackend: backend,
|
||||||
|
Client: client,
|
||||||
|
Peer: peer,
|
||||||
|
RequestID: requestID,
|
||||||
|
Scheme: "https",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write the question: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := http.DefaultTransport.RoundTrip(question)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("send the question: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = res.Body.Close()
|
||||||
|
|
||||||
|
got := <-read
|
||||||
|
if got.question.Visitor == nil {
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
|
||||||
|
if got.question.Client != client || got.question.Peer != peer ||
|
||||||
|
got.question.RequestID != requestID {
|
||||||
|
t.Errorf("read %+v", got.question)
|
||||||
|
}
|
||||||
|
|
||||||
|
sent := got.question.Visitor
|
||||||
|
if sent.Host != "site.example.com" || sent.RequestURI != target ||
|
||||||
|
sent.ContentLength != int64(len(body)) || got.body != held {
|
||||||
|
t.Errorf(
|
||||||
|
"read %s at %s, of length %d, carrying %q",
|
||||||
|
sent.RequestURI, sent.Host, sent.ContentLength, got.body,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, want := range map[string]string{
|
||||||
|
"Content-Length": "10",
|
||||||
|
"User-Agent": "a visitor",
|
||||||
|
"X-Forwarded-Proto": "https",
|
||||||
|
wafprotocol.RequestIDHeader: requestID,
|
||||||
|
wafprotocol.ClientHeader: "",
|
||||||
|
wafprotocol.PeerHeader: "",
|
||||||
|
wafprotocol.ContentLengthHeader: "",
|
||||||
|
} {
|
||||||
|
if sent.Header.Get(name) != want {
|
||||||
|
t.Errorf("%s is %q, want %q", name, sent.Header.Get(name), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAnswerReadsAsWritten checks that ReadAnswer gives back what
|
||||||
|
// SetAnswerHeaders sets, and that headers without a valid disposition give
|
||||||
|
// nothing.
|
||||||
|
func TestAnswerReadsAsWritten(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, want := range []wafprotocol.Answer{
|
||||||
|
{Disposition: wafprotocol.DispositionAllow, ASN: "64496", Country: "DE"},
|
||||||
|
{Disposition: wafprotocol.DispositionAllow, Country: "DE"},
|
||||||
|
{Disposition: wafprotocol.DispositionDeny},
|
||||||
|
{Disposition: wafprotocol.DispositionClose},
|
||||||
|
} {
|
||||||
|
header := http.Header{}
|
||||||
|
wafprotocol.SetAnswerHeaders(header, want)
|
||||||
|
|
||||||
|
got, valid := wafprotocol.ReadAnswer(header)
|
||||||
|
if !valid || got != want {
|
||||||
|
t.Errorf("read %+v (valid: %t), want %+v", got, valid, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
header := http.Header{}
|
||||||
|
header.Set(wafprotocol.DispositionHeader, "maybe")
|
||||||
|
header.Set(wafprotocol.ASNHeader, "64496")
|
||||||
|
|
||||||
|
got, valid := wafprotocol.ReadAnswer(header)
|
||||||
|
if valid || got != (wafprotocol.Answer{}) {
|
||||||
|
t.Errorf("read %+v (valid: %t), want nothing", got, valid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReportReadsAsWritten checks that ReadReport gives back the report
|
||||||
|
// NewReportRequest posts.
|
||||||
|
func TestReportReadsAsWritten(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
want := wafprotocol.Report{
|
||||||
|
RequestID: requestID,
|
||||||
|
Status: http.StatusOK,
|
||||||
|
RequestBytes: 10,
|
||||||
|
ResponseBytes: 20,
|
||||||
|
DurationTotalMS: 1.5,
|
||||||
|
DurationUpstreamTotalMS: 0.5,
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := wafprotocol.NewReportRequest(
|
||||||
|
t.Context(),
|
||||||
|
"http://waf.example.com"+wafprotocol.ReportPath,
|
||||||
|
&want,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write the report: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.Method != http.MethodPost ||
|
||||||
|
req.URL.Path != wafprotocol.ReportPath {
|
||||||
|
t.Errorf("the report is %s %s", req.Method, req.URL.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := wafprotocol.ReadReport(httptest.NewRecorder(), req)
|
||||||
|
if err != nil || got != want {
|
||||||
|
t.Errorf("read %+v (%v), want %+v", got, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user