Compare commits

..
2 Commits
Author SHA1 Message Date
sneak 9ff9f23a8f sww: keep the stop cut-off test within funlen's 80 lines
check / check (push) Waiting to run
On next, TestRProxyStoppingCutsOffARequestAfterFiveSeconds is at the
limit, and this PR's reformatting of the stand-in's answer adds a line,
so the linter fails on the merged tree. The test now uses the file's
rproxyUpstreamURL instead of declaring the same setting name again.

Model: opus-5-5
2026-11-19 19:17:50 +00:00
sneak 73e0137c07 Move what rproxy and waf send each other into wafprotocol (closes #425)
check / check (push) Waiting to run
internal/wafprotocol writes and reads the question, the answer and the
report, and declares once each header, path and limit both daemons use.
rproxy no longer imports internal/waf. Nothing on the wire changes.

Model: opus-5-5
2026-11-19 18:54:41 +00:00
51 changed files with 1560 additions and 1252 deletions

No files matched your search

+2
View File
@@ -902,6 +902,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
- `internal/rproxy`: the reverse proxy in front of one app. - `internal/rproxy`: the reverse proxy in front of one app.
- `internal/waf`: `waf`'s server, which runs the checks and answers its own - `internal/waf`: `waf`'s server, which runs the checks and answers its own
endpoints. endpoints.
- `internal/wafprotocol`: the question, the answer and the report that `rproxy`
and `waf` send each other.
- `internal/ratelimit`, `internal/bans`, `internal/rules`, - `internal/ratelimit`, `internal/bans`, `internal/rules`,
`internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/reputation`, `internal/routewatch`, `internal/anomaly` and
+7 -4
View File
@@ -73,13 +73,16 @@ never as a time.
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts `go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
and stops their background work. and stops their background work.
- `internal/config`: reads the settings, the one place they are read. - `internal/config`: reads the settings, the one place they are read.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds - `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
`waf`'s question, answer and report. It answers `waf`'s own endpoints in `waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
`answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes and anomalies, and writes the log lines.
the log lines.
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf` - `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
about each request. It passes a request to the app with the standard about each request. It passes a request to the app with the standard
library's `httputil.ReverseProxy`, within the time and size limits. library's `httputil.ReverseProxy`, within the time and size limits.
- `internal/wafprotocol`: the question, the answer and the report that
`rproxy` and `waf` send each other (see "`waf`"). It writes and reads each
one, and declares every header, path and limit both sides use.
`internal/rproxy` imports it, not `internal/waf`.
- `internal/ratelimit`: the table of clients. It counts requests, bytes and - `internal/ratelimit`: the table of clients. It counts requests, bytes and
refusals, and keeps each client's history. refusals, and keeps each client's history.
- `internal/bans`: the bans. It holds each netblock's bans with their notes, - `internal/bans`: the bans. It holds each netblock's bans with their notes,
+1 -1
View File
@@ -13,7 +13,6 @@ require (
github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_golang v1.24.1
github.com/spf13/viper v1.21.0 github.com/spf13/viper v1.21.0
go.uber.org/fx v1.24.0 go.uber.org/fx v1.24.0
golang.org/x/net v0.60.0
golang.org/x/sys v0.48.0 golang.org/x/sys v0.48.0
) )
@@ -51,6 +50,7 @@ require (
go.uber.org/zap v1.26.0 // indirect go.uber.org/zap v1.26.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect golang.org/x/sync v0.23.0 // indirect
golang.org/x/text v0.42.0 // indirect golang.org/x/text v0.42.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect google.golang.org/protobuf v1.36.11 // indirect
+4 -8
View File
@@ -54,11 +54,7 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
now := midnight() now := midnight()
scores := map[netip.Prefix]int64{netblock: 100} scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.Ban( limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 { if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made) t.Errorf("made %d bans over a ban for a limit, want none", made)
@@ -123,7 +119,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
expires := now.Add(30 * day) expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100}) ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(limited, now, bans.Notes{})
// listed is off the blacklist when it is fetched after the ban's end. // listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 { if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
@@ -148,7 +144,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100 scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
} }
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(limited, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 { if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made) t.Errorf("made %d bans for 3 netblocks, want 3", made)
@@ -158,7 +154,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
// The ban for an attack takes the place of the one for a limit, and no // The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes. // blacklist ban goes.
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack}) ledger.BanForAttack(attacker, now, bans.Notes{})
want := map[netip.Prefix]int{limited: 0, attacker: 1} want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores { for netblock := range scores {
+23 -64
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2}) wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one. // A new ban drops the ban sww made, and only that one.
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(second, midnight(), bans.Notes{})
wantBans( wantBans(
t, t,
ledger, ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
} }
// And the next drops that one. // And the next drops that one.
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(third, midnight(), bans.Notes{})
wantBans( wantBans(
t, t,
ledger, ledger,
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended. // twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute), ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins") "probes for logins")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(
netblock, netblock,
midnight().Add(time.Minute), midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
) )
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first // A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes. // ban for a limit, its earliest that is not an admin's, goes.
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(other, second.Start, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1}) wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it. // The netblock's next ban still counts it.
third, _ := ledger.Ban( third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) { if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+ t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans) "want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first // The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin // ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban. // then keeps the first ban.
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock, second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
kept := first kept := first
kept.Cause = bans.CauseAdmin kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second}) ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's. // The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.Ban( third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) { if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+ t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans) "want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
limit, _ := ledger.Ban( limit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"), netip.MustParsePrefix("203.0.113.1/32"),
midnight(), midnight(),
bans.Grounds{ bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
) )
byteLimit, _ := ledger.Ban( byteLimit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.3/32"), netip.MustParsePrefix("203.0.113.3/32"),
midnight(), midnight(),
bans.Grounds{ bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
) )
attack, _ := ledger.Ban( attack, _ := ledger.BanForAttack(
netip.MustParsePrefix("203.0.113.2/32"), netip.MustParsePrefix("203.0.113.2/32"),
midnight(), midnight(),
bans.Grounds{ bans.Notes{RuleID: "git-dir", Target: "path"},
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
) )
for _, tc := range []struct{ got, want string }{ for _, tc := range []struct{ got, want string }{
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses") t.Error("the lifted ban refuses")
} }
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour || if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf( t.Errorf(
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
} }
// The next clear sign of attack bans for seven days, as a first does. // The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.Ban( ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
if ban.Expires.Sub(ban.Start) != 7*day { if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf( t.Errorf(
"the next ban for an attack ends at %s, want seven days on", "the next ban for an attack ends at %s, want seven days on",
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel() t.Parallel()
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
made, _ := ledger.Ban( made, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"), netip.MustParsePrefix("203.0.113.1/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
atStart := bans.Ban{ atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"), Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
// An hour's ban for a broken limit. // An hour's ban for a broken limit.
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an // A minute later an admin bans the netblock for good, named by an
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first. // The temporary ban is made first, so that its netblock is looked at first.
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
ledger.BanForAdmin( ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"), netip.MustParsePrefix("203.0.113.0/24"),
+82 -54
View File
@@ -300,50 +300,81 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest return newest
} }
// Grounds are why a request makes a ban: its cause, CauseLimit or // BanForLimit bans netblock at now for a broken limit, with notes, and
// CauseAttack, and its notes. // returns the ban, and true. A first ban lasts LimitBanDuration. A ban
type Grounds struct { // made within LimitBanRepeatWindow after the netblock's ban that ended
Cause string // last, other than one for a clear sign of attack or a lifted one, lasts
Notes Notes // repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
} }
// Ban bans netblock at now on grounds, and returns the ban, and true. If // WouldBanForLimit returns what BanForLimit would, without making the ban:
// a ban on netblock is still active, as when two of its requests break a // what observe mode would have done.
// limit at once, that ban is returned with false, and no other is made. func (l *Ledger) WouldBanForLimit(
// The ledger fills in the notes' Refused and EarlierBans itself. netblock netip.Prefix, now time.Time, notes Notes,
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
) (Ban, bool) { ) (Ban, bool) {
return l.ban(netblock, now, grounds, true) return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
} }
// WouldBan returns what Ban would, without making the ban: what observe // BanForAttack bans netblock at now for a clear sign of attack, with
// mode would have done. // notes, and returns the ban, and whether it made it, as BanForLimit
func (l *Ledger) WouldBan( // does. A first ban lasts AttackBanDuration; once the netblock has had
netblock netip.Prefix, now time.Time, grounds Grounds, // one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) { ) (Ban, bool) {
return l.ban(netblock, now, grounds, false) return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
} }
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit // WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as Ban would make it. It // or CauseAttack, made at now would be permanent, as BanForLimit or
// works out nothing else of the ban. // BanForAttack would make it. It works out nothing else of the ban.
func (l *Ledger) WouldBePermanent( func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause string, netblock netip.Prefix, now time.Time, cause string,
) bool { ) bool {
@@ -380,9 +411,10 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until // BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose // expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another // cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// on netblock is active, since the admin asked for this one. The ledger // ban even while another on netblock is active, since the admin asked
// fills in the notes' EarlierBans, and counts the ban among those made. // for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
func (l *Ledger) BanForAdmin( func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string, netblock netip.Prefix, now, expires time.Time, reason string,
) Ban { ) Ban {
@@ -727,13 +759,15 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
return forgot return forgot
} }
// ban bans netblock at now on grounds, as Ban describes, and returns the // ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// ban, and whether it made it. Unless keep is true, the ban is not made, // reason and notes, as BanForLimit and BanForAttack describe, and returns
// only returned: it is the ban that would have been made. // the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
func (l *Ledger) ban( func (l *Ledger) ban(
netblock netip.Prefix, netblock netip.Prefix,
now time.Time, now time.Time,
grounds Grounds, cause, reason string,
notes Notes,
keep bool, keep bool,
) (Ban, bool) { ) (Ban, bool) {
l.mu.Lock() l.mu.Lock()
@@ -742,8 +776,6 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active. // held are the netblock's bans, none of them active.
var held []Ban var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock) bans, found := l.netblocks.Get(netblock)
if found { if found {
active := activeBan(*bans, now) active := activeBan(*bans, now)
@@ -759,19 +791,15 @@ func (l *Ledger) ban(
ban := Ban{ ban := Ban{
Netblock: netblock, Netblock: netblock,
Start: now, Start: now,
Cause: grounds.Cause, Cause: cause,
Reason: reason,
Notes: notes, Notes: notes,
} }
switch grounds.Cause { if cause == CauseAttack {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
ban.Expires = l.attackExpiry(held, now) ban.Expires = l.attackExpiry(held, now)
default: } else {
panic("bans: a request's ban has the cause " + grounds.Cause) ban.Expires = l.limitExpiry(held, now)
} }
if !keep { if !keep {
@@ -779,7 +807,7 @@ func (l *Ledger) ban(
} }
l.add(ban) l.add(ban)
l.made[grounds.Cause]++ l.made[cause]++
l.markChanged() l.markChanged()
return ban, true return ban, true
+51 -121
View File
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and // Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours. // 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} { for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
length := time.Duration(hours) * time.Hour length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) || if !ban.Expires.Equal(now.Add(length)) ||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is // The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends. // permanent, and never ends.
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() { if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires) t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
} }
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
second, _ := ledger.Ban( second, _ := ledger.BanForLimit(
netblock, netblock,
first.Expires.Add(tc.gap), first.Expires.Add(tc.gap),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if second.Expires.Sub(second.Start) != tc.want || if second.Expires.Sub(second.Start) != tc.want ||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params) ledger := bans.New(params)
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if !ban.Permanent() { if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires) t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight() now := midnight()
for i := range 14 { for i := range 14 {
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
if !ban.Expires.After(ban.Start) { if !ban.Expires.After(ban.Start) {
t.Fatalf( t.Fatalf(
"ban %d starts at %s and ends at %s", "ban %d starts at %s and ends at %s",
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires now = ban.Expires
} }
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() { if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires) t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
} }
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.Ban( first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
if !made { if !made {
t.Error("the first ban was not made") t.Error("the first ban was not made")
} }
again, made := ledger.Ban( again, made := ledger.BanForLimit(
netblock, netblock,
midnight().Add(time.Minute), midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if made || !reflect.DeepEqual(again, first) || if made || !reflect.DeepEqual(again, first) ||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
) )
} }
again, made = ledger.Ban( again, made = ledger.BanForAttack(
netblock, netblock,
midnight().Add(time.Minute), midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseAttack}, bans.Notes{},
) )
if made || !reflect.DeepEqual(again, first) { if made || !reflect.DeepEqual(again, first) {
t.Errorf( t.Errorf(
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}}, bans.Notes{Requests: 5},
) )
for range 3 { for range 3 {
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}}, bans.Notes{Requests: 5},
) )
got, banned, _ := ledger.Find( got, banned, _ := ledger.Find(
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64") d := netip.MustParsePrefix("2001:db8::/64")
now := midnight() now := midnight()
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit}) first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(b, now, bans.Notes{})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(c, now, bans.Notes{})
// A request from a makes b the netblock seen longest ago, and its ban // A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's. // goes to make room for d's.
ledger.Check(a.Addr(), now) ledger.Check(a.Addr(), now)
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(d, now, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1}) wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes. // a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(a, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1}) wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes // With d seen since, a is seen longest ago, and its earlier ban goes
// first. // first.
ledger.Check(d.Addr(), first.Expires) ledger.Check(d.Addr(), first.Expires)
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(b, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1}) wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) { if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params) ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock, second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
held := ledger.Bans(netblock) held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) || if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"} notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.Ban( ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
if !ban.Expires.Equal(midnight().Add(7*day)) || if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack || ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" || ban.Notes.RuleID != "env-file" ||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count. // A ban for a broken limit before does not count.
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock, second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
if second.Expires.Sub(second.Start) != 7*day { if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days", t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent, // Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit. // beside the one for a limit.
third, _ := ledger.Ban( third, _ := ledger.BanForAttack(
netblock, netblock,
second.Expires.Add(30*day), second.Expires.Add(30*day),
bans.Grounds{Cause: bans.CauseAttack}, bans.Notes{},
) )
if !third.Permanent() || if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) { third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams()) ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
// While the first ban lasts, none would be made. // While the first ban lasts, none would be made.
during, would := ledger.WouldBan( during, would := ledger.WouldBanForAttack(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseAttack}, bans.Notes{},
) )
if would || !reflect.DeepEqual(during, first) { if would || !reflect.DeepEqual(during, first) {
t.Errorf( t.Errorf(
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a // As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made. // limit broken again for three hours, but neither is made.
limitGrounds := bans.Grounds{ limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
Cause: bans.CauseLimit, attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}, bans.Notes{RuleID: "git-dir"})
} limit, wouldLimit := ledger.WouldBanForLimit(
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
netblock, netblock,
first.Expires, first.Expires,
limitGrounds, limitNotes,
) )
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) || if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
// The ban made is the one that would have been. // The ban made is the one that would have been.
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds) made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
if !reflect.DeepEqual(made, limit) { if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit) t.Errorf("the ban made is %+v, want %+v", made, limit)
} }
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1) t.Fatalf("ban %d for a limit would be permanent", i+1)
} }
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
now = ban.Expires now = ban.Expires
} }
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
} }
// Once a first ban for an attack has ended, the next would be permanent. // Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.Ban( attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) { if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent") t.Error("a second ban for an attack would not be permanent")
} }
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the // Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does. // ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.Ban( attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
netblock, limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if limit.Expires.Sub(limit.Start) != time.Hour || if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit { limit.Cause != bans.CauseLimit {
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long, Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
} }
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
midnight(), midnight(),
bans.Grounds{ bans.Notes{Request: request},
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
) )
cut := long[:256] cut := long[:256]
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after // A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock. // the first ended, and one on another netblock.
ledger.Ban(netblock, midnight(), bans.Grounds{ ledger.BanForLimit(netblock, midnight(), bans.Notes{
Cause: bans.CauseLimit, ASN: "AS64497", ASName: "Other Net", Country: "FR",
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
}) })
ledger.Ban( ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
netblock, ledger.BanForLimit(other, midnight(), bans.Notes{})
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE") ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+21 -36
View File
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit}) ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes // A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write. // only the counts in the notes, which wait for the interval's write.
ledger.Ban( ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Check(netblock.Addr(), midnight().Add(time.Minute)) ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
// Two bans before the value is read leave one. // Two bans before the value is read leave one.
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"), netip.MustParsePrefix("203.0.113.10/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.11/32"), netip.MustParsePrefix("203.0.113.11/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
wantChanged(t, ledger, true) wantChanged(t, ledger, true)
wantChanged(t, ledger, false) wantChanged(t, ledger, false)
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32") high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32") low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
v6, ledger.BanForLimit(high, midnight(), bans.Notes{})
midnight(), ledger.BanForLimit(low, midnight(), bans.Notes{})
bans.Grounds{Cause: bans.CauseLimit}, ledger.BanForLimit(v6, first.Expires, bans.Notes{})
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
snapshot := ledger.Snapshot() snapshot := ledger.Snapshot()
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams()) before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32") netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{ ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
// Loaded into a new ledger, as across a restart, the ban still refuses // Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three // while it lasts, and once it has ended a broken limit bans for three
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse") t.Error("the loaded ban does not refuse")
} }
again, _ := after.Ban( again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
if again.Expires.Sub(again.Start) != 3*time.Hour || if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf( t.Errorf(
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(), Start: midnight(),
}, },
}) })
ledger.Ban( ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"), netip.MustParsePrefix("198.51.100.7/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
for client, want := range map[string]bool{ for client, want := range map[string]bool{
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
} }
// A limit broken now makes no shorter ban over the permanent one. // A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit}) ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 { if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf( t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+ "a broken limit returned %+v and left the netblock %d bans, "+
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans // Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans // for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's. // before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(
netblock, netblock,
nineHours.Expires.Add(time.Hour), nineHours.Expires.Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
if ban.Expires.Sub(ban.Start) != 27*time.Hour || if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) { ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
} }
// The ledger holds one ban, so it makes two more without dropping any. // The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.Ban( first, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"), netip.MustParsePrefix("198.51.100.7/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
second, _ := ledger.Ban( second, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.8/32"), netip.MustParsePrefix("198.51.100.8/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
want := []bans.Ban{first, second, kept} want := []bans.Ban{first, second, kept}
+25 -4
View File
@@ -19,8 +19,6 @@ import (
"time" "time"
"github.com/spf13/viper" "github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// off is the value that switches a timeout, a size limit or a rate limit // off is the value that switches a timeout, a size limit or a rate limit
@@ -42,7 +40,9 @@ var (
errNotPositive = errors.New("must be more than zero, or off") errNotPositive = errors.New("must be more than zero, or off")
errNotMilliseconds = errors.New( errNotMilliseconds = errors.New(
"is not a whole number of milliseconds above zero, such as 2000") "is not a whole number of milliseconds above zero, such as 2000")
errEmptyItem = errors.New("has an empty item in its list") errEmptyItem = errors.New("has an empty item in its list")
errNotNetblock = errors.New(
"is not a netblock such as 10.0.0.0/8, or an address")
errNotListenAddr = errors.New( errNotListenAddr = errors.New(
"is not an address to listen on, such as :8080") "is not an address to listen on, such as :8080")
errNotCommand = errors.New("is not rproxy or waf") errNotCommand = errors.New("is not rproxy or waf")
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
netblocks := make([]netip.Prefix, 0, len(items)) netblocks := make([]netip.Prefix, 0, len(items))
for _, item := range items { for _, item := range items {
netblock, err := reputation.ParseNetblock(item) netblock, err := ParseNetblock(item)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -264,6 +264,27 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
return netblocks, nil return netblocks, nil
} }
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netblock.Masked(), nil
}
addr, err := netip.ParseAddr(value)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// parseListenAddr checks an address to listen on: an optional host and a // parseListenAddr checks an address to listen on: an optional host and a
// port number. // port number.
func parseListenAddr(value string) (string, error) { func parseListenAddr(value string) (string, error) {
+104 -47
View File
@@ -20,13 +20,11 @@ import (
"unicode/utf8" "unicode/utf8"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// The names of the settings that waf writes into log lines and ban notes, // The names of the settings that waf writes into log lines and ban notes,
@@ -115,7 +113,7 @@ type WAF struct {
// with a tag that percentage of every rate limit and byte limit, from 1 // with a tag that percentage of every rate limit and byte limit, from 1
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in // to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
// lower case, a country code in capitals. // lower case, a country code in capitals.
TagLists []reputation.TagList TagLists []TagList
TagDeny []string TagDeny []string
TagLimitPercent map[string]int64 TagLimitPercent map[string]int64
// RouteWatchURL is where the routewatch instance is that adds the asn= // RouteWatchURL is where the routewatch instance is that adds the asn=
@@ -512,8 +510,12 @@ var (
"is not a whole number above zero, such as 60, or off", "is not a whole number above zero, such as 60, or off",
) )
errNotUTF8 = errors.New("is not valid UTF-8") errNotUTF8 = errors.New("is not valid UTF-8")
errNotASN = errors.New("is not an AS number such as AS64496")
errNotPercentItem = errors.New( errNotPercentItem = errors.New(
"is not a code, : and a percentage, such as AS64496:50 or cn:25") "is not a code, : and a percentage, such as AS64496:50 or cn:25")
errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100",
)
errNotAbuseIPDBThreshold = errors.New( errNotAbuseIPDBThreshold = errors.New(
"is not a whole number from 25 to 100, the scores AbuseIPDB's " + "is not a whole number from 25 to 100, the scores AbuseIPDB's " +
"blacklist can be cut at", "blacklist can be cut at",
@@ -636,7 +638,7 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"), RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
ASNBytesLimitPercent: env.percents( ASNBytesLimitPercent: env.percents(
ASNBytesLimitPercentSetting, ASNBytesLimitPercentSetting,
reputation.ParseASN, ParseASN,
), ),
CountryBytesLimitPercent: env.percents( CountryBytesLimitPercent: env.percents(
CountryBytesLimitPercentSetting, CountryBytesLimitPercentSetting,
@@ -1022,7 +1024,7 @@ func (e *environment) percents(
} }
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default. // tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
func (e *environment) tagLists(name string) []reputation.TagList { func (e *environment) tagLists(name string) []TagList {
lists, err := parseTagLists(e.value(name, "")) lists, err := parseTagLists(e.value(name, ""))
e.check(name, err) e.check(name, err)
@@ -1048,7 +1050,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
// percent reads a setting that is a percentage, from 0 to 100. // percent reads a setting that is a percentage, from 0 to 100.
func (e *environment) percent(name, defaultValue string) int64 { func (e *environment) percent(name, defaultValue string) int64 {
percent, err := reputation.ParsePercent(e.value(name, defaultValue)) percent, err := ParsePercent(e.value(name, defaultValue))
e.check(name, err) e.check(name, err)
return percent return percent
@@ -1133,7 +1135,7 @@ func (e *environment) action(
string(value), string(ReputationActionLimit)+":", string(value), string(ReputationActionLimit)+":",
) )
percent, err := reputation.ParsePercent(percentText) percent, err := ParsePercent(percentText)
if !isLimit || err != nil { if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction)) e.check(name, fmt.Errorf("%q %w", value, errNotAction))
} }
@@ -1142,8 +1144,8 @@ func (e *environment) action(
} }
// zones reads the setting that is the list of DNSBL zones. It is empty by // zones reads the setting that is the list of DNSBL zones. It is empty by
// default. The log shows each zone with its key masked, as // default. The log shows each zone with its key masked, as MaskZoneKey
// reputation.MaskZoneKey masks it. // masks it.
func (e *environment) zones(name string) []string { func (e *environment) zones(name string) []string {
value, _ := e.lookup(name) value, _ := e.lookup(name)
zones, err := parseZones(value) zones, err := parseZones(value)
@@ -1151,7 +1153,7 @@ func (e *environment) zones(name string) []string {
logged := make([]string, len(zones)) logged := make([]string, len(zones))
for i, zone := range zones { for i, zone := range zones {
logged[i] = reputation.MaskZoneKey(zone) logged[i] = MaskZoneKey(zone)
} }
e.settings = append( e.settings = append(
@@ -1312,11 +1314,7 @@ func (e *environment) checkTags(cfg *WAF) {
// checkTag returns an error when tag would never apply, as checkTags says. // checkTag returns an error when tag would never apply, as checkTags says.
func checkTag(cfg *WAF, tag string) error { func checkTag(cfg *WAF, tag string) error {
if !requestCanGetTag(cfg, tag) { if !requestCanGetTag(cfg, tag) {
return fmt.Errorf( return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
"%q %w",
reputation.MaskZoneKey(tag),
errNoRequestGetsTag,
)
} }
return nil return nil
@@ -1324,7 +1322,7 @@ func checkTag(cfg *WAF, tag string) error {
// requestCanGetTag reports whether anything cfg sets up can give a request tag. // requestCanGetTag reports whether anything cfg sets up can give a request tag.
func requestCanGetTag(cfg *WAF, tag string) bool { func requestCanGetTag(cfg *WAF, tag string) bool {
isListName := func(list reputation.TagList) bool { return list.Name == tag } isListName := func(list TagList) bool { return list.Name == tag }
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) { if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
return true return true
} }
@@ -1348,7 +1346,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
return isZoneTag && return isZoneTag &&
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool { slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
return strings.ToLower(reputation.MaskZoneKey(each)) == zone return strings.ToLower(MaskZoneKey(each)) == zone
}) })
} }
@@ -1846,9 +1844,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock) return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
} }
netblock, err := reputation.ParseNetblock( netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
strings.TrimSpace(netblockText),
)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -2054,6 +2050,21 @@ func parseCountry(value string) (string, error) {
return country, nil return country, nil
} }
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
if !hasAS || err != nil {
return "", fmt.Errorf("%q %w", value, errNotASN)
}
return "AS" + strconv.FormatUint(number, 10), nil
}
// The tags a request can get that the settings can name, as waf // The tags a request can get that the settings can name, as waf
// writes them. The prefixes are followed by a value. // writes them. The prefixes are followed by a value.
const ( const (
@@ -2067,17 +2078,24 @@ const (
ExploitSignatureTag = "exploitsignature" ExploitSignatureTag = "exploitsignature"
) )
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
// requests of the clients it holds, and the URL it is fetched from.
type TagList struct {
Name string
URL string
}
// parseTagLists reads a comma-separated list of name=URL pairs, such as // parseTagLists reads a comma-separated list of name=URL pairs, such as
// torexit=https://example.com/tor.txt, and returns each list with its name // torexit=https://example.com/tor.txt, and returns each list with its name
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the // in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
// = before http. A name or a URL listed twice is an error. // = before http. A name or a URL listed twice is an error.
func parseTagLists(value string) ([]reputation.TagList, error) { func parseTagLists(value string) ([]TagList, error) {
items, err := parseList(value) items, err := parseList(value)
if err != nil { if err != nil {
return nil, err return nil, err
} }
lists := make([]reputation.TagList, 0, len(items)) lists := make([]TagList, 0, len(items))
for _, item := range items { for _, item := range items {
end := strings.Index(item, "=http") end := strings.Index(item, "=http")
@@ -2085,12 +2103,9 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
return nil, fmt.Errorf("%q %w", item, errNotTagList) return nil, fmt.Errorf("%q %w", item, errNotTagList)
} }
list := reputation.TagList{ list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
Name: strings.ToLower(item[:end]),
URL: item[end+1:],
}
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool { if slices.ContainsFunc(lists, func(earlier TagList) bool {
return earlier.Name == list.Name || earlier.URL == list.URL return earlier.Name == list.Name || earlier.URL == list.URL
}) { }) {
return nil, fmt.Errorf("%q %w", item, errListedTwice) return nil, fmt.Errorf("%q %w", item, errListedTwice)
@@ -2103,8 +2118,7 @@ func parseTagLists(value string) ([]reputation.TagList, error) {
} }
// parseTags reads a comma-separated list of tags, which parseTag reads. A // parseTags reads a comma-separated list of tags, which parseTag reads. A
// tag listed twice is an error, which shows the tag as // tag listed twice is an error, which shows the tag as MaskZoneKey does.
// reputation.MaskZoneKey does.
func parseTags(value string) ([]string, error) { func parseTags(value string) ([]string, error) {
items, err := parseList(value) items, err := parseList(value)
if err != nil { if err != nil {
@@ -2124,7 +2138,7 @@ func parseTags(value string) ([]string, error) {
if slices.Contains(tags, tag) { if slices.Contains(tags, tag) {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"%q %w", "%q %w",
reputation.MaskZoneKey(item), MaskZoneKey(item),
errListedTwice, errListedTwice,
) )
} }
@@ -2193,7 +2207,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if colon <= 0 { if colon <= 0 {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"%q %w", "%q %w",
reputation.MaskZoneKey(item), MaskZoneKey(item),
errNotTagPercentItem, errNotTagPercentItem,
) )
} }
@@ -2207,7 +2221,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if err != nil || percent < 1 || percent > 99 { if err != nil || percent < 1 || percent > 99 {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"%q %w", "%q %w",
reputation.MaskZoneKey(item[colon+1:]), MaskZoneKey(item[colon+1:]),
errNotTagPercent, errNotTagPercent,
) )
} }
@@ -2219,11 +2233,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
if _, listed := percents[tag]; listed { if _, listed := percents[tag]; listed {
// tag, since item[:colon] may end in a space, which MaskZoneKey // tag, since item[:colon] may end in a space, which MaskZoneKey
// would not mask. // would not mask.
return nil, fmt.Errorf( return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
"%q %w",
reputation.MaskZoneKey(tag),
errListedTwice,
)
} }
percents[tag] = percent percents[tag] = percent
@@ -2257,7 +2267,7 @@ func parsePercents(
return nil, err return nil, err
} }
percent, err := reputation.ParsePercent(percentText) percent, err := ParsePercent(percentText)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -2272,6 +2282,38 @@ func parsePercents(
return percents, nil return percents, nil
} }
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
}
return percent, nil
}
// headerNameChars are the characters RFC 9110 allows in a header name:
// letters, digits and these marks.
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
"0123456789!#$%&'*+-.^_`|~"
// IsHeaderName reports whether name can be a header name: one or more of
// the characters RFC 9110 allows in one.
func IsHeaderName(name string) bool {
if name == "" {
return false
}
for _, char := range name {
if !strings.ContainsRune(headerNameChars, char) {
return false
}
}
return true
}
// parseHeaderNames reads a comma-separated list of header names in either // parseHeaderNames reads a comma-separated list of header names in either
// case, and returns them in lower case. Host and Transfer-Encoding are // case, and returns them in lower case. Host and Transfer-Encoding are
// refused: Go's HTTP server takes them out of the request's headers. // refused: Go's HTTP server takes them out of the request's headers.
@@ -2284,7 +2326,7 @@ func parseHeaderNames(value string) ([]string, error) {
headers := make([]string, 0, len(items)) headers := make([]string, 0, len(items))
for _, item := range items { for _, item := range items {
if !httpguts.ValidHeaderFieldName(item) { if !IsHeaderName(item) {
return nil, fmt.Errorf("%q %w", item, errNotHeaderName) return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
} }
@@ -2440,8 +2482,8 @@ const (
// other name for one that does not exist, so that the zone would list no // other name for one that does not exist, so that the zone would list no
// client. A zone listed twice is an error, whatever the case of its // client. A zone listed twice is an error, whatever the case of its
// letters, which DNS names ignore, and whatever its key, since // letters, which DNS names ignore, and whatever its key, since
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a // MaskZoneKey shows two keys of one zone alike. An error shows a zone as
// zone as it does. // MaskZoneKey does.
func parseZones(value string) ([]string, error) { func parseZones(value string) ([]string, error) {
zones, err := parseList(value) zones, err := parseList(value)
if err != nil { if err != nil {
@@ -2451,14 +2493,11 @@ func parseZones(value string) ([]string, error) {
} }
for i, zone := range zones { for i, zone := range zones {
shown := reputation.MaskZoneKey(zone) shown := MaskZoneKey(zone)
listedBefore := slices.ContainsFunc( listedBefore := slices.ContainsFunc(
zones[:i], zones[:i],
func(earlier string) bool { func(earlier string) bool {
return strings.EqualFold( return strings.EqualFold(MaskZoneKey(earlier), shown)
reputation.MaskZoneKey(earlier),
shown,
)
}, },
) )
@@ -2475,6 +2514,24 @@ func parseZones(value string) ([]string, error) {
return zones, nil return zones, nil
} }
// MaskZoneKey returns zone with ******** in place of its key, if it is a
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
// other zone it returns as it is. A zone is shown so wherever it leaves
// the process: in the log, the alerts and the metrics.
func MaskZoneKey(zone string) string {
// DNS names ignore case, and a name may be written with a dot at its
// end.
name := strings.TrimSuffix(strings.ToLower(zone), ".")
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
return zone
}
_, rest, _ := strings.Cut(zone, ".")
return masked + "." + rest
}
// isZone reports whether each label of zone is as parseZones takes it. It // isZone reports whether each label of zone is as parseZones takes it. It
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too. // also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
func isZone(zone string) bool { func isZone(zone string) bool {
@@ -2530,7 +2587,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
name, headerValue, found := strings.Cut(item, ":") name, headerValue, found := strings.Cut(item, ":")
name = strings.TrimSpace(name) name = strings.TrimSpace(name)
if !found || !httpguts.ValidHeaderFieldName(name) || if !found || !IsHeaderName(name) ||
strings.ContainsAny(headerValue, "\r\n\x00") { strings.ContainsAny(headerValue, "\r\n\x00") {
return nil, "", fmt.Errorf( return nil, "", fmt.Errorf(
"item %d %w", "item %d %w",
+20 -29
View File
@@ -18,12 +18,10 @@ import (
"time" "time"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/reputation"
) )
// The settings, by name. // The settings, by name.
@@ -866,32 +864,6 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
} }
} }
// config and rules check header names with httpguts.ValidHeaderFieldName.
// It takes a name of one or more of the characters RFC 9110 allows in one,
// as the check it replaced did: letters, digits and these marks.
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
t.Parallel()
const marks = "!#$%&'*+-.^_`|~"
if httpguts.ValidHeaderFieldName("") {
t.Error("the empty name is taken")
}
for b := range 256 {
char := byte(b)
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
// for a byte above 0x7f.
name := string([]byte{char})
if got := httpguts.ValidHeaderFieldName(name); got != want {
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
}
}
}
func TestRemoteLogSettingsDefaults(t *testing.T) { func TestRemoteLogSettingsDefaults(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1722,7 +1694,7 @@ func TestTagSettingsAsSet(t *testing.T) {
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99", tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
}) })
wantLists := []reputation.TagList{ wantLists := []config.TagList{
{Name: torExitTag, URL: torURL}, {Name: torExitTag, URL: torURL},
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"}, {Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
} }
@@ -2316,6 +2288,25 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
} }
} }
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
t *testing.T,
) {
t.Parallel()
for zone, want := range map[string]string{
spamhaus: spamhausMasked,
spamhaus + ".": spamhausMasked + ".",
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
dronebl: dronebl,
"dq.spamhaus.net": "dq.spamhaus.net",
spamhaus + ".example": spamhaus + ".example",
} {
if got := config.MaskZoneKey(zone); got != want {
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
}
}
}
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) { func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
t.Parallel() t.Parallel()
+3 -4
View File
@@ -14,6 +14,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -300,7 +301,7 @@ const sourceLabel = "source"
// AddReputation adds the metrics of the lists fetched from URLs and of the // AddReputation adds the metrics of the lists fetched from URLs and of the
// DNSBL zones, by source, each list's URL or each zone, its key masked as // DNSBL zones, by source, each list's URL or each zone, its key masked as
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a // config.MaskZoneKey masks it: the requests whose client a blocklist, a
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, // zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
// and, read from lists and dnsbl as the metrics are asked for, for a list, // and, read from lists and dnsbl as the metrics are asked for, for a list,
// the fetches that failed and when the copy in use was fetched, and for a // the fetches that failed and when the copy in use was fetched, and for a
@@ -319,9 +320,7 @@ func (m *Metrics) AddReputation(
m.registry.MustRegister(m.reputationHits) m.registry.MustRegister(m.reputationHits)
for _, zone := range dnsbl.Zones() { for _, zone := range dnsbl.Zones() {
source := prometheus.Labels{ source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
sourceLabel: reputation.MaskZoneKey(zone),
}
m.addReputationQueries( m.addReputationQueries(
source, source,
+4 -23
View File
@@ -16,6 +16,7 @@ import (
"github.com/hashicorp/golang-lru/v2/simplelru" "github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
const ( const (
@@ -31,8 +32,6 @@ const (
// it fails, and no client is checked with AbuseIPDB after a check // it fails, and no client is checked with AbuseIPDB after a check
// fails, so that a source refusing them is not asked on every request. // fails, so that a source refusing them is not asked on every request.
failureDelay = time.Minute failureDelay = time.Minute
// maskedKey is what MaskZoneKey shows in place of a zone's key.
maskedKey = "********"
) )
var ( var (
@@ -132,24 +131,6 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
} }
} }
// MaskZoneKey returns zone with ******** in place of its key, if it is a
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
// other zone it returns as it is. A zone is shown so wherever it leaves
// the process: in the log, the alerts and the metrics.
func MaskZoneKey(zone string) string {
// DNS names ignore case, and a name may be written with a dot at its
// end.
name := strings.TrimSuffix(strings.ToLower(zone), ".")
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
return zone
}
_, rest, _ := strings.Cut(zone, ".")
return maskedKey + "." + rest
}
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them. // Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
func (d *DNSBL) Zones() []string { func (d *DNSBL) Zones() []string {
return slices.Clone(d.params.Zones) return slices.Clone(d.params.Zones)
@@ -157,7 +138,7 @@ func (d *DNSBL) Zones() []string {
// ListedBy returns the zones whose verdict on addr, a client's address, // ListedBy returns the zones whose verdict on addr, a client's address,
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key // lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
// masked, as MaskZoneKey masks it, since they go to the request // masked, as config.MaskZoneKey masks it, since they go to the request
// log, the alerts and the metrics. A verdict is used until CacheTTL has // log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr // passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the // in the background, unless a query about addr to it is under way, the
@@ -182,7 +163,7 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
switch { switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL: case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed { if kept.Listed {
listedBy = append(listedBy, MaskZoneKey(zone)) listedBy = append(listedBy, config.MaskZoneKey(zone))
} }
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) && case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
len(d.asking) < maxQueries: len(d.asking) < maxQueries:
@@ -311,7 +292,7 @@ func (d *DNSBL) ask(ctx context.Context, q query) {
if isFailure { if isFailure {
const failed = "asking a DNSBL zone failed" const failed = "asking a DNSBL zone failed"
shown := MaskZoneKey(q.zone) shown := config.MaskZoneKey(q.zone)
// Raised before it is logged, so that the alert is there once the // Raised before it is logged, so that the alert is there once the
// log line is. // log line is.
-25
View File
@@ -860,28 +860,3 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
t.Errorf("asked about %v, want %v", got, want) t.Errorf("asked about %v, want %v", got, want)
} }
} }
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
t *testing.T,
) {
t.Parallel()
const (
dronebl = "dnsbl.dronebl.org"
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
spamhausMasked = "********.xbl.dq.spamhaus.net"
)
for zone, want := range map[string]string{
spamhaus: spamhausMasked,
spamhaus + ".": spamhausMasked + ".",
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
dronebl: dronebl,
"dq.spamhaus.net": "dq.spamhaus.net",
spamhaus + ".example": spamhaus + ".example",
} {
if got := reputation.MaskZoneKey(zone); got != want {
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
}
}
}
+11 -77
View File
@@ -7,10 +7,8 @@
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks // SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent // clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to files and reads them // today. The state package writes all of these to files and reads them
// from there, so that a restart keeps them too. It fetches AbuseIPDB's // from there, so that a restart keeps them too. Last, it fetches
// blacklist, which it does not keep. Last, it reads the netblocks, AS // AbuseIPDB's blacklist, which it does not keep.
// numbers and percentages that both the settings and the lists hold, and
// masks the keys of DNSBL zones.
package reputation package reputation
import ( import (
@@ -22,12 +20,12 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"slices" "slices"
"strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
const ( const (
@@ -42,28 +40,15 @@ const (
) )
var ( var (
errStatus = errors.New("the server answered") errStatus = errors.New("the server answered")
errTooLong = errors.New("the list is longer than 16 MiB") errTooLong = errors.New("the list is longer than 16 MiB")
errNotNetblockLine = errors.New( errNotNetblock = errors.New(
"is not an address or a netblock, such as 192.0.2.0/24", "is not an address or a netblock, such as 192.0.2.0/24",
) )
errNotASNPercent = errors.New( errNotASNPercent = errors.New(
"is not an AS number, : and a percentage, such as AS64496:50") "is not an AS number, : and a percentage, such as AS64496:50")
errNotNetblock = errors.New(
"is not a netblock such as 10.0.0.0/8, or an address")
errNotASN = errors.New("is not an AS number such as AS64496")
errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100",
)
) )
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
// requests of the clients it holds, and the URL it is fetched from.
type TagList struct {
Name string
URL string
}
// List is a list as lists.json holds it: the URL it is fetched from, // List is a list as lists.json holds it: the URL it is fetched from,
// when it was last tried, the fetch failed or not, and its last good copy: // when it was last tried, the fetch failed or not, and its last good copy:
// when that was fetched, and its lines, as fetched, comment lines // when that was fetched, and its lines, as fetched, comment lines
@@ -83,7 +68,7 @@ type Params struct {
BlocklistURLs []string BlocklistURLs []string
ASNLimitPercentURL string ASNLimitPercentURL string
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS). // TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
TagLists []TagList TagLists []config.TagList
// Refresh is how long after a list was last fetched or tried it is // Refresh is how long after a list was last fetched or tried it is
// fetched again (SWW_WAF_BLOCKLIST_REFRESH). // fetched again (SWW_WAF_BLOCKLIST_REFRESH).
Refresh time.Duration Refresh time.Duration
@@ -480,11 +465,7 @@ func parseNetblocks(lines []string) (entries, error) {
netblock, ok := parseNetblock(text) netblock, ok := parseNetblock(text)
if !ok { if !ok {
return entries{}, fmt.Errorf( return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
"line %d %w",
i+1,
errNotNetblockLine,
)
} }
found.netblocks[netblock] = true found.netblocks[netblock] = true
@@ -504,7 +485,7 @@ func parseNetblocks(lines []string) (entries, error) {
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which // stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
// stands for none, is refused. // stands for none, is refused.
func parseNetblock(text string) (netip.Prefix, bool) { func parseNetblock(text string) (netip.Prefix, bool) {
netblock, err := ParseNetblock(text) netblock, err := config.ParseNetblock(text)
if err != nil { if err != nil {
return netip.Prefix{}, false return netip.Prefix{}, false
} }
@@ -539,9 +520,9 @@ func parsePercents(lines []string) (entries, error) {
} }
asnText, percentText, _ := strings.Cut(text, ":") asnText, percentText, _ := strings.Cut(text, ":")
asn, asnErr := ParseASN(asnText) asn, asnErr := config.ParseASN(asnText)
percent, percentErr := ParsePercent(percentText) percent, percentErr := config.ParsePercent(percentText)
if asnErr != nil || percentErr != nil { if asnErr != nil || percentErr != nil {
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent) return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
} }
@@ -555,53 +536,6 @@ func parsePercents(lines []string) (entries, error) {
return found, nil return found, nil
} }
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
// address is a netblock of that address alone, a /32 or a /128. A
// blocklist's lines are read with it too.
func ParseNetblock(value string) (netip.Prefix, error) {
if strings.Contains(value, "/") {
netblock, err := netip.ParsePrefix(value)
if err != nil {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netblock.Masked(), nil
}
addr, err := netip.ParseAddr(value)
if err != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
}
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
// ParseASN reads an AS number such as AS64496, in either case, and
// returns it as the lookup gives it: AS and the number, in capitals and
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
// read with it too.
func ParseASN(value string) (string, error) {
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
number, err := strconv.ParseUint(digits, 10, 32)
if !hasAS || err != nil {
return "", fmt.Errorf("%q %w", value, errNotASN)
}
return "AS" + strconv.FormatUint(number, 10), nil
}
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
func ParsePercent(value string) (int64, error) {
percent, err := strconv.ParseInt(value, 10, 64)
if err != nil || percent < 0 || percent > 100 {
return 0, fmt.Errorf("%q %w", value, errNotPercent)
}
return percent, nil
}
// withoutComment returns line without anything after a ; or a #, and // withoutComment returns line without anything after a ; or a #, and
// without the spaces around what is left. // without the spaces around what is left.
func withoutComment(line string) string { func withoutComment(line string) string {
-10
View File
@@ -7,16 +7,6 @@ import (
"strings" "strings"
) )
// The headers rproxy sets on the requests it passes to the app. Go writes every
// header name in this form, as it sends it and as it receives it, so
// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever
// their case; header names are not case-sensitive.
const (
requestIDHeader = "X-Request-ID"
asnHeader = "X-Client-Asn"
countryHeader = "X-Client-Country"
)
// peerAddress is the address of the request's TCP peer, normally traefik, without // peerAddress is the address of the request's TCP peer, normally traefik, without
// any IPv6 zone, which waf refuses. // any IPv6 zone, which waf refuses.
func peerAddress(r *http.Request) netip.Addr { func peerAddress(r *http.Request) netip.Addr {
-6
View File
@@ -2,12 +2,6 @@ package rproxy
import "time" import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time. // Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct { type Breaker struct {
b breaker b breaker
+7 -16
View File
@@ -20,7 +20,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// post is the start of a POST whose headers end with what follows it. // post is the start of a POST whose headers end with what follows it.
@@ -492,7 +492,7 @@ func TestASettingOfOffSetsNoLimit(t *testing.T) {
strings.NewReader("a body"))), strings.NewReader("a body"))),
) )
if got := get(t, addr, waf.HealthPath); got.status != http.StatusOK { if got := get(t, addr, wafprotocol.HealthPath); got.status != http.StatusOK {
t.Errorf("answered %+v, want rproxy's own 200", got) t.Errorf("answered %+v, want rproxy's own 200", got)
} }
}) })
@@ -870,14 +870,10 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) { func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
t.Parallel() t.Parallel()
// rproxy is waiting for the rest of the body well within this limit, even on
// a busy host, and answers when it runs out, well before dial's deadline.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, greet) app := startApp(t, greet)
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{ addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10), maxRequestTime: "3000",
}) })
conn := dial(t, addr) conn := dial(t, addr)
@@ -1266,9 +1262,7 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
) { ) {
t.Parallel() t.Parallel()
// The app's answer arrives well within this limit, even on a busy host, and const limit = 2 * time.Second
// the connection closes when it runs out, well before dial's deadline.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, answerBeforeTheBody) app := startApp(t, answerBeforeTheBody)
@@ -1433,20 +1427,17 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
const answer = "the end of a long answer" const answer = "the end of a long answer"
// The request reaches the app well within this limit, even on a busy host.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, func(w http.ResponseWriter, r *http.Request) { app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body) _, _ = io.Copy(io.Discard, r.Body)
_ = http.NewResponseController(w).Flush() _ = http.NewResponseController(w).Flush()
time.Sleep(limit + 100*time.Millisecond) time.Sleep(2100 * time.Millisecond)
_, _ = io.WriteString(w, answer) _, _ = io.WriteString(w, answer)
}) })
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{ addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10), maxRequestTime: "2000",
}) })
got := visit( got := visit(
@@ -1480,7 +1471,7 @@ func TestAVisitorThatStopsALittleShortGetsAnAnswerRProxyGivesAtOnce(
{"a deny", post, http.StatusForbidden}, {"a deny", post, http.StatusForbidden},
{ {
"the health check", "the health check",
"GET " + waf.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n", "GET " + wafprotocol.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
http.StatusOK, http.StatusOK,
}, },
} { } {
+13 -13
View File
@@ -12,11 +12,11 @@ import (
"strings" "strings"
"testing" "testing"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// questionBodyMaxBytes is the most of a body a question carries. // questionBodyMaxBytes is the most of a body a question carries.
const questionBodyMaxBytes = waf.QuestionBodyMaxBytes const questionBodyMaxBytes = wafprotocol.QuestionBodyMaxBytes
// chunked is the encoding of a body whose length the visitor did not announce. // chunked is the encoding of a body whose length the visitor did not announce.
const chunked = "chunked" const chunked = "chunked"
@@ -42,9 +42,9 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
req.Header.Set(forwardedForHeader, "198.51.100.1") req.Header.Set(forwardedForHeader, "198.51.100.1")
req.Header.Set(forwardedProtoHeader, "gopher") req.Header.Set(forwardedProtoHeader, "gopher")
req.Header.Set(requestIDHeader, "the-visitor's") req.Header.Set(requestIDHeader, "the-visitor's")
req.Header.Set(waf.ClientHeader, "198.51.100.2") req.Header.Set(wafprotocol.ClientHeader, "198.51.100.2")
req.Header.Set(waf.PeerHeader, "198.51.100.3") req.Header.Set(wafprotocol.PeerHeader, "198.51.100.3")
req.Header.Set(waf.ContentLengthHeader, "12345") req.Header.Set(wafprotocol.ContentLengthHeader, "12345")
// Go's client then sends no User-Agent, so that one in the question would be // Go's client then sends no User-Agent, so that one in the question would be
// Go's own. // Go's own.
req.Header["User-Agent"] = nil req.Header["User-Agent"] = nil
@@ -73,8 +73,8 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
want.Set(forwardedForHeader, "198.51.100.1") want.Set(forwardedForHeader, "198.51.100.1")
want.Set(forwardedProtoHeader, "http") want.Set(forwardedProtoHeader, "http")
want.Set(requestIDHeader, asked.header.Get(requestIDHeader)) want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
want.Set(waf.ClientHeader, localhost) want.Set(wafprotocol.ClientHeader, localhost)
want.Set(waf.PeerHeader, localhost) want.Set(wafprotocol.PeerHeader, localhost)
if !maps.EqualFunc(asked.header, want, slices.Equal) { if !maps.EqualFunc(asked.header, want, slices.Equal) {
t.Errorf( t.Errorf(
@@ -124,8 +124,8 @@ func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) {
wantGreeting(t, visit(t, req)) wantGreeting(t, visit(t, req))
asked := standIn.question(t) asked := standIn.question(t)
if asked.header.Get(waf.ClientHeader) != tc.client || if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
asked.header.Get(waf.PeerHeader) != localhost || asked.header.Get(wafprotocol.PeerHeader) != localhost ||
asked.header.Get(forwardedForHeader) != visitor || asked.header.Get(forwardedForHeader) != visitor ||
asked.header.Get(forwardedProtoHeader) != tc.proto { asked.header.Get(forwardedProtoHeader) != tc.proto {
t.Errorf("the question's headers are %v", asked.header) t.Errorf("the question's headers are %v", asked.header)
@@ -191,7 +191,7 @@ func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) {
req.Header[forwardedForHeader] = tc.header req.Header[forwardedForHeader] = tc.header
wantGreeting(t, visit(t, req)) wantGreeting(t, visit(t, req))
if got := standIn.question(t).header.Get(waf.ClientHeader); got != if got := standIn.question(t).header.Get(wafprotocol.ClientHeader); got !=
tc.client { tc.client {
t.Errorf("the question names %s, want %s", got, tc.client) t.Errorf("the question names %s, want %s", got, tc.client)
} }
@@ -282,7 +282,7 @@ func wantQuestionBody(
) )
} }
values := asked.header.Values(waf.ContentLengthHeader) values := asked.header.Values(wafprotocol.ContentLengthHeader)
if announced { if announced {
if !slices.Equal(values, []string{strconv.Itoa(size)}) || if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
@@ -586,8 +586,8 @@ func TestAQuestionNamesNoZone(t *testing.T) {
server.Handler.ServeHTTP(httptest.NewRecorder(), req) server.Handler.ServeHTTP(httptest.NewRecorder(), req)
asked := standIn.question(t) asked := standIn.question(t)
if asked.header.Get(waf.ClientHeader) != tc.client || if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
asked.header.Get(waf.PeerHeader) != tc.peer { asked.header.Get(wafprotocol.PeerHeader) != tc.peer {
t.Errorf("the question's headers are %v", asked.header) t.Errorf("the question's headers are %v", asked.header)
} }
+11 -13
View File
@@ -18,17 +18,13 @@ import (
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as // flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
// soon as it arrives. // soon as it arrives.
const flushAfterEachWrite time.Duration = -1 const flushAfterEachWrite time.Duration = -1
// reportStatusMax is the highest status waf takes in a report. An app can
// send a higher one, up to 999, and the visitor gets that as it is.
const reportStatusMax = 599
// plainText is the Content-Type of the answers rproxy gives itself. // plainText is the Content-Type of the answers rproxy gives itself.
const plainText = "text/plain; charset=utf-8" const plainText = "text/plain; charset=utf-8"
@@ -154,7 +150,7 @@ func (rq *request) readFirstPartOfBody() bool {
_ = rq.rc.SetReadDeadline(rq.requestDeadline()) _ = rq.rc.SetReadDeadline(rq.requestDeadline())
held, err := io.ReadAll( held, err := io.ReadAll(
io.LimitReader(rq.body, waf.QuestionBodyMaxBytes), io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
) )
// From here on, the request timer ends a read of the rest of the body, so that // From here on, the request timer ends a read of the rest of the body, so that
// the answer can name the side rproxy was waiting on. // the answer can name the side rproxy was waiting on.
@@ -228,16 +224,17 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
setPathAsSent(pr.Out.URL, pr.In) setPathAsSent(pr.Out.URL, pr.In)
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted) setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
pr.Out.Header.Set(requestIDHeader, rq.id) pr.Out.Header.Set(wafprotocol.RequestIDHeader, rq.id)
pr.Out.Header.Del(asnHeader) // Del removes a visitor's own, whatever their case.
pr.Out.Header.Del(countryHeader) pr.Out.Header.Del(wafprotocol.ASNHeader)
pr.Out.Header.Del(wafprotocol.CountryHeader)
if rq.asn != "" { if rq.asn != "" {
pr.Out.Header.Set(asnHeader, rq.asn) pr.Out.Header.Set(wafprotocol.ASNHeader, rq.asn)
} }
if rq.country != "" { if rq.country != "" {
pr.Out.Header.Set(countryHeader, rq.country) pr.Out.Header.Set(wafprotocol.CountryHeader, rq.country)
} }
} }
@@ -470,9 +467,10 @@ func (rq *request) finish(ctx context.Context) {
return return
} }
report := &waf.Report{ // An app can send a status up to 999, and the visitor gets it as it is.
report := &wafprotocol.Report{
RequestID: rq.id, RequestID: rq.id,
Status: min(rq.out.status, reportStatusMax), Status: min(rq.out.status, wafprotocol.ReportStatusMax),
RequestBytes: requestBytes, RequestBytes: requestBytes,
ResponseBytes: responseBytes, ResponseBytes: responseBytes,
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)), DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
+12 -17
View File
@@ -24,7 +24,7 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// How rproxy keeps connections to the app and to waf open between // How rproxy keeps connections to the app and to waf open between
@@ -47,10 +47,6 @@ const (
idleTimeout = 120 * time.Second idleTimeout = 120 * time.Second
) )
// ownEndpointPrefix starts the path of every request for sww itself, which
// never reaches the app.
const ownEndpointPrefix = "/_sww/"
// allowedMethods lists the methods rproxy passes on, as an Allow header gives // allowedMethods lists the methods rproxy passes on, as an Allow header gives
// them. It answers any other method 405 itself. // them. It answers any other method 405 itself.
// The WebDAV ones are for Nextcloud. // The WebDAV ones are for Nextcloud.
@@ -92,9 +88,10 @@ func New(params Params) *Server {
transport: newTransport(), transport: newTransport(),
wafTransport: newWAFTransport(), wafTransport: newWAFTransport(),
reportURL: (&url.URL{ reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath, Scheme: backend.Scheme,
Host: backend.Host,
Path: wafprotocol.ReportPath,
}).String(), }).String(),
breakerClock: time.Now,
} }
return &Server{ return &Server{
@@ -134,10 +131,8 @@ type handler struct {
wafTransport http.RoundTripper wafTransport http.RoundTripper
// reportURL is where the reports go. // reportURL is where the reports go.
reportURL string reportURL string
// breaker is the circuit breaker for asks to waf, and breakerClock gives it // breaker is the circuit breaker for asks to waf.
// the time. breaker breaker
breaker breaker
breakerClock func() time.Time
// reports waits for the reports being sent. reportsMu guards starting one // reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another. // against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup reports sync.WaitGroup
@@ -179,7 +174,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer rq.readRestOfBody() defer rq.readRestOfBody()
defer rq.finish(r.Context()) defer rq.finish(r.Context())
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath { if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
rq.answerHealth() rq.answerHealth()
return return
@@ -212,9 +207,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
rq.noteAnswer(answer) rq.noteAnswer(answer)
switch rq.disposition { switch rq.disposition {
case waf.DispositionAllow: case wafprotocol.DispositionAllow:
rq.passToApp(r.Context()) rq.passToApp(r.Context())
case waf.DispositionClose: case wafprotocol.DispositionClose:
// This panic has Go's server close the connection without an answer, // This panic has Go's server close the connection without an answer,
// and log nothing; the deferred finish still runs. // and log nothing; the deferred finish still runs.
panic(http.ErrAbortHandler) panic(http.ErrAbortHandler)
@@ -224,8 +219,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
// passToApp passes the request on to the app, unless a limit or its path keeps // passToApp passes the request on to the app, unless a limit or its path keeps
// it from the app. waf answers a path under ownEndpointPrefix itself, so // it from the app. waf answers a path under wafprotocol.OwnEndpointPrefix
// such a request gets here only when // itself, so such a request gets here only when
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it. // SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
// ctx is the request's own context. // ctx is the request's own context.
func (rq *request) passToApp(ctx context.Context) { func (rq *request) passToApp(ctx context.Context) {
@@ -234,7 +229,7 @@ func (rq *request) passToApp(ctx context.Context) {
switch { switch {
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes): case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
rq.answer(http.StatusRequestEntityTooLarge) rq.answer(http.StatusRequestEntityTooLarge)
case strings.HasPrefix(rq.in.URL.Path, ownEndpointPrefix): case strings.HasPrefix(rq.in.URL.Path, wafprotocol.OwnEndpointPrefix):
rq.answer(http.StatusNotFound) rq.answer(http.StatusNotFound)
default: default:
rq.forward(ctx) rq.forward(ctx)
+33 -66
View File
@@ -23,7 +23,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -56,7 +56,7 @@ const (
// The headers the tests set and look at. // The headers the tests set and look at.
const ( const (
dispositionHeader = waf.DispositionHeader dispositionHeader = wafprotocol.DispositionHeader
forwardedForHeader = "X-Forwarded-For" forwardedForHeader = "X-Forwarded-For"
forwardedProtoHeader = "X-Forwarded-Proto" forwardedProtoHeader = "X-Forwarded-Proto"
requestIDHeader = "X-Request-ID" requestIDHeader = "X-Request-ID"
@@ -72,9 +72,9 @@ const (
// What the stand-in for waf answers. // What the stand-in for waf answers.
const ( const (
dispositionAllow = waf.DispositionAllow dispositionAllow = wafprotocol.DispositionAllow
dispositionDeny = waf.DispositionDeny dispositionDeny = wafprotocol.DispositionDeny
dispositionClose = waf.DispositionClose dispositionClose = wafprotocol.DispositionClose
// deniedBody is the body of a deny. // deniedBody is the body of a deny.
deniedBody = "refused by waf" deniedBody = "refused by waf"
) )
@@ -175,7 +175,7 @@ type WAF struct {
*httptest.Server *httptest.Server
questions chan question questions chan question
reports chan waf.Report reports chan wafprotocol.Report
} }
// startWAF starts a stand-in for waf. // startWAF starts a stand-in for waf.
@@ -184,11 +184,11 @@ func startWAF(t *testing.T, answer http.HandlerFunc) *WAF {
s := &WAF{ s := &WAF{
questions: make(chan question, 10), questions: make(chan question, 10),
reports: make(chan waf.Report, 10), reports: make(chan wafprotocol.Report, 10),
} }
s.Server = httptest.NewServer(http.HandlerFunc( s.Server = httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) { func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) == "" { if r.Header.Get(wafprotocol.ClientHeader) == "" {
s.takeReport(t, w, r) s.takeReport(t, w, r)
return return
@@ -217,11 +217,11 @@ func (s *WAF) takeReport(
) { ) {
t.Helper() t.Helper()
var report waf.Report var report wafprotocol.Report
err := json.NewDecoder(r.Body).Decode(&report) err := json.NewDecoder(r.Body).Decode(&report)
if err != nil || r.Method != http.MethodPost || if err != nil || r.Method != http.MethodPost ||
r.URL.Path != waf.ReportPath { r.URL.Path != wafprotocol.ReportPath {
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err) t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
} }
@@ -242,7 +242,7 @@ func (s *WAF) wantNoReport(t *testing.T) {
} }
// report waits for a report. // report waits for a report.
func (s *WAF) report(t *testing.T) waf.Report { func (s *WAF) report(t *testing.T) wafprotocol.Report {
t.Helper() t.Helper()
select { select {
@@ -251,7 +251,7 @@ func (s *WAF) report(t *testing.T) waf.Report {
case <-time.After(waitLimit): case <-time.After(waitLimit):
t.Fatal("no report") t.Fatal("no report")
return waf.Report{} return wafprotocol.Report{}
} }
} }
@@ -279,7 +279,7 @@ func startWAFAnsweringReports(
backend := httptest.NewServer(http.HandlerFunc( backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) { func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" { if r.Header.Get(wafprotocol.ClientHeader) != "" {
answer(w, r) answer(w, r)
return return
@@ -356,19 +356,6 @@ func startServerLogging(
) (string, *rproxy.Server) { ) (string, *rproxy.Server) {
t.Helper() t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{ settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL, "SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000", wafBackendTimeout: "60000",
@@ -380,7 +367,7 @@ func newServer(
t.Fatalf("settings %v: %v", settings, err) t.Fatalf("settings %v: %v", settings, err)
} }
return rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: requestlog.NewProcessLogger(
@@ -389,11 +376,6 @@ func newServer(
slog.LevelInfo, slog.LevelInfo,
), ),
}) })
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
t.Context(), t.Context(),
@@ -414,7 +396,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
server.WaitForReports() server.WaitForReports()
}) })
return listener.Addr().String() return listener.Addr().String(), server
} }
// answered is the visitor's answer. // answered is the visitor's answer.
@@ -829,7 +811,7 @@ func TestAnAnswerRProxyHoldsWholeIsSentBeforeItsLogLine(t *testing.T) {
}{ }{
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden}, {"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest}, {"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
{"the health check", "GET " + waf.HealthPath + host, "ok\n", http.StatusOK}, {"the health check", "GET " + wafprotocol.HealthPath + host, "ok\n", http.StatusOK},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1452,18 +1434,14 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
greet(w, r) greet(w, r)
}) })
out := &output{} addr, out := startRProxy(
server := newServer( t,
t, standIn.URL, app.URL, standIn.URL,
map[string]string{wafBackendTimeout: "200"}, app.URL,
out, out, map[string]string{
wafBackendTimeout: "200",
},
) )
// The breaker's clock stands still, as in
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
// failures could take more than five seconds, and the next request could
// come more than a second later.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
// An ask can time out before it reaches the stand-in, so these // An ask can time out before it reaches the stand-in, so these
// may leave fewer than five questions, or leave some late. // may leave fewer than five questions, or leave some late.
@@ -1554,13 +1532,9 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r) tooLarge(w, r)
}) })
app := startApp(t, greet) app := startApp(t, greet)
out := &output{} addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
server := newServer(t, standIn.URL, app.URL, nil, out, out) wafBackendTimeout: "60000",
// The breaker's clock stands still. On a slow host, the five failures could })
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 { for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway { if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
@@ -1602,19 +1576,19 @@ func TestHealthCheckIsAnsweredWhileWAFIsDown(t *testing.T) {
app := startApp(t, greet) app := startApp(t, greet)
addr, out := startRProxy(t, nowhere, app.URL, nil) addr, out := startRProxy(t, nowhere, app.URL, nil)
got := get(t, addr, waf.HealthPath) got := get(t, addr, wafprotocol.HealthPath)
if got.status != http.StatusOK || got.body != "ok\n" { if got.status != http.StatusOK || got.body != "ok\n" {
t.Errorf("answered %+v, want ok", got) t.Errorf("answered %+v, want ok", got)
} }
line := out.requestLine(t) line := out.requestLine(t)
if line["path"] != waf.HealthPath || line["disposition"] != nil { if line["path"] != wafprotocol.HealthPath || line["disposition"] != nil {
t.Errorf("line %v, want one with no disposition", line) t.Errorf("line %v, want one with no disposition", line)
} }
// With waf up too, it is not asked. // With waf up too, it is not asked.
addr, _ = startRProxy(t, standIn.URL, app.URL, nil) addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
get(t, addr, waf.HealthPath) get(t, addr, wafprotocol.HealthPath)
if len(standIn.questions) != 0 { if len(standIn.questions) != 0 {
t.Error("waf was asked about the health check") t.Error("waf was asked about the health check")
@@ -1775,7 +1749,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
release := make(chan struct{}) release := make(chan struct{})
backend := httptest.NewServer(http.HandlerFunc( backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) { func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" { if r.Header.Get(wafprotocol.ClientHeader) != "" {
allow(w, r) allow(w, r)
return return
@@ -1835,15 +1809,11 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) { func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
t.Parallel() t.Parallel()
// The app's answer to the upgrade arrives well within these limits, even on a
// busy host.
const limit = 5 * time.Second
standIn := startWAF(t, allow) standIn := startWAF(t, allow)
app := startApp(t, echoAfterUpgrade) app := startApp(t, echoAfterUpgrade)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{ addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10), maxRequestTime: "2000",
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10), maxResponseTime: "2000",
}) })
conn := dial(t, addr) conn := dial(t, addr)
@@ -1868,10 +1838,7 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
} }
// Past both time limits, the connection still carries what is sent. // Past both time limits, the connection still carries what is sent.
time.Sleep(limit + 100*time.Millisecond) time.Sleep(2100 * time.Millisecond)
// dial's deadline ran from before the upgrade and this wait.
_ = conn.SetDeadline(time.Now().Add(waitLimit))
const message = "still here\n" const message = "still here\n"
+37 -84
View File
@@ -1,17 +1,17 @@
package rproxy package rproxy
import ( import (
"bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"maps" "maps"
"net/http" "net/http"
"strings" "strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// answerBodyMaxBytes is the most of the body of waf's answer that is read. // answerBodyMaxBytes is the most of the body of waf's answer that is read.
@@ -40,15 +40,12 @@ type wafAnswer struct {
// notes nothing, and no report is due for it: waf holds no decision to report // notes nothing, and no report is due for it: waf holds no decision to report
// on. // on.
func (rq *request) noteAnswer(answer *wafAnswer) { func (rq *request) noteAnswer(answer *wafAnswer) {
disposition := answer.header.Get(waf.DispositionHeader) read, valid := wafprotocol.ReadAnswer(answer.header)
if valid {
switch disposition { rq.disposition = read.Disposition
case waf.DispositionAllow, waf.DispositionDeny,
waf.DispositionClose:
rq.disposition = disposition
rq.answered = true rq.answered = true
rq.asn = answer.header.Get(asnHeader) rq.asn = read.ASN
rq.country = answer.header.Get(countryHeader) rq.country = read.Country
} }
} }
@@ -79,7 +76,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
rq.wafNoAnswer = true rq.wafNoAnswer = true
if rq.disposition == waf.DispositionAllow { if rq.disposition == wafprotocol.DispositionAllow {
rq.passToApp(ctx) rq.passToApp(ctx)
} else { } else {
rq.answer(http.StatusServiceUnavailable) rq.answer(http.StatusServiceUnavailable)
@@ -96,7 +93,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context, ctx context.Context,
) (*wafAnswer, error) { ) (*wafAnswer, error) {
b := &rq.h.breaker b := &rq.h.breaker
if !b.mayAsk(rq.h.breakerClock()) { if !b.mayAsk(time.Now()) {
return nil, errBreakerOpen return nil, errBreakerOpen
} }
@@ -106,7 +103,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil: case err == nil:
b.noteAnswer() b.noteAnswer()
case ctx.Err() == nil: case ctx.Err() == nil:
b.noteFailure(rq.h.breakerClock()) b.noteFailure(time.Now())
} }
return answer, err return answer, err
@@ -119,7 +116,21 @@ func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout) ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
defer cancel() defer cancel()
question, err := rq.newQuestion(ctx) var held []byte
if rq.body != nil {
held = rq.body.held
}
question, err := wafprotocol.NewQuestion(ctx, wafprotocol.QuestionParams{
Visitor: rq.in,
PathAsSent: rules.Path(rq.in),
Body: held,
WAFBackend: rq.h.config.WAFBackend,
Client: rq.client,
Peer: rq.peer,
RequestID: rq.id,
Scheme: scheme(rq.in, rq.peerTrusted),
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -176,69 +187,13 @@ func canBeAsked(r *http.Request) bool {
len(r.Header.Values("User-Agent")) <= 1 len(r.Header.Values("User-Agent")) <= 1
} }
// newQuestion returns the question about the request, as "The question" in
// SPEC.md gives it: the visitor's request as the visitor sent it, with the part
// of the body it holds and rproxy's headers.
func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) {
in := rq.in
var held []byte
if rq.body != nil {
held = rq.body.held
}
question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
ctx,
in.Method,
rq.h.config.WAFBackend.String(),
bytes.NewReader(held),
)
if err != nil {
return nil, fmt.Errorf("make the question: %w", err)
}
backend := question.URL
target := *in.URL
target.Scheme = backend.Scheme
target.Host = backend.Host
setPathAsSent(&target, in)
question.URL = &target
question.Host = in.Host
// The length NewRequestWithContext set is that of what is held. This line makes
// a body whose length the visitor did not announce go chunked, as "The question"
// in SPEC.md requires.
question.TransferEncoding = in.TransferEncoding
question.Header = in.Header.Clone()
question.Header.Set(waf.ClientHeader, rq.client.String())
question.Header.Set(waf.PeerHeader, rq.peer.String())
question.Header.Del(waf.ContentLengthHeader)
announced := in.Header.Get("Content-Length")
if announced != "" {
question.Header.Set(waf.ContentLengthHeader, announced)
}
question.Header.Set(requestIDHeader, rq.id)
question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted))
// Go's client adds a User-Agent of its own to a request without the key.
_, sent := question.Header["User-Agent"]
if !sent {
question.Header["User-Agent"] = nil
}
return question, nil
}
// answerWith gives the visitor waf's answer, less its disposition. // answerWith gives the visitor waf's answer, less its disposition.
func (rq *request) answerWith(answer *wafAnswer) { func (rq *request) answerWith(answer *wafAnswer) {
rq.prepareOwnAnswer(rq.requestDeadline()) rq.prepareOwnAnswer(rq.requestDeadline())
header := rq.out.Header() header := rq.out.Header()
maps.Copy(header, answer.header) maps.Copy(header, answer.header)
header.Del(waf.DispositionHeader) header.Del(wafprotocol.DispositionHeader)
// Go's server adds a Content-Type it guesses from the body to a response whose // Go's server adds a Content-Type it guesses from the body to a response whose
// headers have no Content-Type key. The key with no value sends none. // headers have no Content-Type key. The key with no value sends none.
@@ -252,7 +207,10 @@ func (rq *request) answerWith(answer *wafAnswer) {
// sendReport sends waf report on a goroutine of its own, so that the // sendReport sends waf report on a goroutine of its own, so that the
// visitor does not wait for it. The report goes on after ctx, the request's own // visitor does not wait for it. The report goes on after ctx, the request's own
// context, has ended. // context, has ended.
func (h *handler) sendReport(ctx context.Context, report *waf.Report) { func (h *handler) sendReport(
ctx context.Context,
report *wafprotocol.Report,
) {
h.reportsMu.Lock() h.reportsMu.Lock()
defer h.reportsMu.Unlock() defer h.reportsMu.Unlock()
@@ -270,23 +228,18 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
// postReport posts report to waf, and waits for its answer for up to // postReport posts report to waf, and waits for its answer for up to
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS. // SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
func (h *handler) postReport(ctx context.Context, report *waf.Report) error { func (h *handler) postReport(
body, err := json.Marshal(report) ctx context.Context,
if err != nil { report *wafprotocol.Report,
return fmt.Errorf("encode the report: %w", err) ) error {
}
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout) ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
defer cancel() defer cancel()
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL req, err := wafprotocol.NewReportRequest(ctx, h.reportURL, report)
ctx, http.MethodPost, h.reportURL, bytes.NewReader(body))
if err != nil { if err != nil {
return fmt.Errorf("make the request: %w", err) return err
} }
req.Header.Set("Content-Type", "application/json")
res, err := h.wafTransport.RoundTrip(req) res, err := h.wafTransport.RoundTrip(req)
if err != nil { if err != nil {
return fmt.Errorf("send the report: %w", err) return fmt.Errorf("send the report: %w", err)
+2 -2
View File
@@ -21,9 +21,9 @@ import (
"time" "time"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"golang.org/x/net/http/httpguts"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
// The actions a rule takes when it matches. // The actions a rule takes when it matches.
@@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
rule.Target, rule.Target,
errNotTarget, errNotTarget,
) )
case isHeader && !httpguts.ValidHeaderFieldName(headerName): case isHeader && !config.IsHeaderName(headerName):
return Rule{}, false, fmt.Errorf( return Rule{}, false, fmt.Errorf(
"the target %q %w", "the target %q %w",
rule.Target, rule.Target,
+18 -30
View File
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the // A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it. // write off no further, and is written with it.
first, _ := params.Ledger.Ban( first, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
second, _ := params.Ledger.Ban( second, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"), netip.MustParsePrefix("203.0.113.10/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
time.Sleep(5*time.Second - time.Nanosecond) time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run) run(t, load(t, params).Run)
// A ban, and bans.json written with it. // A ban, and bans.json written with it.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"), params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit}) midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay) time.Sleep(params.BansWriteDelay)
synctest.Wait() synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before // A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and // then, an admin saves bans.json with the first ban lifted and
// another added. // another added.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"), params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit}) midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay - time.Nanosecond) time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait() synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON) edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err) t.Fatalf("mkdir: %v", err)
} }
params.Ledger.Ban( params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
params.Limiter.Count( params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err) t.Fatalf("mkdir: %v", err)
} }
params.Ledger.Ban( params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"), netip.MustParsePrefix("203.0.113.9/32"),
midnight(), midnight(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
err = files.WriteAll() err = files.WriteAll()
@@ -2135,11 +2135,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted. // The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32") netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.Ban( params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting()) _, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned { if !banned {
@@ -2514,15 +2510,11 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()}) params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.Ban(client, now, bans.Grounds{ params.Ledger.BanForLimit(client, now, bans.Notes{
Cause: bans.CauseLimit, ASN: asn, ASName: asName, Country: "DE", Limit: 1,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
}) })
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now, params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{ bans.Notes{RuleID: "env-file", Target: "path"})
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole) params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2708,11 +2700,7 @@ func wantLiftedBanKept(
} }
// Were the lifted ban counted, the next would last three hours. // Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.Ban( ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
if ban.Expires.Sub(ban.Start) != time.Hour { if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf( t.Errorf(
"the next ban lasts %s, want 1h", "the next ban lasts %s, want 1h",
+2 -2
View File
@@ -13,7 +13,7 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// healthCheckTimeout bounds the whole health check. // healthCheckTimeout bounds the whole health check.
@@ -116,7 +116,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
health := "http://" + net.JoinHostPort( health := "http://" + net.JoinHostPort(
"127.0.0.1", "127.0.0.1",
port, port,
) + waf.HealthPath ) + wafprotocol.HealthPath
req, err := http.NewRequestWithContext( req, err := http.NewRequestWithContext(
ctx, ctx,
+24 -21
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/sww" "sneak.berlin/go/smallwebwaf/internal/sww"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -164,7 +164,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
// One request's three lines carry one id, rproxy's with waf's answer. // One request's three lines carry one id, rproxy's with waf's answer.
line := firstOut.line(t, "path", bannedProbe) line := firstOut.line(t, "path", bannedProbe)
if line["disposition"] != waf.DispositionDeny || if line["disposition"] != wafprotocol.DispositionDeny ||
line["client_ip"] != placed { line["client_ip"] != placed {
t.Errorf("rproxy's line is %v", line) t.Errorf("rproxy's line is %v", line)
} }
@@ -249,7 +249,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
wantStatus(t, site+"/after-stop", placed, http.StatusOK) wantStatus(t, site+"/after-stop", placed, http.StatusOK)
line := out.line(t, "path", "/after-stop") line := out.line(t, "path", "/after-stop")
if line["disposition"] != waf.DispositionAllow || if line["disposition"] != wafprotocol.DispositionAllow ||
line["waf_backend_no_answer"] != true { line["waf_backend_no_answer"] != true {
t.Errorf("rproxy's line is %v", line) t.Errorf("rproxy's line is %v", line)
} }
@@ -437,7 +437,7 @@ func TestAChunkedBodyStaysABodyAtWAF(t *testing.T) {
const smuggledID = "smuggled-request" const smuggledID = "smuggled-request"
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" + smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
waf.ClientHeader + ": " + otherClient + "\r\n" + wafprotocol.ClientHeader + ": " + otherClient + "\r\n" +
"X-Request-ID: " + smuggledID + "\r\n\r\n" "X-Request-ID: " + smuggledID + "\r\n\r\n"
status := sendRaw( status := sendRaw(
@@ -605,10 +605,11 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
protocols := make(chan string, 10) protocols := make(chan string, 10)
backend := httptest.NewUnstartedServer(http.HandlerFunc( backend := httptest.NewUnstartedServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) { func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" { if r.Header.Get(wafprotocol.ClientHeader) != "" {
protocols <- r.Proto protocols <- r.Proto
w.Header().Set(waf.DispositionHeader, waf.DispositionDeny) w.Header().
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionDeny)
w.WriteHeader(http.StatusForbidden) w.WriteHeader(http.StatusForbidden)
} }
})) }))
@@ -690,8 +691,9 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
release := make(chan struct{}) release := make(chan struct{})
backend := httptest.NewServer(http.HandlerFunc( backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) { func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" { if r.Header.Get(wafprotocol.ClientHeader) != "" {
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow) w.Header().
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
return return
} }
@@ -755,7 +757,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
} }
} }
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. // sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn { func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper() t.Helper()
@@ -768,6 +771,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close() _ = conn.Close()
}) })
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request) _, err = io.WriteString(conn, request)
if err != nil { if err != nil {
t.Fatalf("send: %v", err) t.Fatalf("send: %v", err)
@@ -779,10 +787,7 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) { func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
t.Parallel() t.Parallel()
const ( const cutOffAfter = 5 * time.Second
cutOffAfter = 5 * time.Second
upstreamSetting = "SWW_RPROXY_UPSTREAM_URL"
)
asked := make(chan struct{}) asked := make(chan struct{})
release := make(chan struct{}) release := make(chan struct{})
@@ -796,7 +801,8 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc( backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) { func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow) w.Header().
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
})) }))
t.Cleanup(backend.Close) t.Cleanup(backend.Close)
@@ -804,9 +810,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
defer stop() defer stop()
env := map[string]string{ env := map[string]string{
rproxyListenAddr: localhost + ":0", rproxyListenAddr: localhost + ":0",
upstreamSetting: app.URL, rproxyUpstreamURL: app.URL,
wafBackend: backend.URL, wafBackend: backend.URL,
} }
out := &output{} out := &output{}
exited := make(chan int, 1) exited := make(chan int, 1)
@@ -832,9 +838,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop() stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn) text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 { if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err) t.Errorf("read %q (%v), want the connection cut off", text, err)
@@ -932,7 +935,7 @@ func TestMainRunsRProxy(t *testing.T) {
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9") addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
"http://"+addr+waf.HealthPath, http.NoBody) "http://"+addr+wafprotocol.HealthPath, http.NoBody)
if err != nil { if err != nil {
t.Fatalf("new request: %v", err) t.Fatalf("new request: %v", err)
} }
+2 -2
View File
@@ -23,7 +23,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/sww" "sneak.berlin/go/smallwebwaf/internal/sww"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -1813,7 +1813,7 @@ func statusFrom(t *testing.T, url, from string) int {
} }
req.Header.Set("X-Forwarded-For", from) req.Header.Set("X-Forwarded-For", from)
req.Header.Set(waf.ClientHeader, from) req.Header.Set(wafprotocol.ClientHeader, from)
req.Header.Set("X-Request-ID", rand.Text()) req.Header.Set("X-Request-ID", rand.Text())
transport := &http.Transport{} transport := &http.Transport{}
+14 -8
View File
@@ -20,7 +20,7 @@ import (
_ "time/tzdata" _ "time/tzdata"
"sneak.berlin/go/smallwebwaf/internal/sww" "sneak.berlin/go/smallwebwaf/internal/sww"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// commandForMain, while set in the environment, has this test binary run Main // commandForMain, while set in the environment, has this test binary run Main
@@ -106,7 +106,7 @@ func TestMainRunsWAF(t *testing.T) {
starting := out.lineBeforeExit(t, "starting", exited) starting := out.lineBeforeExit(t, "starting", exited)
addr, _ := starting["address"].(string) addr, _ := starting["address"].(string)
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow) ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
// The files are written as sww stops. // The files are written as sww stops.
err = cmd.Process.Signal(syscall.SIGTERM) err = cmd.Process.Signal(syscall.SIGTERM)
@@ -217,7 +217,7 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
t.Fatalf("new request: %v", err) t.Fatalf("new request: %v", err)
} }
req.Header.Set(waf.ClientHeader, placed) req.Header.Set(wafprotocol.ClientHeader, placed)
req.Header.Set("X-Request-ID", rand.Text()) req.Header.Set("X-Request-ID", rand.Text())
transport := &http.Transport{} transport := &http.Transport{}
@@ -232,11 +232,11 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
_ = res.Body.Close() _ = res.Body.Close()
if res.StatusCode != status || if res.StatusCode != status ||
res.Header.Get(waf.DispositionHeader) != disposition { res.Header.Get(wafprotocol.DispositionHeader) != disposition {
t.Errorf( t.Errorf(
"answered %d with disposition %q, want %d with %q", "answered %d with disposition %q, want %d with %q",
res.StatusCode, res.StatusCode,
res.Header.Get(waf.DispositionHeader), res.Header.Get(wafprotocol.DispositionHeader),
status, status,
disposition, disposition,
) )
@@ -287,12 +287,18 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
} }
runWAF(t, env, func(addr string) { runWAF(t, env, func(addr string) {
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow) ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
// The default rule file bans a probe for this. // The default rule file bans a probe for this.
ask(t, addr, "/.env", http.StatusForbidden, waf.DispositionDeny) ask(
t,
addr,
"/.env",
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
}) })
runWAF(t, env, func(addr string) { runWAF(t, env, func(addr string) {
ask(t, addr, "/", http.StatusForbidden, waf.DispositionDeny) ask(t, addr, "/", http.StatusForbidden, wafprotocol.DispositionDeny)
}) })
} }
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"", "",
map[string]string{metricsToken: token}, map[string]string{metricsToken: token},
) )
server.Ledger.Ban( server.Ledger.BanForLimit(
netip.MustParsePrefix(otherClient+"/32"), netip.MustParsePrefix(otherClient+"/32"),
clk.Now(), clk.Now(),
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
before := server.Ledger.Snapshot() before := server.Ledger.Snapshot()
+5 -3
View File
@@ -130,9 +130,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make // A ban for a clear sign of attack, which a request under it would make
// permanent. // permanent.
group := netip.MustParsePrefix(ipv6Group) group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{ attackBan, _ := server.Ledger.BanForAttack(
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"}, group,
}) start,
bans.Notes{RuleID: "probe"},
)
// The third request breaks the limit, and so does the fourth, within the // The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack. // cooldown, which raises nothing. The probe is a clear sign of attack.
+63 -25
View File
@@ -26,6 +26,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -84,7 +85,7 @@ func TestSixteenThousandCookiesAreRefusedBeforeTheCoreRuleSetReadsThem(
got := do(t, req) got := do(t, req)
wantDisposition(t, got, tooLarge, waf.DispositionDeny) wantDisposition(t, got, tooLarge, wafprotocol.DispositionDeny)
line := s.out.requestLine(t) line := s.out.requestLine(t)
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge) wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
@@ -164,9 +165,14 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck(
line := s.out.requestLine(t) line := s.out.requestLine(t)
if tc.status == http.StatusOK { if tc.status == http.StatusOK {
wantDisposition(t, got, tc.status, waf.DispositionAllow) wantDisposition(
t,
got,
tc.status,
wafprotocol.DispositionAllow,
)
} else { } else {
wantDisposition(t, got, tc.status, waf.DispositionDeny) wantDisposition(t, got, tc.status, wafprotocol.DispositionDeny)
} }
wantLine(t, line, tc.status, tc.action) wantLine(t, line, tc.status, tc.action)
@@ -219,9 +225,14 @@ func TestAHeaderLineOverTheLimitIsRefusedWith431(t *testing.T) {
got := do(t, req) got := do(t, req)
if tc.refused { if tc.refused {
wantDisposition(t, got, tooLarge, waf.DispositionDeny) wantDisposition(
t,
got,
tooLarge,
wafprotocol.DispositionDeny,
)
} else { } else {
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
} }
}) })
} }
@@ -244,10 +255,7 @@ func TestRequestLineRefusalNamesItsHeuristicLengthAndLimit(t *testing.T) {
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) { func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
t.Parallel() t.Parallel()
repeated := make([]string, headerFieldLimit) repeated := slices.Repeat([]string{"1"}, headerFieldLimit)
for i := range repeated {
repeated[i] = "1"
}
for _, tc := range []struct { for _, tc := range []struct {
name string name string
@@ -294,7 +302,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
line := s.out.requestLine(t) line := s.out.requestLine(t)
if tc.refused { if tc.refused {
wantDisposition(t, got, tooLarge, waf.DispositionDeny) wantDisposition(
t,
got,
tooLarge,
wafprotocol.DispositionDeny,
)
wantLine( wantLine(
t, t,
line, line,
@@ -305,7 +318,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
return return
} }
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
got,
http.StatusOK,
wafprotocol.DispositionAllow,
)
wantLine(t, line, http.StatusOK, requestlog.ActionForward) wantLine(t, line, http.StatusOK, requestlog.ActionForward)
if line.DurationCoreRuleSetMS == nil { if line.DurationCoreRuleSetMS == nil {
@@ -332,7 +350,12 @@ func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) {
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Set(cookieHeader, cookies(cookieLimit+1)) req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny) wantDisposition(
t,
do(t, req),
tooLarge,
wafprotocol.DispositionDeny,
)
}) })
} }
} }
@@ -348,7 +371,12 @@ func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) {
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Set(cookieHeader, cookies(cookieLimit+1)) req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
} }
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) { func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
@@ -364,7 +392,7 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
got := do(t, req) got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
line := s.out.requestLine(t) line := s.out.requestLine(t)
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge || if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
@@ -386,14 +414,14 @@ func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
for range 2 { for range 2 {
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Set(cookieHeader, cookies(cookieLimit+1)) req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny) wantDisposition(t, do(t, req), tooLarge, wafprotocol.DispositionDeny)
} }
wantDisposition( wantDisposition(
t, t,
s.ask(client, "/"), s.ask(client, "/"),
http.StatusOK, http.StatusOK,
waf.DispositionAllow, wafprotocol.DispositionAllow,
) )
} }
@@ -466,7 +494,12 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
req := s.question(siteA, client, tc.path) req := s.question(siteA, client, tc.path)
req.Header.Set(cookieHeader, cookies(cookieLimit+1)) req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny) wantDisposition(
t,
do(t, req),
tooLarge,
wafprotocol.DispositionDeny,
)
wantLine( wantLine(
t, t,
s.out.requestLine(t), s.out.requestLine(t),
@@ -503,11 +536,16 @@ func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
over := s.question(siteA, client, "/") over := s.question(siteA, client, "/")
over.Header.Set(cookieHeader, cookies(cookieLimit+1)) over.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny) wantDisposition(t, do(t, over), tooLarge, wafprotocol.DispositionDeny)
within := s.question(siteA, client, "/") within := s.question(siteA, client, "/")
within.Header.Set(cookieHeader, cookies(cookieLimit)) within.Header.Set(cookieHeader, cookies(cookieLimit))
wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
do(t, within),
http.StatusOK,
wafprotocol.DispositionAllow,
)
lines := s.out.requestLines(t, 2) lines := s.out.requestLines(t, 2)
@@ -564,7 +602,7 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
t, t,
s.ask(fromDE, "/"), s.ask(fromDE, "/"),
http.StatusForbidden, http.StatusForbidden,
waf.DispositionDeny, wafprotocol.DispositionDeny,
) )
line := s.out.requestLine(t) line := s.out.requestLine(t)
@@ -601,7 +639,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
t, t,
s.ask(client, "/"), s.ask(client, "/"),
http.StatusForbidden, http.StatusForbidden,
waf.DispositionDeny, wafprotocol.DispositionDeny,
) )
line := s.out.requestLine(t) line := s.out.requestLine(t)
@@ -630,7 +668,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
req.Header.Set(cookieHeader, cookies(cookieLimit+1)) req.Header.Set(cookieHeader, cookies(cookieLimit+1))
got := do(t, req) got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" { if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
t.Errorf( t.Errorf(
@@ -672,7 +710,7 @@ func askDirectly(
http.NoBody, http.NoBody,
) )
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234") req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
req.Header.Set(waf.ClientHeader, client) req.Header.Set(wafprotocol.ClientHeader, client)
req.Header.Set(requestIDHeader, "the-id") req.Header.Set(requestIDHeader, "the-id")
change(req) change(req)
@@ -686,10 +724,10 @@ func askDirectly(
// banClient bans client at now. // banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) { func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.Ban( server.Ledger.BanForLimit(
netip.MustParsePrefix(client+"/32"), netip.MustParsePrefix(client+"/32"),
now, now,
bans.Grounds{Cause: bans.CauseLimit}, bans.Notes{},
) )
} }
+53 -72
View File
@@ -103,8 +103,9 @@ func (rq *request) countedBytes() int64 {
// one whose bytes broke a byte limit, the refusal for one that broke the // one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a // refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets // rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what // the client's counters back to zero. In observe mode it makes no ban and
// happens instead. // sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (rq *request) banForLimit( func (rq *request) banForLimit(
now time.Time, now time.Time,
hit ratelimit.Hit, hit ratelimit.Hit,
@@ -122,11 +123,23 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit) rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{ notes := bans.Notes{
Kind: hit.Kind, ASN: rq.line.ASN,
Limit: hit.Limit, ASName: rq.line.ASName,
Window: hit.Window, Country: rq.line.Country,
Count: hit.Count, Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
} }
switch hit.Kind { switch hit.Kind {
@@ -136,54 +149,47 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged() notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
} }
rq.banClient( if rq.h.config.Observe {
now, ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
status, if wouldBan {
bans.Grounds{Cause: bans.CauseLimit, Notes: notes}, rq.alertBan(ban)
) }
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
}
} }
// banForAttack bans the client's netblock at now for a clear sign of // banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path // attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient // asked for. It fills in the rest of the notes. In observe mode it makes
// says what happens instead. // no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) string {
netblock := banNetblock(rq.h.config, rq.client) netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
if rq.h.config.Observe { if rq.h.config.Observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) { ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
if wouldBan { if wouldBan {
rq.alertBan(ban) rq.alertBan(ban)
} }
@@ -191,13 +197,7 @@ func (rq *request) banClient(
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock) ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
rq.line.BanExpires = banExpires(ban) rq.line.BanExpires = banExpires(ban)
if made { if made {
@@ -207,25 +207,6 @@ func (rq *request) banClient(
return banAction(ban.Permanent()) return banAction(ban.Permanent())
} }
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause // wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would // made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per // have made is worked out only then, at most once per
+5 -4
View File
@@ -548,10 +548,6 @@ func (s *sender) requestWithHeader(
// body is sent as it is after the headers, header holding its // body is sent as it is after the headers, header holding its
// Content-Length or Transfer-Encoding. header may hold several lines, // Content-Length or Transfer-Encoding. header may hold several lines,
// separated by "\r\n". It returns the whole answer. // separated by "\r\n". It returns the whole answer.
//
// It waits for the answer as long as it takes, since a busy host can slow
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
// whose answer never comes.
func (s *sender) requestWithBody( func (s *sender) requestWithBody(
method, from, path, header, body string, status int, action string, method, from, path, header, body string, status int, action string,
) (logLine, answer) { ) (logLine, answer) {
@@ -570,6 +566,11 @@ func (s *sender) requestWithBody(
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+ send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body) forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
if err != nil {
s.t.Fatalf("set read deadline: %v", err)
}
var got answer var got answer
res, err := http.ReadResponse(bufio.NewReader(conn), nil) res, err := http.ReadResponse(bufio.NewReader(conn), nil)
+2 -1
View File
@@ -19,6 +19,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// The biased thresholds. // The biased thresholds.
@@ -589,7 +590,7 @@ func serveFromDE(
t.Helper() t.Helper()
req := httptest.NewRequestWithContext(t.Context(), method, "/", body) req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
req.Header.Set(waf.ClientHeader, fromDE) req.Header.Set(wafprotocol.ClientHeader, fromDE)
req.Header.Set(requestIDHeader, "the-id") req.Header.Set(requestIDHeader, "the-id")
server.Handler.ServeHTTP(httptest.NewRecorder(), req) server.Handler.ServeHTTP(httptest.NewRecorder(), req)
+2 -4
View File
@@ -7,6 +7,7 @@ import (
"slices" "slices"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// peerAddress is the address of the request's TCP peer. // peerAddress is the address of the request's TCP peer.
@@ -19,14 +20,11 @@ func peerAddress(r *http.Request) netip.Addr {
return addrPort.Addr().Unmap() return addrPort.Addr().Unmap()
} }
// requestIDHeader carries the request's id, from rproxy.
const requestIDHeader = "X-Request-ID"
// requestID is the request's id: the one a question gives, or for any other // requestID is the request's id: the one a question gives, or for any other
// request a new random one. // request a new random one.
func requestID(r *http.Request, isQuestion bool) string { func requestID(r *http.Request, isQuestion bool) string {
if isQuestion { if isQuestion {
return r.Header.Get(requestIDHeader) return r.Header.Get(wafprotocol.RequestIDHeader)
} }
return rand.Text() return rand.Text()
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and // The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
@@ -395,9 +395,9 @@ func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) {
}) })
wantDisposition(t, do(t, s.question(siteA, client, fileName)), wantDisposition(t, do(t, s.question(siteA, client, fileName)),
http.StatusOK, waf.DispositionAllow) http.StatusOK, wafprotocol.DispositionAllow)
wantDisposition(t, do(t, s.question(siteB, client, fileName)), wantDisposition(t, do(t, s.question(siteB, client, fileName)),
http.StatusForbidden, waf.DispositionDeny) http.StatusForbidden, wafprotocol.DispositionDeny)
lines := s.out.requestLines(t, 2) lines := s.out.requestLines(t, 2)
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward) wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
+8 -3
View File
@@ -5,7 +5,7 @@ import (
"testing" "testing"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) { func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
@@ -25,8 +25,13 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
) )
for range healthChecks { for range healthChecks {
got := s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "", got := s.ownEndpointRequest(
http.StatusOK) "",
http.MethodGet,
wafprotocol.HealthPath,
"",
http.StatusOK,
)
if string(got.body) != "ok\n" || if string(got.body) != "ok\n" ||
got.header.Get("Content-Type") != contentType { got.header.Get("Content-Type") != contentType {
+2 -1
View File
@@ -11,6 +11,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
@@ -118,7 +119,7 @@ func TestOwnEndpointsAreNotInTheHistory(t *testing.T) {
s.ownEndpointRequest( s.ownEndpointRequest(
"", "",
http.MethodGet, http.MethodGet,
waf.HealthPath, wafprotocol.HealthPath,
"", "",
http.StatusOK, http.StatusOK,
) )
-22
View File
@@ -3,7 +3,6 @@ package waf
import ( import (
"context" "context"
"log/slog" "log/slog"
"net/http"
"net/netip" "net/netip"
"strings" "strings"
"time" "time"
@@ -16,15 +15,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
) )
// The headers in which an answer of allow gives rproxy the client's AS number
// and country, for the app, while SWW_WAF_ADD_LOOKUP_HEADERS is true. Go writes
// every header name in this form, as it sends it and as it receives it, so
// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive.
const (
asnHeader = "X-Client-Asn"
countryHeader = "X-Client-Country"
)
// lookupResult is what the lookup gave for a request: whether its client was // lookupResult is what the lookup gave for a request: whether its client was
// looked up, the answer, and whether the request skipped the wait for GeoJS. // looked up, the answer, and whether the request skipped the wait for GeoJS.
type lookupResult struct { type lookupResult struct {
@@ -167,18 +157,6 @@ func GeoJSFailed(
m.GeoJSFailures.Inc() m.GeoJSFailures.Inc()
} }
// setLookupHeaders sets the headers that give the client's AS number and
// country, leaving out one that is unknown.
func setLookupHeaders(header http.Header, asn, country string) {
if asn != "" {
header.Set(asnHeader, asn)
}
if country != "" {
header.Set(countryHeader, country)
}
}
// canBePlaced reports whether a lookup can place addr: private, loopback // canBePlaced reports whether a lookup can place addr: private, loopback
// and link-local addresses have no AS number or country. // and link-local addresses have no AS number or country.
func canBePlaced(addr netip.Addr) bool { func canBePlaced(addr netip.Addr) bool {
+2 -1
View File
@@ -16,6 +16,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -66,7 +67,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound}, {http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound}, {http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound}, {http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
{http.MethodPost, waf.HealthPath, "", http.StatusNotFound}, {http.MethodPost, wafprotocol.HealthPath, "", http.StatusNotFound},
} { } {
got := s.ownEndpointRequest( got := s.ownEndpointRequest(
tc.authorization, tc.authorization,
-8
View File
@@ -28,14 +28,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
) )
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
// never sees a path under it.
const ownEndpointPrefix = "/_sww/"
// HealthPath is the health endpoint of waf and of rproxy, which the
// container's health check asks.
const HealthPath = "/_sww/healthz"
// MetricsPath is where the metrics are, for a request that carries // MetricsPath is where the metrics are, for a request that carries
// SWW_WAF_METRICS_TOKEN. // SWW_WAF_METRICS_TOKEN.
const MetricsPath = "/_sww/metrics" const MetricsPath = "/_sww/metrics"
+6 -2
View File
@@ -193,11 +193,11 @@ func (o *output) requestLine(t *testing.T) logLine {
// endedLines waits for count requests that went through rproxy, or to waf's // endedLines waits for count requests that went through rproxy, or to waf's
// own endpoints, to have ended, and returns the last log line of each, in the // own endpoints, to have ended, and returns the last log line of each, in the
// order waf took them. A question's last line is the one its report writes. // order waf took them. A question's last line is the one its report writes.
// It waits as long as it takes, as requestWithBody does.
func (o *output) endedLines(t *testing.T, count int) []logLine { func (o *output) endedLines(t *testing.T, count int) []logLine {
t.Helper() t.Helper()
for { deadline := time.Now().Add(waitLimit)
for time.Now().Before(deadline) {
found := o.lastLines(t) found := o.lastLines(t)
ended := len(found) >= count ended := len(found) >= count
@@ -213,6 +213,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
time.Sleep(pollInterval) time.Sleep(pollInterval)
} }
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
return nil
} }
// lastLines returns the last request log line written so far of each request, // lastLines returns the last request log line written so far of each request,
+2 -2
View File
@@ -13,7 +13,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -248,7 +248,7 @@ func TestTimingsAreInOrder(t *testing.T) {
wantStatus(t, do(t, req), http.StatusForbidden) wantStatus(t, do(t, req), http.StatusForbidden)
refused := out.endedLines(t, 2)[1] refused := out.endedLines(t, 2)[1]
wantStatus(t, get(t, server.Addr, waf.HealthPath), http.StatusOK) wantStatus(t, get(t, server.Addr, wafprotocol.HealthPath), http.StatusOK)
health := out.endedLines(t, 3)[2] health := out.endedLines(t, 3)[2]
// A request passed to the app has every timing; one refused, none of // A request passed to the app has every timing; one refused, none of
+54 -226
View File
@@ -1,75 +1,21 @@
package waf package waf
import ( import (
"bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt"
"io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/netip"
"strconv"
"strings" "strings"
"time" "time"
"unicode/utf8"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
// The headers of rproxy's question and of waf's answer, as
// "waf" in SPEC.md gives them.
const (
// ClientHeader marks a request as rproxy's question, and gives the
// visitor's address.
ClientHeader = "X-Sww-Client"
// PeerHeader gives the address of rproxy's own TCP peer.
PeerHeader = "X-Sww-Peer"
// ContentLengthHeader gives the body length the visitor announced.
ContentLengthHeader = "X-Sww-Content-Length"
// DispositionHeader gives waf's answer.
DispositionHeader = "X-Sww-Disposition"
)
// The values of DispositionHeader.
const (
// DispositionAllow tells rproxy to pass the request to the app.
DispositionAllow = "allow"
// DispositionDeny tells rproxy to give the visitor the answer's status,
// headers and body.
DispositionDeny = "deny"
// DispositionClose tells rproxy to close the visitor's connection
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
// waf closing the question's connection instead would read as
// waf being down, and rproxy would then let the request through.
DispositionClose = "close"
)
// ReportPath is where rproxy sends the report of each request waf
// answered.
const ReportPath = "/_sww/report"
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
const QuestionBodyMaxBytes = 64 << 10
const ( const (
// maxAwaitingReport is the most answered requests held until their // maxAwaitingReport is the most answered requests held until their
// reports come. Past it, the request answered longest ago is dropped. // reports come. Past it, the request answered longest ago is dropped.
maxAwaitingReport = 10000 maxAwaitingReport = 10000
// maxRequestIDBytes is the longest id a question may give.
maxRequestIDBytes = 128
// reportBodyMaxBytes is the most of a report's body that is read. It is
// far above any report's.
reportBodyMaxBytes = 1 << 20
// maxHTTPStatus is the highest status a report may give.
maxHTTPStatus = 599
// maxReportedBytes is the most bytes a report may give for each way, 1
// PiB, so that adding them up cannot overflow.
maxReportedBytes = 1 << 50
// maxReportedMilliseconds is the longest duration a report may give,
// about 31 years, so that it fits a time.Duration.
maxReportedMilliseconds = 1e12
// wafTimeout bounds how long a request to waf takes to // wafTimeout bounds how long a request to waf takes to
// arrive, and its answer to leave, and wafIdleTimeout how long a // arrive, and its answer to leave, and wafIdleTimeout how long a
// kept-open connection waits for its next request. // kept-open connection waits for its next request.
@@ -77,26 +23,6 @@ const (
wafIdleTimeout = 120 * time.Second wafIdleTimeout = 120 * time.Second
) )
var (
errReportFieldMissing = errors.New("a field is missing")
errReportValue = errors.New("a value is out of range")
)
// Report is the JSON object rproxy posts to ReportPath once the visitor's
// answer has ended, its fields named as the request log's of the same
// meaning. RequestID is the question's id. Status is what the visitor got, 0
// if nothing. The durations are in milliseconds.
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Report struct {
RequestID string `json:"request_id"`
Status int `json:"status"`
RequestBytes int64 `json:"request_bytes"`
ResponseBytes int64 `json:"response_bytes"`
DurationTotalMS float64 `json:"duration_total_ms"`
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
}
// New returns the server `sww waf` runs. It answers // New returns the server `sww waf` runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It // rproxy's questions, takes rproxy's reports and answers its own endpoints. It
// passes nothing to an app. rproxy applies the size and time limits to the // passes nothing to an app. rproxy applies the size and time limits to the
@@ -132,10 +58,10 @@ func New(params Params) *Server {
} }
// ServeHTTP handles one request to waf: a request carrying // ServeHTTP handles one request to waf: a request carrying
// ClientHeader is rproxy's question, and any other is for waf // wafprotocol.ClientHeader is rproxy's question, and any other is for waf
// itself. // itself.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(r.Header.Values(ClientHeader)) > 0 { if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
h.answerQuestion(w, r) h.answerQuestion(w, r)
return return
@@ -162,19 +88,16 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
// carries, and answers rproxy what to do with it. A denied request's counting // carries, and answers rproxy what to do with it. A denied request's counting
// is finished here, and an allowed one's by its report. // is finished here, and an allowed one's by its report.
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) { func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
client, body, ok := h.readQuestion(w, r) question, ok := h.readQuestion(w, r)
if !ok { if !ok {
return return
} }
// One that cannot be read leaves the log line's peer_ip invalid.
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
rq := h.startRequest( rq := h.startRequest(
w, w,
asTheVisitorSentIt(r, body), question.Visitor,
client.Unmap(), question.Client,
peer.Unmap(), question.Peer,
true, true,
) )
// These run even when a check panics. finish runs first, so that what is // These run even when a check panics. finish runs first, so that what is
@@ -189,7 +112,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
rq.line.WAFLine = requestlog.WAFLineDecision rq.line.WAFLine = requestlog.WAFLineDecision
refused := rq.check(r.Context()) refused := rq.check(r.Context())
rq.checked = time.Now() rq.checked = time.Now()
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix) underOwnPrefix := strings.HasPrefix(
r.URL.Path,
wafprotocol.OwnEndpointPrefix,
)
if refused == nil && !underOwnPrefix { if refused == nil && !underOwnPrefix {
rq.allow() rq.allow()
@@ -205,7 +131,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
// The app never sees this prefix. waf's own endpoints are at its // The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does. // own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny) wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error( http.Error(
rq.out, rq.out,
http.StatusText(http.StatusNotFound), http.StatusText(http.StatusNotFound),
@@ -216,45 +145,19 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
rq.banAndCount(r.Context(), afterAnswer()) rq.banAndCount(r.Context(), afterAnswer())
} }
// readQuestion returns the visitor's address and the body a question carries, // readQuestion reads a question, and reports whether waf takes it. It answers
// and whether waf takes the question. It answers a question it does not // a question it does not take with 400 and no disposition. rproxy passes that
// take with 400 and no disposition. rproxy passes that 400 to the visitor, // 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
// and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does not apply. The id of // not apply. The id of a question waf takes is held from here on, without a
// a question waf takes is held from here on, without a decision yet, so // decision yet, so that another question with that id is refused while this
// that another question with that id is refused while this one is being // one is being handled.
// handled.
// //
// The body is read before any check, since Go's server does not send an answer // A question whose body stops arriving gets no answer, and is not looked at.
// until it has read the body. A question whose body stops arriving gets no
// answer, and is not looked at.
func (h *handler) readQuestion( func (h *handler) readQuestion(
w http.ResponseWriter, r *http.Request, w http.ResponseWriter, r *http.Request,
) (netip.Addr, []byte, bool) { ) (wafprotocol.Question, bool) {
client, err := netip.ParseAddr(r.Header.Get(ClientHeader)) question, err := wafprotocol.ReadQuestion(w, r)
if err != nil || client.Zone() != "" { if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
return netip.Addr{}, nil, false
}
id := r.Header.Get(requestIDHeader)
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
return netip.Addr{}, nil, false
}
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
)
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
h.refuseQuestion(w, "the body is over 64 KiB")
return netip.Addr{}, nil, false
}
if err != nil {
h.processLog.Warn( h.processLog.Warn(
"a question's body stopped arriving", "a question's body stopped arriving",
"error", "error",
@@ -265,17 +168,23 @@ func (h *handler) readQuestion(
panic(http.ErrAbortHandler) panic(http.ErrAbortHandler)
} }
held, _ := h.awaitingReport.ContainsOrAdd(id, nil) if err != nil {
h.refuseQuestion(w, err.Error())
return question, false
}
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
if held { if held {
h.refuseQuestion( h.refuseQuestion(
w, w,
requestIDHeader+" is the id of a request waf holds", wafprotocol.RequestIDHeader+" is the id of a request waf holds",
) )
return netip.Addr{}, nil, false return question, false
} }
return client, body, true return question, true
} }
// refuseQuestion answers a question waf does not take, for reason. // refuseQuestion answers a question waf does not take, for reason.
@@ -284,48 +193,18 @@ func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
http.Error(w, reason, http.StatusBadRequest) http.Error(w, reason, http.StatusBadRequest)
} }
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the // allow answers a question with wafprotocol.DispositionAllow and, while
// log line are to see it: without the headers rproxy adds for waf,
// with body, which has been read of r, and with the length the visitor
// announced, as -1 when it announced none. The question's own length covers
// only the part of the body it carries.
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
sent := r.Clone(r.Context())
announced := sent.Header.Get(ContentLengthHeader)
sent.Header.Del(ClientHeader)
sent.Header.Del(PeerHeader)
sent.Header.Del(ContentLengthHeader)
sent.Header.Del("Content-Length")
sent.ContentLength = -1
length, err := strconv.ParseInt(announced, 10, 64)
if err == nil && length >= 0 {
sent.ContentLength = length
sent.Header.Set("Content-Length", announced)
}
sent.Body = http.NoBody
if len(body) > 0 {
sent.Body = io.NopCloser(bytes.NewReader(body))
}
return sent
}
// allow answers a question with DispositionAllow and, while
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country. // SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
func (rq *request) allow() { func (rq *request) allow() {
rq.allowed = true rq.allowed = true
header := rq.out.Header() answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
header.Set(DispositionHeader, DispositionAllow)
if rq.h.config.AddLookupHeaders { if rq.h.config.AddLookupHeaders {
setLookupHeaders(header, rq.line.ASN, rq.line.Country) answer.ASN = rq.line.ASN
answer.Country = rq.line.Country
} }
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
rq.out.WriteHeader(http.StatusOK) rq.out.WriteHeader(http.StatusOK)
} }
@@ -343,13 +222,19 @@ func (rq *request) deny(r refusal) {
if r.status == 0 { if r.status == 0 {
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's // SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
// status stays 0: the visitor is sent nothing. // status stays 0: the visitor is sent nothing.
rq.out.Header().Set(DispositionHeader, DispositionClose) wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
)
rq.out.ResponseWriter.WriteHeader(http.StatusOK) rq.out.ResponseWriter.WriteHeader(http.StatusOK)
return return
} }
rq.out.Header().Set(DispositionHeader, DispositionDeny) wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(rq.out, http.StatusText(r.status), r.status) http.Error(rq.out, http.StatusText(r.status), r.status)
} }
@@ -357,7 +242,7 @@ func (rq *request) deny(r refusal) {
// runs on it: the address is on the private network, where the peers are // runs on it: the address is on the private network, where the peers are
// rproxy and the admin's tools, not visitors. // rproxy and the admin's tools, not visitors.
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) { func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == ReportPath { if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
h.takeReport(w, r) h.takeReport(w, r)
return return
@@ -368,7 +253,7 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
rq := h.startRequest(w, r, peer, peer, false) rq := h.startRequest(w, r, peer, peer, false)
defer rq.finish() defer rq.finish()
if r.Method == http.MethodGet && r.URL.Path == HealthPath { if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
rq.answerHealth() rq.answerHealth()
return return
@@ -377,11 +262,11 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
rq.answerAdmin() rq.answerAdmin()
} }
// takeReport answers POST ReportPath: it finishes the request the report // takeReport answers POST wafprotocol.ReportPath: it finishes the request the
// names, if it holds that request's decision, and ignores the report // report names, if it holds that request's decision, and ignores the report
// otherwise. // otherwise.
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) { func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
report, err := readReport(w, r) report, err := wafprotocol.ReadReport(w, r)
if err != nil { if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error()) h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest) http.Error(w, "the body is not a report", http.StatusBadRequest)
@@ -399,63 +284,6 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
// readReport reads a report from r's body, which must be one JSON object with
// all of Report's fields, in the ranges SPEC.md gives.
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
var report Report
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
)
if err != nil {
return report, fmt.Errorf("read the body: %w", err)
}
var fields map[string]json.RawMessage
err = json.Unmarshal(body, &fields)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
for _, name := range []string{
"request_id", "status", "request_bytes", "response_bytes",
"duration_total_ms", "duration_upstream_total_ms",
} {
value, there := fields[name]
if !there || string(value) == "null" {
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
}
}
err = json.Unmarshal(body, &report)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
return report, report.checkRanges()
}
// checkRanges returns an error for a value of the report that cannot be true.
func (report *Report) checkRanges() error {
switch {
case report.Status != 0 &&
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
return fmt.Errorf("%w: status", errReportValue)
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
return fmt.Errorf("%w: request_bytes", errReportValue)
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
return fmt.Errorf("%w: response_bytes", errReportValue)
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_total_ms", errReportValue)
case report.DurationUpstreamTotalMS < 0 ||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
}
return nil
}
// finishWith finishes a request with what its report says, and writes the // finishWith finishes a request with what its report says, and writes the
// request's last log line, which holds its decision's fields and the report's. // request's last log line, which holds its decision's fields and the report's.
// For an allowed request it first runs the heuristics of onReport, and adds the // For an allowed request it first runs the heuristics of onReport, and adds the
@@ -465,7 +293,7 @@ func (report *Report) checkRanges() error {
func (h *handler) finishWith( func (h *handler) finishWith(
ctx context.Context, ctx context.Context,
held *decision, held *decision,
report Report, report wafprotocol.Report,
) { ) {
rq := &request{ rq := &request{
decision: *held, decision: *held,
+195 -86
View File
@@ -22,6 +22,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
) )
const ( const (
@@ -106,8 +107,8 @@ func (s *WAF) questionWith(
req := newRequest(s.t, method, s.addr, path, body) req := newRequest(s.t, method, s.addr, path, body)
req.Host = host req.Host = host
req.Header.Set(waf.ClientHeader, from) req.Header.Set(wafprotocol.ClientHeader, from)
req.Header.Set(waf.PeerHeader, rproxyPeer) req.Header.Set(wafprotocol.PeerHeader, rproxyPeer)
req.Header.Set(requestIDHeader, rand.Text()) req.Header.Set(requestIDHeader, rand.Text())
return req return req
@@ -134,13 +135,18 @@ func (s *WAF) allowed(from string) string {
s.t.Helper() s.t.Helper()
req := s.question(siteA, from, "/") req := s.question(siteA, from, "/")
wantDisposition(s.t, do(s.t, req), http.StatusOK, waf.DispositionAllow) wantDisposition(
s.t,
do(s.t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
return req.Header.Get(requestIDHeader) return req.Header.Get(requestIDHeader)
} }
// report sends rproxy's report and returns the answer. // report sends rproxy's report and returns the answer.
func (s *WAF) report(report waf.Report) answer { func (s *WAF) report(report wafprotocol.Report) answer {
s.t.Helper() s.t.Helper()
body, err := json.Marshal(report) body, err := json.Marshal(report)
@@ -155,8 +161,11 @@ func (s *WAF) report(report waf.Report) answer {
func (s *WAF) reportBody(body string) answer { func (s *WAF) reportBody(body string) answer {
s.t.Helper() s.t.Helper()
return do(s.t, newRequest(s.t, http.MethodPost, s.addr, waf.ReportPath, return do(
strings.NewReader(body))) s.t,
newRequest(s.t, http.MethodPost, s.addr, wafprotocol.ReportPath,
strings.NewReader(body)),
)
} }
// passedOn asks about a request from the visitor at from on siteA, and reports // passedOn asks about a request from the visitor at from on siteA, and reports
@@ -164,7 +173,7 @@ func (s *WAF) reportBody(body string) answer {
func (s *WAF) passedOn(from string) { func (s *WAF) passedOn(from string) {
s.t.Helper() s.t.Helper()
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: s.allowed(from), RequestID: s.allowed(from),
Status: http.StatusOK, Status: http.StatusOK,
RequestBytes: bodyBytes, RequestBytes: bodyBytes,
@@ -210,7 +219,7 @@ func wantDisposition(
t.Helper() t.Helper()
wantStatus(t, got, status) wantStatus(t, got, status)
if answered := got.header.Get(waf.DispositionHeader); answered != if answered := got.header.Get(wafprotocol.DispositionHeader); answered !=
disposition { disposition {
t.Errorf( t.Errorf(
"answered with disposition %q, want %q", "answered with disposition %q, want %q",
@@ -272,10 +281,10 @@ func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) {
req := s.question(siteA, client, "/page?q=1&token=visible") req := s.question(siteA, client, "/page?q=1&token=visible")
req.Header.Set(requestIDHeader, "the-id") req.Header.Set(requestIDHeader, "the-id")
req.Header.Set("X-Forwarded-Proto", "https") req.Header.Set("X-Forwarded-Proto", "https")
req.Header.Set(waf.ContentLengthHeader, "1234") req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
got := do(t, req) got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
wantBody(t, got, "") wantBody(t, got, "")
if id := got.header.Get(requestIDHeader); id != "" { if id := got.header.Get(requestIDHeader); id != "" {
@@ -308,7 +317,12 @@ func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) {
s.allowed("2001:db8::1") s.allowed("2001:db8::1")
got := s.ask("2001:db8::2", "/") got := s.ask("2001:db8::2", "/")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
} }
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) { func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
@@ -317,9 +331,14 @@ func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
s := startWAF(t, "", map[string]string{denyNets: client}) s := startWAF(t, "", map[string]string{denyNets: client})
req := s.question(siteA, "::ffff:"+client, "/") req := s.question(siteA, "::ffff:"+client, "/")
req.Header.Set(waf.PeerHeader, "::ffff:"+rproxyPeer) req.Header.Set(wafprotocol.PeerHeader, "::ffff:"+rproxyPeer)
wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
do(t, req),
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
line := s.out.requestLine(t) line := s.out.requestLine(t)
if line.ClientIP != client || line.PeerIP != rproxyPeer { if line.ClientIP != client || line.PeerIP != rproxyPeer {
@@ -339,13 +358,13 @@ func TestMalformedQuestionGets400AndAWarningAndNoLine(t *testing.T) {
change func(*http.Request) change func(*http.Request)
}{ }{
{"client not an address", func(req *http.Request) { {"client not an address", func(req *http.Request) {
req.Header.Set(waf.ClientHeader, "not an address") req.Header.Set(wafprotocol.ClientHeader, "not an address")
}}, }},
{"client empty", func(req *http.Request) { {"client empty", func(req *http.Request) {
req.Header.Set(waf.ClientHeader, "") req.Header.Set(wafprotocol.ClientHeader, "")
}}, }},
{"client address with a zone", func(req *http.Request) { {"client address with a zone", func(req *http.Request) {
req.Header.Set(waf.ClientHeader, "2001:db8::1%eth0") req.Header.Set(wafprotocol.ClientHeader, "2001:db8::1%eth0")
}}, }},
{"id absent", func(req *http.Request) { {"id absent", func(req *http.Request) {
req.Header.Del(requestIDHeader) req.Header.Del(requestIDHeader)
@@ -390,7 +409,12 @@ func TestLongestValidIDIsTaken(t *testing.T) {
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Set(requestIDHeader, strings.Repeat("x", 128)) req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
} }
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) { func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
@@ -399,16 +423,18 @@ func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
s := startWAF(t, "", nil) s := startWAF(t, "", nil)
over := strings.NewReader( over := strings.NewReader(
strings.Repeat("x", waf.QuestionBodyMaxBytes+1), strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes+1),
) )
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over)) got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
wantDisposition(t, got, http.StatusBadRequest, "") wantDisposition(t, got, http.StatusBadRequest, "")
wantProcessMessages(t, s, "a question is refused", 1) wantProcessMessages(t, s, "a question is refused", 1)
wantNoRequestLines(t, s) wantNoRequestLines(t, s)
whole := strings.NewReader(strings.Repeat("x", waf.QuestionBodyMaxBytes)) whole := strings.NewReader(
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes),
)
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole)) got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
} }
func TestQuestionWithAHeldIDIsRefused(t *testing.T) { func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
@@ -424,12 +450,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
wantProcessMessages(t, s, "a question is refused", 1) wantProcessMessages(t, s, "a question is refused", 1)
// A denied request's id is held as well. // A denied request's id is held as well.
denied := s.question(siteA, otherClient, waf.HealthPath) denied := s.question(siteA, otherClient, wafprotocol.HealthPath)
wantDisposition( wantDisposition(
t, t,
do(t, denied), do(t, denied),
http.StatusNotFound, http.StatusNotFound,
waf.DispositionDeny, wafprotocol.DispositionDeny,
) )
again = s.question(siteA, client, "/") again = s.question(siteA, client, "/")
@@ -437,7 +463,9 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
wantDisposition(t, do(t, again), http.StatusBadRequest, "") wantDisposition(t, do(t, again), http.StatusBadRequest, "")
// The report ends the holding, and the refused questions changed nothing. // The report ends the holding, and the refused questions changed nothing.
got := s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes}) got := s.report(
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
if history := s.history(); history.ResponseBytes != answerBytes { if history := s.history(); history.ResponseBytes != answerBytes {
@@ -446,7 +474,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
again = s.question(siteA, client, "/") again = s.question(siteA, client, "/")
again.Header.Set(requestIDHeader, id) again.Header.Set(requestIDHeader, id)
wantDisposition(t, do(t, again), http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
do(t, again),
http.StatusOK,
wafprotocol.DispositionAllow,
)
} }
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) { func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
@@ -481,7 +514,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
// A report for a question not yet answered is ignored, and that question's // A report for a question not yet answered is ignored, and that question's
// id is taken. // id is taken.
got := s.report(waf.Report{RequestID: id, ResponseBytes: 1000}) got := s.report(wafprotocol.Report{RequestID: id, ResponseBytes: 1000})
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
// This would wait on the same lookup were the id not taken. // This would wait on the same lookup were the id not taken.
@@ -496,7 +529,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
select { select {
case got := <-answered: case got := <-answered:
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
case <-time.After(waitLimit): case <-time.After(waitLimit):
t.Fatal("the question got no answer") t.Fatal("the question got no answer")
} }
@@ -515,7 +548,9 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
} }
// The true report comes after, and is the one that counts. // The true report comes after, and is the one that counts.
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes}) got = s.report(
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
if history := s.history(); history.ResponseBytes != answerBytes { if history := s.history(); history.ResponseBytes != answerBytes {
@@ -533,17 +568,17 @@ func TestPeerAbsentOrUnreadableIsAnsweredAndLoggedInvalid(t *testing.T) {
s := startWAF(t, "", nil) s := startWAF(t, "", nil)
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Del(waf.PeerHeader) req.Header.Del(wafprotocol.PeerHeader)
if peer != "" { if peer != "" {
req.Header.Set(waf.PeerHeader, peer) req.Header.Set(wafprotocol.PeerHeader, peer)
} }
wantDisposition( wantDisposition(
t, t,
do(t, req), do(t, req),
http.StatusOK, http.StatusOK,
waf.DispositionAllow, wafprotocol.DispositionAllow,
) )
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" { if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
@@ -571,12 +606,12 @@ func TestContentLengthUnreadableOrNegativeCountsAsAbsent(t *testing.T) {
}) })
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Set(waf.ContentLengthHeader, announced) req.Header.Set(wafprotocol.ContentLengthHeader, announced)
wantDisposition( wantDisposition(
t, t,
do(t, req), do(t, req),
http.StatusOK, http.StatusOK,
waf.DispositionAllow, wafprotocol.DispositionAllow,
) )
if line := s.out.requestLine(t); line.ContentLength != 0 { if line := s.out.requestLine(t); line.ContentLength != 0 {
@@ -595,13 +630,18 @@ func TestHeaderSentTwiceUsesItsFirstValue(t *testing.T) {
s := startWAF(t, "", nil) s := startWAF(t, "", nil)
req := s.question(siteA, client, "/") req := s.question(siteA, client, "/")
req.Header.Add(waf.ClientHeader, otherClient) req.Header.Add(wafprotocol.ClientHeader, otherClient)
req.Header.Add(waf.PeerHeader, "172.18.0.3") req.Header.Add(wafprotocol.PeerHeader, "172.18.0.3")
req.Header.Set(requestIDHeader, "first") req.Header.Set(requestIDHeader, "first")
req.Header.Add(requestIDHeader, "second") req.Header.Add(requestIDHeader, "second")
req.Header.Set(waf.ContentLengthHeader, "1234") req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
req.Header.Add(waf.ContentLengthHeader, "5678") req.Header.Add(wafprotocol.ContentLengthHeader, "5678")
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
line := s.out.requestLine(t) line := s.out.requestLine(t)
if line.ClientIP != client || line.PeerIP != rproxyPeer || if line.ClientIP != client || line.PeerIP != rproxyPeer ||
@@ -630,7 +670,7 @@ func sendQuestionWithPartOfItsBody(t *testing.T, conn net.Conn) {
_, err := fmt.Fprintf(conn, _, err := fmt.Fprintf(conn,
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+ "POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
"Content-Length: 100\r\n\r\npart", "Content-Length: 100\r\n\r\npart",
siteA, waf.ClientHeader, client, requestIDHeader) siteA, wafprotocol.ClientHeader, client, requestIDHeader)
if err != nil { if err != nil {
t.Fatalf("send the question: %v", err) t.Fatalf("send the question: %v", err)
} }
@@ -699,7 +739,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
t, t,
do(t, again), do(t, again),
http.StatusOK, http.StatusOK,
waf.DispositionAllow, wafprotocol.DispositionAllow,
) )
}) })
} }
@@ -717,7 +757,7 @@ func TestQuestionDeniedGetsTheBanResponse(t *testing.T) {
}) })
got := s.ask(client, "/") got := s.ask(client, "/")
wantDisposition(t, got, status, waf.DispositionDeny) wantDisposition(t, got, status, wafprotocol.DispositionDeny)
wantBody(t, got, http.StatusText(status)+"\n") wantBody(t, got, http.StatusText(status)+"\n")
wantLine( wantLine(
t, t,
@@ -737,7 +777,7 @@ func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) {
}) })
got := s.ask(client, "/") got := s.ask(client, "/")
wantDisposition(t, got, http.StatusOK, waf.DispositionClose) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionClose)
wantBody(t, got, "") wantBody(t, got, "")
// The visitor is sent nothing. // The visitor is sent nothing.
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied) wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
@@ -761,12 +801,22 @@ func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse(
}) })
got := s.ask(client, "/blocked") got := s.ask(client, "/blocked")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
wantLine(t, s.out.requestLine(t), http.StatusForbidden, wantLine(t, s.out.requestLine(t), http.StatusForbidden,
requestlog.ActionBlockedByBlockRule) requestlog.ActionBlockedByBlockRule)
got = s.ask(client, "/.env") got = s.ask(client, "/.env")
wantDisposition(t, got, http.StatusTooManyRequests, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusTooManyRequests,
wafprotocol.DispositionDeny,
)
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests, wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
requestlog.ActionTemporarilyBanned) requestlog.ActionTemporarilyBanned)
} }
@@ -781,10 +831,15 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
}) })
for _, path := range []string{ for _, path := range []string{
waf.HealthPath, waf.MetricsPath, waf.ReportPath, wafprotocol.HealthPath, waf.MetricsPath, wafprotocol.ReportPath,
} { } {
got := s.ask(client, path) got := s.ask(client, path)
wantDisposition(t, got, http.StatusNotFound, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusNotFound,
wafprotocol.DispositionDeny,
)
} }
wantLine( wantLine(
@@ -795,7 +850,12 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
) )
got := s.ask(otherClient, waf.MetricsPath) got := s.ask(otherClient, waf.MetricsPath)
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
} }
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) { func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
@@ -804,7 +864,7 @@ func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client}) s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
got := s.ask(client, "/") got := s.ask(client, "/")
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
line := s.out.requestLine(t) line := s.out.requestLine(t)
wantLine(t, line, http.StatusOK, requestlog.ActionForward) wantLine(t, line, http.StatusOK, requestlog.ActionForward)
@@ -822,7 +882,7 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheRefusalLimit(
for range 2 { for range 2 {
got := s.ask(client, "/blocked") got := s.ask(client, "/blocked")
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
} }
lines := s.out.requestLines(t, 2) lines := s.out.requestLines(t, 2)
@@ -846,7 +906,12 @@ func TestDeniedQuestionCountsForTheRefusalLimit(t *testing.T) {
for range 2 { for range 2 {
got := s.ask(client, "/blocked") got := s.ask(client, "/blocked")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
} }
lines := s.out.requestLines(t, 2) lines := s.out.requestLines(t, 2)
@@ -882,7 +947,12 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
}) })
got := s.ask(fromDE, "/") got := s.ask(fromDE, "/")
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
got,
http.StatusOK,
wafprotocol.DispositionAllow,
)
asn, country := got.header.Get( asn, country := got.header.Get(
"X-Client-Asn", "X-Client-Asn",
@@ -916,7 +986,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
} }
// 100 bytes are over the limit of 99. // 100 bytes are over the limit of 99.
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusBadGateway, RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes, RequestBytes: bodyBytes, ResponseBytes: answerBytes,
}) })
@@ -940,7 +1010,12 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
} }
got = s.ask(client, "/") got = s.ask(client, "/")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
} }
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) { func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
@@ -958,12 +1033,12 @@ func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
t, t,
s.ask(fromKP, "/"), s.ask(fromKP, "/"),
http.StatusOK, http.StatusOK,
waf.DispositionAllow, wafprotocol.DispositionAllow,
) )
s.passedOn(fromDE) s.passedOn(fromDE)
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden, wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
waf.DispositionDeny) wafprotocol.DispositionDeny)
} }
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters( func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
@@ -989,7 +1064,7 @@ func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
) )
} }
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusBadGateway, RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes, RequestBytes: bodyBytes, ResponseBytes: answerBytes,
}) })
@@ -1021,7 +1096,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
t, t,
do(t, denied), do(t, denied),
http.StatusForbidden, http.StatusForbidden,
waf.DispositionDeny, wafprotocol.DispositionDeny,
) )
counters := s.server.Anomalies.Snapshot() counters := s.server.Anomalies.Snapshot()
@@ -1039,7 +1114,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
wantMetric(t, s.metrics(), series, 1) wantMetric(t, s.metrics(), series, 1)
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: denied.Header.Get( RequestID: denied.Header.Get(
requestIDHeader, requestIDHeader,
), Status: http.StatusForbidden, ), Status: http.StatusForbidden,
@@ -1082,7 +1157,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
1, 1,
) // the scrape itself ) // the scrape itself
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusBadGateway, RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes, RequestBytes: bodyBytes, ResponseBytes: answerBytes,
DurationTotalMS: 5, DurationUpstreamTotalMS: 4, DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
@@ -1100,7 +1175,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
// check needs a server that has not been scraped yet. // check needs a server that has not been scraped yet.
unscraped := startWAF(t, "", map[string]string{metricsToken: token}) unscraped := startWAF(t, "", map[string]string{metricsToken: token})
got = unscraped.report(waf.Report{ got = unscraped.report(wafprotocol.Report{
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway, RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
DurationTotalMS: 5, DurationTotalMS: 5,
}) })
@@ -1118,7 +1193,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: "no-such-request", ResponseBytes: 1000, RequestID: "no-such-request", ResponseBytes: 1000,
}) })
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
@@ -1137,7 +1212,9 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
// A second report of the same request does not count it twice. // A second report of the same request does not count it twice.
id := s.allowed(client) id := s.allowed(client)
for range 2 { for range 2 {
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes}) got = s.report(
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
} }
@@ -1347,7 +1424,7 @@ func TestBytesAtTheirMostEachWayBanRatherThanOverflow(t *testing.T) {
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: s.allowed(client), Status: http.StatusOK, RequestID: s.allowed(client), Status: http.StatusOK,
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit, RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
}) })
@@ -1441,9 +1518,14 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields(
req.Header.Set("X-Forwarded-Proto", "https") req.Header.Set("X-Forwarded-Proto", "https")
id := req.Header.Get(requestIDHeader) id := req.Header.Get(requestIDHeader)
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusCreated, RequestID: id, Status: http.StatusCreated,
RequestBytes: bodyBytes, ResponseBytes: answerBytes, RequestBytes: bodyBytes, ResponseBytes: answerBytes,
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25, DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
@@ -1490,7 +1572,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/", "close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
0, requestlog.ActionClientDenied, 0, requestlog.ActionClientDenied,
}, },
{"own prefix", nil, waf.HealthPath, http.StatusNotFound, {"own prefix", nil, wafprotocol.HealthPath, http.StatusNotFound,
requestlog.ActionOwnEndpoint}, requestlog.ActionOwnEndpoint},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
@@ -1502,7 +1584,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
id := req.Header.Get(requestIDHeader) id := req.Header.Get(requestIDHeader)
do(t, req) do(t, req)
got := s.report(waf.Report{ got := s.report(wafprotocol.Report{
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3, RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
}) })
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
@@ -1542,13 +1624,13 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
t, t,
readAnswer(res), readAnswer(res),
http.StatusOK, http.StatusOK,
waf.DispositionAllow, wafprotocol.DispositionAllow,
) )
} }
// The table is full again with next and those, without oldest. // The table is full again with next and those, without oldest.
got := s.report( got := s.report(
waf.Report{RequestID: oldest, ResponseBytes: answerBytes}, wafprotocol.Report{RequestID: oldest, ResponseBytes: answerBytes},
) )
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
@@ -1559,7 +1641,9 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
) )
} }
got = s.report(waf.Report{RequestID: next, ResponseBytes: answerBytes}) got = s.report(
wafprotocol.Report{RequestID: next, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent) wantStatus(t, got, http.StatusNoContent)
if counted := s.history().Requests; counted != 1 { if counted := s.history().Requests; counted != 1 {
@@ -1580,10 +1664,20 @@ func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) {
) )
got := do(t, s.question(siteA, client, "/.env")) got := do(t, s.question(siteA, client, "/.env"))
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
got = do(t, s.question(siteB, client, "/")) got = do(t, s.question(siteB, client, "/"))
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
lines := s.out.requestLines(t, 2) lines := s.out.requestLines(t, 2)
if lines[0].Host != siteA || lines[1].Host != siteB { if lines[0].Host != siteA || lines[1].Host != siteB {
@@ -1621,7 +1715,12 @@ func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) {
s.allowed(client) s.allowed(client)
got := do(t, s.question(siteB, client, "/")) got := do(t, s.question(siteB, client, "/"))
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
} }
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) { func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
@@ -1636,13 +1735,13 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
path, authorization string path, authorization string
want int want int
}{ }{
{waf.HealthPath, "", http.StatusOK}, {wafprotocol.HealthPath, "", http.StatusOK},
{waf.MetricsPath, bearer, http.StatusOK}, {waf.MetricsPath, bearer, http.StatusOK},
{waf.MetricsPath, "", http.StatusUnauthorized}, {waf.MetricsPath, "", http.StatusUnauthorized},
{waf.BansPath, adminBearer, http.StatusOK}, {waf.BansPath, adminBearer, http.StatusOK},
{waf.BansPath, "", http.StatusUnauthorized}, {waf.BansPath, "", http.StatusUnauthorized},
{waf.ClientsPath + client, adminBearer, http.StatusOK}, {waf.ClientsPath + client, adminBearer, http.StatusOK},
{waf.ReportPath, "", http.StatusNotFound}, {wafprotocol.ReportPath, "", http.StatusNotFound},
{"/_sww/other", "", http.StatusNotFound}, {"/_sww/other", "", http.StatusNotFound},
{"/", "", http.StatusNotFound}, {"/", "", http.StatusNotFound},
} { } {
@@ -1659,10 +1758,10 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
t, t,
do(t, req), do(t, req),
http.StatusNotFound, http.StatusNotFound,
waf.DispositionDeny, wafprotocol.DispositionDeny,
) )
} else { } else {
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) wantDisposition(t, do(t, req), http.StatusOK, wafprotocol.DispositionAllow)
} }
} }
} }
@@ -1672,11 +1771,11 @@ func TestOwnEndpointsLinesDoNotSayWhichOfTheTwoTheyAre(t *testing.T) {
s := startWAF(t, "", nil) s := startWAF(t, "", nil)
wantStatus(t, get(t, s.addr, waf.HealthPath), http.StatusOK) wantStatus(t, get(t, s.addr, wafprotocol.HealthPath), http.StatusOK)
line := s.out.requestLine(t) line := s.out.requestLine(t)
if _, there := line.fields["waf_line"]; there || if _, there := line.fields["waf_line"]; there ||
line.Path != waf.HealthPath { line.Path != wafprotocol.HealthPath {
t.Errorf("the health check's log line is %+v", line.fields) t.Errorf("the health check's log line is %+v", line.fields)
} }
} }
@@ -1688,7 +1787,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
req := s.questionWith(http.MethodPost, siteA, client, "/", req := s.questionWith(http.MethodPost, siteA, client, "/",
strings.NewReader("a body")) strings.NewReader("a body"))
req.Header.Set(waf.ContentLengthHeader, "1073741824") req.Header.Set(wafprotocol.ContentLengthHeader, "1073741824")
req.Header.Set(forwardedFor, otherClient) req.Header.Set(forwardedFor, otherClient)
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the // 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
@@ -1698,7 +1797,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
} }
got := do(t, req) got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
line := s.out.requestLine(t) line := s.out.requestLine(t)
if line.ClientIP != client || line.ContentLength != 1<<30 { if line.ClientIP != client || line.ContentLength != 1<<30 {
@@ -1776,7 +1875,7 @@ carried header:Content-Length block ^6$
strings.NewReader("a body")) strings.NewReader("a body"))
if tc.announced != "" { if tc.announced != "" {
req.Header.Set(waf.ContentLengthHeader, tc.announced) req.Header.Set(wafprotocol.ContentLengthHeader, tc.announced)
} }
wantStatus(t, do(t, req), tc.want) wantStatus(t, do(t, req), tc.want)
@@ -1794,19 +1893,24 @@ func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) {
req := s.questionWith(http.MethodPost, siteA, client, "/", req := s.questionWith(http.MethodPost, siteA, client, "/",
strings.NewReader(body)) strings.NewReader(body))
req.Header.Set("Content-Type", contentType) req.Header.Set("Content-Type", contentType)
req.Header.Set(waf.ContentLengthHeader, "100000") req.Header.Set(wafprotocol.ContentLengthHeader, "100000")
return do(t, req) return do(t, req)
} }
got := ask(formData, "id=1'%20OR%20'1'='1") got := ask(formData, "id=1'%20OR%20'1'='1")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
// The first 8 KiB of a longer JSON body cannot be parsed, which is no // The first 8 KiB of a longer JSON body cannot be parsed, which is no
// attack, so the Core Rule Set does not read it. // attack, so the Core Rule Set does not read it.
cut := `{"text":"` + strings.Repeat("x", 8<<10) cut := `{"text":"` + strings.Repeat("x", 8<<10)
got = ask("application/json", cut) got = ask("application/json", cut)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
} }
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) { func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
@@ -1815,7 +1919,12 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
s := startWAF(t, "", map[string]string{coreRuleSetMode: block}) s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
got := s.ask(client, sqlInjection) got := s.ask(client, sqlInjection)
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook] waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || if len(waiting) != 1 ||
@@ -1836,7 +1945,7 @@ func TestReputationHitAlertNamesTheSite(t *testing.T) {
loadLists(t, s.server, map[string][]string{dropURL: {client}}) loadLists(t, s.server, map[string][]string{dropURL: {client}})
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden, wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
waf.DispositionDeny) wafprotocol.DispositionDeny)
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook] waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit || if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
+46
View File
@@ -0,0 +1,46 @@
package wafprotocol
import "net/http"
// Answer is what waf's answer to a question tells rproxy in its headers.
// The status, the other headers and the body of an answer of deny are what
// the visitor gets.
type Answer struct {
// Disposition is what rproxy does with the request: DispositionAllow,
// DispositionDeny or DispositionClose.
Disposition string
// ASN and Country are the client's AS number and country, each empty
// when the answer does not give it.
ASN, Country string
}
// SetAnswerHeaders sets the headers of waf's answer that tell rproxy answer,
// leaving out an ASN or Country that is empty.
func SetAnswerHeaders(header http.Header, answer Answer) {
header.Set(DispositionHeader, answer.Disposition)
if answer.ASN != "" {
header.Set(ASNHeader, answer.ASN)
}
if answer.Country != "" {
header.Set(CountryHeader, answer.Country)
}
}
// ReadAnswer returns what the headers of waf's answer tell rproxy, and
// whether they give a valid disposition.
func ReadAnswer(header http.Header) (Answer, bool) {
disposition := header.Get(DispositionHeader)
switch disposition {
case DispositionAllow, DispositionDeny, DispositionClose:
return Answer{
Disposition: disposition,
ASN: header.Get(ASNHeader),
Country: header.Get(CountryHeader),
}, true
}
return Answer{}, false
}
+189
View File
@@ -0,0 +1,189 @@
package wafprotocol
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/netip"
"net/url"
"strconv"
"unicode/utf8"
)
// maxRequestIDBytes is the longest id a question may give.
const maxRequestIDBytes = 128
// ErrBodyStoppedArriving is the error ReadQuestion wraps when the question's
// body stops arriving. Every other error it returns is why waf refuses the
// question.
var ErrBodyStoppedArriving = errors.New("the body stopped arriving")
// Why waf refuses a question. Each is also the body of the refusal.
var (
errClientNotAnAddress = errors.New(
ClientHeader + " is not an address without a zone",
)
errRequestIDNotValid = errors.New(RequestIDHeader + " is not a valid id")
errBodyTooLarge = errors.New("the body is over 64 KiB")
)
// QuestionParams are what NewQuestion needs.
type QuestionParams struct {
// Visitor is the visitor's request.
Visitor *http.Request
// PathAsSent is Visitor's path as the visitor sent it, which the question
// sends unchanged.
PathAsSent string
// Body is the part of Visitor's body the question carries, at most
// QuestionBodyMaxBytes.
Body []byte
// WAFBackend is waf's address. The question takes its scheme and host.
WAFBackend *url.URL
// Client is the visitor's address, and Peer that of rproxy's own TCP
// peer.
Client, Peer netip.Addr
// RequestID is the request's id.
RequestID string
// Scheme is how the visitor reached rproxy, or the trusted proxy in front
// of it.
Scheme string
}
// Question is a question as waf reads it.
type Question struct {
// Client is the visitor's address.
Client netip.Addr
// Peer is the address of rproxy's own TCP peer. It is not valid when the
// question gives none that can be read.
Peer netip.Addr
// RequestID is the request's id.
RequestID string
// Visitor is the visitor's request, as asTheVisitorSentIt gives it.
Visitor *http.Request
}
// NewQuestion returns the question about a visitor's request, as "The
// question" in SPEC.md gives it: the request as the visitor sent it, with the
// part of the body it carries and rproxy's headers.
func NewQuestion(
ctx context.Context,
params QuestionParams,
) (*http.Request, error) {
in := params.Visitor
question, err := http.NewRequestWithContext(
ctx,
in.Method,
params.WAFBackend.String(),
bytes.NewReader(params.Body),
)
if err != nil {
return nil, fmt.Errorf("make the question: %w", err)
}
backend := question.URL
target := *in.URL
target.Scheme = backend.Scheme
target.Host = backend.Host
target.Opaque = params.PathAsSent
question.URL = &target
question.Host = in.Host
// The length NewRequestWithContext set is that of the body carried. This
// line makes a body whose length the visitor did not announce go chunked,
// as "The question" in SPEC.md requires.
question.TransferEncoding = in.TransferEncoding
question.Header = in.Header.Clone()
question.Header.Set(ClientHeader, params.Client.String())
question.Header.Set(PeerHeader, params.Peer.String())
question.Header.Del(ContentLengthHeader)
announced := in.Header.Get("Content-Length")
if announced != "" {
question.Header.Set(ContentLengthHeader, announced)
}
question.Header.Set(RequestIDHeader, params.RequestID)
question.Header.Set("X-Forwarded-Proto", params.Scheme)
// Go's client adds a User-Agent of its own to a request without the key.
_, sent := question.Header["User-Agent"]
if !sent {
question.Header["User-Agent"] = nil
}
return question, nil
}
// ReadQuestion reads the question r, with the body it carries. An error
// wrapping ErrBodyStoppedArriving means the body stopped arriving. Any other
// error is why waf refuses the question, and the question's body is then not
// read. The body is read whole, since Go's server does not send an answer
// until it has read the body.
func ReadQuestion(w http.ResponseWriter, r *http.Request) (Question, error) {
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
if err != nil || client.Zone() != "" {
return Question{}, errClientNotAnAddress
}
id := r.Header.Get(RequestIDHeader)
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
return Question{}, errRequestIDNotValid
}
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
)
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
return Question{}, errBodyTooLarge
}
if err != nil {
return Question{}, fmt.Errorf("%w: %w", ErrBodyStoppedArriving, err)
}
// One that cannot be read leaves the log line's peer_ip invalid.
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
return Question{
Client: client.Unmap(),
Peer: peer.Unmap(),
RequestID: id,
Visitor: asTheVisitorSentIt(r, body),
}, nil
}
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
// log line are to see it: without the headers rproxy adds for waf,
// with body, which has been read of r, and with the length the visitor
// announced, as -1 when it announced none. The question's own length covers
// only the part of the body it carries.
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
sent := r.Clone(r.Context())
announced := sent.Header.Get(ContentLengthHeader)
sent.Header.Del(ClientHeader)
sent.Header.Del(PeerHeader)
sent.Header.Del(ContentLengthHeader)
sent.Header.Del("Content-Length")
sent.ContentLength = -1
length, err := strconv.ParseInt(announced, 10, 64)
if err == nil && length >= 0 {
sent.ContentLength = length
sent.Header.Set("Content-Length", announced)
}
sent.Body = http.NoBody
if len(body) > 0 {
sent.Body = io.NopCloser(bytes.NewReader(body))
}
return sent
}
+123
View File
@@ -0,0 +1,123 @@
package wafprotocol
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
)
const (
// reportBodyMaxBytes is the most of a report's body that is read. It is
// far above any report's.
reportBodyMaxBytes = 1 << 20
// maxReportedBytes is the most bytes a report may give for each way, 1
// PiB, so that adding them up cannot overflow.
maxReportedBytes = 1 << 50
// maxReportedMilliseconds is the longest duration a report may give,
// about 31 years, so that it fits a time.Duration.
maxReportedMilliseconds = 1e12
)
var (
errReportFieldMissing = errors.New("a field is missing")
errReportValue = errors.New("a value is out of range")
)
// Report is the JSON object rproxy posts to ReportPath once the visitor's
// answer has ended, its fields named as the request log's of the same
// meaning. RequestID is the question's id. Status is what the visitor got, 0
// if nothing. The durations are in milliseconds.
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Report struct {
RequestID string `json:"request_id"`
Status int `json:"status"`
RequestBytes int64 `json:"request_bytes"`
ResponseBytes int64 `json:"response_bytes"`
DurationTotalMS float64 `json:"duration_total_ms"`
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
}
// NewReportRequest returns the request that posts report to reportURL, which
// is waf's ReportPath.
func NewReportRequest(
ctx context.Context,
reportURL string,
report *Report,
) (*http.Request, error) {
body, err := json.Marshal(report)
if err != nil {
return nil, fmt.Errorf("encode the report: %w", err)
}
req, err := http.NewRequestWithContext(
ctx, http.MethodPost, reportURL, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("make the request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
return req, nil
}
// ReadReport reads a report from r's body, which must be one JSON object with
// all of Report's fields, in the ranges SPEC.md gives.
func ReadReport(w http.ResponseWriter, r *http.Request) (Report, error) {
var report Report
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
)
if err != nil {
return report, fmt.Errorf("read the body: %w", err)
}
var fields map[string]json.RawMessage
err = json.Unmarshal(body, &fields)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
for _, name := range []string{
"request_id", "status", "request_bytes", "response_bytes",
"duration_total_ms", "duration_upstream_total_ms",
} {
value, there := fields[name]
if !there || string(value) == "null" {
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
}
}
err = json.Unmarshal(body, &report)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
return report, report.checkRanges()
}
// checkRanges returns an error for a value of the report that cannot be true.
func (report *Report) checkRanges() error {
switch {
case report.Status != 0 &&
(report.Status < http.StatusContinue || report.Status > ReportStatusMax):
return fmt.Errorf("%w: status", errReportValue)
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
return fmt.Errorf("%w: request_bytes", errReportValue)
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
return fmt.Errorf("%w: response_bytes", errReportValue)
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_total_ms", errReportValue)
case report.DurationUpstreamTotalMS < 0 ||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
}
return nil
}
+61
View File
@@ -0,0 +1,61 @@
// Package wafprotocol holds the three messages rproxy and waf send each
// other, as "`waf`" in SPEC.md gives them. rproxy asks waf a question about
// each visitor's request, and waf answers it. Once the visitor's answer has
// ended, rproxy sends waf a report. This package writes and reads each
// message, and declares every name and limit both sides must agree on.
package wafprotocol
// The headers of the question and of the answer. Go writes every header name
// in this form, as it sends it and as it receives it, so X-Client-ASN arrives
// as X-Client-Asn; header names are not case-sensitive.
const (
// ClientHeader marks a request as rproxy's question, and gives the
// visitor's address.
ClientHeader = "X-Sww-Client"
// PeerHeader gives the address of rproxy's own TCP peer.
PeerHeader = "X-Sww-Peer"
// ContentLengthHeader gives the body length the visitor announced.
ContentLengthHeader = "X-Sww-Content-Length"
// RequestIDHeader gives the request's id, which rproxy makes. rproxy
// also sets it on the request it passes to the app.
RequestIDHeader = "X-Request-ID"
// DispositionHeader gives waf's answer.
DispositionHeader = "X-Sww-Disposition"
// ASNHeader and CountryHeader give the client's AS number and country in
// an answer of allow. rproxy passes them to the app.
ASNHeader = "X-Client-Asn"
CountryHeader = "X-Client-Country"
)
// The values of DispositionHeader.
const (
// DispositionAllow tells rproxy to pass the request to the app.
DispositionAllow = "allow"
// DispositionDeny tells rproxy to give the visitor the answer's status,
// headers and body.
DispositionDeny = "deny"
// DispositionClose tells rproxy to close the visitor's connection
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
// waf closing the question's connection instead would read as
// waf being down, and rproxy would then let the request through.
DispositionClose = "close"
)
// OwnEndpointPrefix starts the path of every request for sww itself: waf's
// own endpoints and rproxy's health endpoint. The app never sees a path under
// it.
const OwnEndpointPrefix = "/_sww/"
// HealthPath is the health endpoint of waf and of rproxy, which the
// container's health check asks.
const HealthPath = "/_sww/healthz"
// ReportPath is where rproxy sends the report of each request waf
// answered.
const ReportPath = "/_sww/report"
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
const QuestionBodyMaxBytes = 64 << 10
// ReportStatusMax is the highest status a report may give.
const ReportStatusMax = 599
+204
View File
@@ -0,0 +1,204 @@
package wafprotocol_test
import (
"io"
"net/http"
"net/http/httptest"
"net/netip"
"net/url"
"strings"
"testing"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// requestID is the id the tests' questions and reports give.
const requestID = "an-id"
// questionRead is what the server startQuestionReader starts read of a
// question: the question, and the body it carries.
type questionRead struct {
question wafprotocol.Question
body string
}
// startQuestionReader starts a server that reads each question it gets with
// ReadQuestion. It returns the server's URL and the channel what it reads goes
// to.
func startQuestionReader(t *testing.T) (*url.URL, <-chan questionRead) {
t.Helper()
read := make(chan questionRead, 1)
server := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
question, err := wafprotocol.ReadQuestion(w, r)
if err != nil {
t.Errorf("read the question: %v", err)
}
var body []byte
if question.Visitor != nil {
body, _ = io.ReadAll(question.Visitor.Body)
}
read <- questionRead{question, string(body)}
},
))
t.Cleanup(server.Close)
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("parse the server's URL: %v", err)
}
return serverURL, read
}
// TestQuestionReadsAsWritten sends a question NewQuestion writes over HTTP,
// and checks that ReadQuestion gives back the visitor's request and what
// rproxy added.
func TestQuestionReadsAsWritten(t *testing.T) {
t.Parallel()
const (
target = "/a%2Fb?q=1"
body = "name=value"
held = "name"
)
client := netip.MustParseAddr("2001:db8::1")
peer := netip.MustParseAddr("192.0.2.1")
backend, read := startQuestionReader(t)
visitor := httptest.NewRequestWithContext(
t.Context(),
http.MethodPost,
"http://site.example.com"+target,
strings.NewReader(body),
)
visitor.Header.Set("Content-Length", "10")
visitor.Header.Set("User-Agent", "a visitor")
visitor.Header.Set(wafprotocol.PeerHeader, "the visitor's own")
question, err := wafprotocol.NewQuestion(
t.Context(),
wafprotocol.QuestionParams{
Visitor: visitor,
PathAsSent: "/a%2Fb",
Body: []byte(held),
WAFBackend: backend,
Client: client,
Peer: peer,
RequestID: requestID,
Scheme: "https",
},
)
if err != nil {
t.Fatalf("write the question: %v", err)
}
res, err := http.DefaultTransport.RoundTrip(question)
if err != nil {
t.Fatalf("send the question: %v", err)
}
_ = res.Body.Close()
got := <-read
if got.question.Visitor == nil {
t.FailNow()
}
if got.question.Client != client || got.question.Peer != peer ||
got.question.RequestID != requestID {
t.Errorf("read %+v", got.question)
}
sent := got.question.Visitor
if sent.Host != "site.example.com" || sent.RequestURI != target ||
sent.ContentLength != int64(len(body)) || got.body != held {
t.Errorf(
"read %s at %s, of length %d, carrying %q",
sent.RequestURI, sent.Host, sent.ContentLength, got.body,
)
}
for name, want := range map[string]string{
"Content-Length": "10",
"User-Agent": "a visitor",
"X-Forwarded-Proto": "https",
wafprotocol.RequestIDHeader: requestID,
wafprotocol.ClientHeader: "",
wafprotocol.PeerHeader: "",
wafprotocol.ContentLengthHeader: "",
} {
if sent.Header.Get(name) != want {
t.Errorf("%s is %q, want %q", name, sent.Header.Get(name), want)
}
}
}
// TestAnswerReadsAsWritten checks that ReadAnswer gives back what
// SetAnswerHeaders sets, and that headers without a valid disposition give
// nothing.
func TestAnswerReadsAsWritten(t *testing.T) {
t.Parallel()
for _, want := range []wafprotocol.Answer{
{Disposition: wafprotocol.DispositionAllow, ASN: "64496", Country: "DE"},
{Disposition: wafprotocol.DispositionAllow, Country: "DE"},
{Disposition: wafprotocol.DispositionDeny},
{Disposition: wafprotocol.DispositionClose},
} {
header := http.Header{}
wafprotocol.SetAnswerHeaders(header, want)
got, valid := wafprotocol.ReadAnswer(header)
if !valid || got != want {
t.Errorf("read %+v (valid: %t), want %+v", got, valid, want)
}
}
header := http.Header{}
header.Set(wafprotocol.DispositionHeader, "maybe")
header.Set(wafprotocol.ASNHeader, "64496")
got, valid := wafprotocol.ReadAnswer(header)
if valid || got != (wafprotocol.Answer{}) {
t.Errorf("read %+v (valid: %t), want nothing", got, valid)
}
}
// TestReportReadsAsWritten checks that ReadReport gives back the report
// NewReportRequest posts.
func TestReportReadsAsWritten(t *testing.T) {
t.Parallel()
want := wafprotocol.Report{
RequestID: requestID,
Status: http.StatusOK,
RequestBytes: 10,
ResponseBytes: 20,
DurationTotalMS: 1.5,
DurationUpstreamTotalMS: 0.5,
}
req, err := wafprotocol.NewReportRequest(
t.Context(),
"http://waf.example.com"+wafprotocol.ReportPath,
&want,
)
if err != nil {
t.Fatalf("write the report: %v", err)
}
if req.Method != http.MethodPost ||
req.URL.Path != wafprotocol.ReportPath {
t.Errorf("the report is %s %s", req.Method, req.URL.Path)
}
got, err := wafprotocol.ReadReport(httptest.NewRecorder(), req)
if err != nil || got != want {
t.Errorf("read %+v (%v), want %+v", got, err, want)
}
}