Compare commits

..
Author SHA1 Message Date
sneak 298e850e9e Move what rproxy and waf send each other into wafprotocol (closes #425)
check / check (push) Waiting to run
internal/wafprotocol writes and reads the question, the answer and the
report, and declares once each header, path and limit both daemons use.
rproxy no longer imports internal/waf. Nothing on the wire changes.

Model: opus-5-5
2026-11-19 16:46:33 +00:00
50 changed files with 1454 additions and 1136 deletions

No files matched your search

+2
View File
@@ -902,6 +902,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
- `internal/rproxy`: the reverse proxy in front of one app.
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
endpoints.
- `internal/wafprotocol`: the question, the answer and the report that `rproxy`
and `waf` send each other.
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
+7 -4
View File
@@ -73,13 +73,16 @@ never as a time.
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
and stops their background work.
- `internal/config`: reads the settings, the one place they are read.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
`answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes
the log lines.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
and anomalies, and writes the log lines.
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
about each request. It passes a request to the app with the standard
library's `httputil.ReverseProxy`, within the time and size limits.
- `internal/wafprotocol`: the question, the answer and the report that
`rproxy` and `waf` send each other (see "`waf`"). It writes and reads each
one, and declares every header, path and limit both sides use.
`internal/rproxy` imports it, not `internal/waf`.
- `internal/ratelimit`: the table of clients. It counts requests, bytes and
refusals, and keeps each client's history.
- `internal/bans`: the bans. It holds each netblock's bans with their notes,
+4 -8
View File
@@ -54,11 +54,7 @@ func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
now := midnight()
scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made)
@@ -123,7 +119,7 @@ func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(limited, now, bans.Notes{})
// listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
@@ -148,7 +144,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
}
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(limited, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made)
@@ -158,7 +154,7 @@ func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
// The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes.
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
ledger.BanForAttack(attacker, now, bans.Notes{})
want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores {
+23 -64
View File
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
// A new ban drops the ban sww made, and only that one.
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(second, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
}
// And the next drops that one.
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(third, midnight(), bans.Notes{})
wantBans(
t,
ledger,
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
// twice, each time after its last ban has ended.
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
"probes for logins")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
// A ban on another netblock fills the ledger, and the netblock's first
// ban for a limit, its earliest that is not an admin's, goes.
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(other, second.Start, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
// The netblock's next ban still counts it.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
// The netblock breaks a limit twice, the second time after its first
// ban has ended, and the second ban's notes count the first. An admin
// then keeps the first ban.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
kept := first
kept.Cause = bans.CauseAdmin
ledger.LoadEdit([]bans.Ban{kept, second})
// The netblock's next ban counts the kept ban once, as an admin's.
third, _ := ledger.Ban(
netblock,
second.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
t.Errorf("the third ban for a limit has earlier bans %+v, "+
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
ledger := bans.New(defaultParams())
limit, _ := ledger.Ban(
limit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "requests",
Limit: 1000,
Window: "minute",
},
},
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
)
byteLimit, _ := ledger.Ban(
byteLimit, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.3/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
Kind: "bytes",
Limit: 10 << 30,
Window: "hour",
},
},
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
)
attack, _ := ledger.Ban(
attack, _ := ledger.BanForAttack(
netip.MustParsePrefix("203.0.113.2/32"),
midnight(),
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
},
bans.Notes{RuleID: "git-dir", Target: "path"},
)
for _, tc := range []struct{ got, want string }{
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
t.Error("the lifted ban refuses")
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
}
// The next clear sign of attack bans for seven days, as a first does.
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if ban.Expires.Sub(ban.Start) != 7*day {
t.Errorf(
"the next ban for an attack ends at %s, want seven days on",
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
made, _ := ledger.Ban(
made, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.1/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
atStart := bans.Ban{
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
ledger := bans.New(defaultParams())
// An hour's ban for a broken limit.
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A minute later an admin bans the netblock for good, named by an
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
ledger := bans.New(defaultParams())
// The temporary ban is made first, so that its netblock is looked at first.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.BanForAdmin(
netip.MustParsePrefix("203.0.113.0/24"),
+82 -54
View File
@@ -300,50 +300,81 @@ func activeBan(bans []Ban, now time.Time) *Ban {
return newest
}
// Grounds are why a request makes a ban: its cause, CauseLimit or
// CauseAttack, and its notes.
type Grounds struct {
Cause string
Notes Notes
// BanForLimit bans netblock at now for a broken limit, with notes, and
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
// made within LimitBanRepeatWindow after the netblock's ban that ended
// last, other than one for a clear sign of attack or a lifted one, lasts
// repeatFactor times as long as that one. A ban that would be longer
// than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still
// active, as when two of its requests break a limit at once, that ban is
// returned with false, and no other is made. The ledger fills in the
// notes' Refused and EarlierBans itself, and gives the ban the reason
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
// as "requests per minute over the limit of 1000".
func (l *Ledger) BanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
true,
)
}
// Ban bans netblock at now on grounds, and returns the ban, and true. If
// a ban on netblock is still active, as when two of its requests break a
// limit at once, that ban is returned with false, and no other is made.
// The ledger fills in the notes' Refused and EarlierBans itself.
//
// A ban for a broken limit:
// - lasts LimitBanDuration the first time;
// - lasts repeatFactor times as long as the netblock's ban that ended
// last, when made within LimitBanRepeatWindow after it, unless that
// ban was for a clear sign of attack or was lifted;
// - is permanent instead when it would be longer than
// LimitBanPermanentAbove;
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
// the notes, such as "requests per minute over the limit of 1000".
//
// A ban for a clear sign of attack:
// - lasts AttackBanDuration the first time;
// - is permanent once the netblock has had one that was not lifted;
// - has the reason "matched the rule <RuleID>", or "asked for the trap
// path <TrapPath>".
func (l *Ledger) Ban(
netblock netip.Prefix, now time.Time, grounds Grounds,
// WouldBanForLimit returns what BanForLimit would, without making the ban:
// what observe mode would have done.
func (l *Ledger) WouldBanForLimit(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, true)
return l.ban(
netblock,
now,
CauseLimit,
limitReason(notes),
notes,
false,
)
}
// WouldBan returns what Ban would, without making the ban: what observe
// mode would have done.
func (l *Ledger) WouldBan(
netblock netip.Prefix, now time.Time, grounds Grounds,
// BanForAttack bans netblock at now for a clear sign of attack, with
// notes, and returns the ban, and whether it made it, as BanForLimit
// does. A first ban lasts AttackBanDuration; once the netblock has had
// one that was not lifted, the next is permanent. Its reason is "matched
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
func (l *Ledger) BanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(netblock, now, grounds, false)
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
true,
)
}
// WouldBanForAttack returns what BanForAttack would, without making the
// ban: what observe mode would have done.
func (l *Ledger) WouldBanForAttack(
netblock netip.Prefix, now time.Time, notes Notes,
) (Ban, bool) {
return l.ban(
netblock,
now,
CauseAttack,
attackReason(notes),
notes,
false,
)
}
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
// or CauseAttack, made at now would be permanent, as Ban would make it. It
// works out nothing else of the ban.
// or CauseAttack, made at now would be permanent, as BanForLimit or
// BanForAttack would make it. It works out nothing else of the ban.
func (l *Ledger) WouldBePermanent(
netblock netip.Prefix, now time.Time, cause string,
) bool {
@@ -380,9 +411,10 @@ func attackReason(notes Notes) string {
// BanForAdmin bans netblock at now for an admin, with reason, until
// expires, or for good when expires is zero, and returns the ban, whose
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
// on netblock is active, since the admin asked for this one. The ledger
// fills in the notes' EarlierBans, and counts the ban among those made.
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
// ban even while another on netblock is active, since the admin asked
// for this one. The ledger fills in the notes' EarlierBans, and counts
// the ban among those made.
func (l *Ledger) BanForAdmin(
netblock netip.Prefix, now, expires time.Time, reason string,
) Ban {
@@ -727,13 +759,15 @@ func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
return forgot
}
// ban bans netblock at now on grounds, as Ban describes, and returns the
// ban, and whether it made it. Unless keep is true, the ban is not made,
// only returned: it is the ban that would have been made.
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not
// made, only returned: it is the ban that would have been made.
func (l *Ledger) ban(
netblock netip.Prefix,
now time.Time,
grounds Grounds,
cause, reason string,
notes Notes,
keep bool,
) (Ban, bool) {
l.mu.Lock()
@@ -742,8 +776,6 @@ func (l *Ledger) ban(
// held are the netblock's bans, none of them active.
var held []Ban
notes := grounds.Notes
bans, found := l.netblocks.Get(netblock)
if found {
active := activeBan(*bans, now)
@@ -759,19 +791,15 @@ func (l *Ledger) ban(
ban := Ban{
Netblock: netblock,
Start: now,
Cause: grounds.Cause,
Cause: cause,
Reason: reason,
Notes: notes,
}
switch grounds.Cause {
case CauseLimit:
ban.Reason = limitReason(notes)
ban.Expires = l.limitExpiry(held, now)
case CauseAttack:
ban.Reason = attackReason(notes)
if cause == CauseAttack {
ban.Expires = l.attackExpiry(held, now)
default:
panic("bans: a request's ban has the cause " + grounds.Cause)
} else {
ban.Expires = l.limitExpiry(held, now)
}
if !keep {
@@ -779,7 +807,7 @@ func (l *Ledger) ban(
}
l.add(ban)
l.made[grounds.Cause]++
l.made[cause]++
l.markChanged()
return ban, true
+51 -121
View File
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
// 81 hours.
for i, hours := range []int{1, 3, 9, 27, 81} {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
length := time.Duration(hours) * time.Hour
if !ban.Expires.Equal(now.Add(length)) ||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
// The sixth would last 243 hours, more than seven days: it is
// permanent, and never ends.
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
}
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netblock,
first.Expires.Add(tc.gap),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if second.Expires.Sub(second.Start) != tc.want ||
@@ -108,10 +104,10 @@ func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
ledger := bans.New(params)
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if !ban.Permanent() {
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -131,11 +127,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now := midnight()
for i := range 14 {
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Expires.After(ban.Start) {
t.Fatalf(
"ban %d starts at %s and ends at %s",
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
now = ban.Expires
}
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() {
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
}
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, made := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
if !made {
t.Error("the first ban was not made")
}
again, made := ledger.Ban(
again, made := ledger.BanForLimit(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) ||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
)
}
again, made = ledger.Ban(
again, made = ledger.BanForAttack(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if made || !reflect.DeepEqual(again, first) {
t.Errorf(
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
for range 3 {
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
bans.Notes{Requests: 5},
)
got, banned, _ := ledger.Find(
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
d := netip.MustParsePrefix("2001:db8::/64")
now := midnight()
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
ledger.BanForLimit(b, now, bans.Notes{})
ledger.BanForLimit(c, now, bans.Notes{})
// A request from a makes b the netblock seen longest ago, and its ban
// goes to make room for d's.
ledger.Check(a.Addr(), now)
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(d, now, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
// a is banned again once its ban has ended; c, seen longest ago, goes.
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(a, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
// With d seen since, a is seen longest ago, and its earlier ban goes
// first.
ledger.Check(d.Addr(), first.Expires)
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(b, first.Expires, bans.Notes{})
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
ledger := bans.New(params)
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
held := ledger.Bans(netblock)
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
notes := bans.Notes{RuleID: "env-file", Target: "path"}
ban, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
if !ban.Expires.Equal(midnight().Add(7*day)) ||
ban.Cause != bans.CauseAttack ||
ban.Notes.RuleID != "env-file" ||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
netblock := netip.MustParsePrefix("203.0.113.9/32")
// A ban for a broken limit before does not count.
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
second, _ := ledger.Ban(
netblock,
first.Expires,
bans.Grounds{Cause: bans.CauseAttack},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
if second.Expires.Sub(second.Start) != 7*day {
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
// Its notes show the earlier ban for an attack that makes it permanent,
// beside the one for a limit.
third, _ := ledger.Ban(
third, _ := ledger.BanForAttack(
netblock,
second.Expires.Add(30*day),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if !third.Permanent() ||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// While the first ban lasts, none would be made.
during, would := ledger.WouldBan(
during, would := ledger.WouldBanForAttack(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
bans.Notes{},
)
if would || !reflect.DeepEqual(during, first) {
t.Errorf(
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
// As it ends, a clear sign of attack would ban for seven days, and a
// limit broken again for three hours, but neither is made.
limitGrounds := bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
}
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "git-dir"},
})
limit, wouldLimit := ledger.WouldBan(
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
bans.Notes{RuleID: "git-dir"})
limit, wouldLimit := ledger.WouldBanForLimit(
netblock,
first.Expires,
limitGrounds,
limitNotes,
)
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
wantChanged(t, ledger, false)
// The ban made is the one that would have been.
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
if !reflect.DeepEqual(made, limit) {
t.Errorf("the ban made is %+v, want %+v", made, limit)
}
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
t.Fatalf("ban %d for a limit would be permanent", i+1)
}
ban, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
now = ban.Expires
}
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
}
// Once a first ban for an attack has ended, the next would be permanent.
attack, _ := ledger.Ban(
netblock,
now,
bans.Grounds{Cause: bans.CauseAttack},
)
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
t.Error("a second ban for an attack would not be permanent")
}
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
// Three times the seven days would be permanent; a limit broken as the
// ban for an attack ends bans for an hour, as a first broken limit does.
attack, _ := ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseAttack},
)
limit, _ := ledger.Ban(
netblock,
attack.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
if limit.Expires.Sub(limit.Start) != time.Hour ||
limit.Cause != bans.CauseLimit {
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
}
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
midnight(),
bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Request: request},
},
bans.Notes{Request: request},
)
cut := long[:256]
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
// A ban made with the client's lookup, one made before it came, after
// the first ended, and one on another netblock.
ledger.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{
ASN: "AS64497",
ASName: "Other Net",
Country: "FR",
},
ledger.BanForLimit(netblock, midnight(), bans.Notes{
ASN: "AS64497", ASName: "Other Net", Country: "FR",
})
ledger.Ban(
netblock,
midnight().Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
ledger.BanForLimit(other, midnight(), bans.Notes{})
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
+21 -36
View File
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
wantChanged(t, ledger, false)
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
wantChanged(t, ledger, true)
// A limit broken during the ban makes no other, and a refusal changes
// only the counts in the notes, which wait for the interval's write.
ledger.Ban(
netblock,
midnight().Add(time.Minute),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
wantChanged(t, ledger, false)
// Two bans before the value is read leave one.
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.11/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
wantChanged(t, ledger, true)
wantChanged(t, ledger, false)
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
high := netip.MustParsePrefix("203.0.113.10/32")
low := netip.MustParsePrefix("203.0.113.9/32")
first, _ := ledger.Ban(
v6,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
ledger.BanForLimit(high, midnight(), bans.Notes{})
ledger.BanForLimit(low, midnight(), bans.Notes{})
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
snapshot := ledger.Snapshot()
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
before := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{Limit: 1},
})
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
// Loaded into a new ledger, as across a restart, the ban still refuses
// while it lasts, and once it has ended a broken limit bans for three
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
t.Error("the loaded ban does not refuse")
}
again, _ := after.Ban(
netblock,
ban.Expires,
bans.Grounds{Cause: bans.CauseLimit},
)
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
if again.Expires.Sub(again.Start) != 3*time.Hour ||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
t.Errorf(
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
Start: midnight(),
},
})
ledger.Ban(
ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
for client, want := range map[string]bool{
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
}
// A limit broken now makes no shorter ban over the permanent one.
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
t.Errorf(
"a broken limit returned %+v and left the netblock %d bans, "+
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
// Once both have ended, a limit broken within the repeat window bans
// for three times the 9 hours, and the notes count the two bans
// before the 9-hour one and it, for a limit, and the admin's.
ban, _ := ledger.Ban(
ban, _ := ledger.BanForLimit(
netblock,
nineHours.Expires.Add(time.Hour),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
}
// The ledger holds one ban, so it makes two more without dropping any.
first, _ := ledger.Ban(
first, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.7/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
second, _ := ledger.Ban(
second, _ := ledger.BanForLimit(
netip.MustParsePrefix("198.51.100.8/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
want := []bans.Ban{first, second, kept}
+32 -93
View File
@@ -75,7 +75,7 @@ type WAF struct {
BytesLimitPerMinute int64
BytesLimitPerHour int64
BytesLimitPerDay int64
BytesCounted BytesCounted
BytesCounted string
// LookupSource is where each client's AS number and country are looked
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
@@ -86,7 +86,7 @@ type WAF struct {
// AddLookupHeaders is true when the app is passed the client's AS number
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
// LookupSource is off.
LookupSource LookupSource
LookupSource string
LookupDBPath string
ASNAndCountryLookupTimeout time.Duration
AddLookupHeaders bool
@@ -131,7 +131,7 @@ type WAF struct {
// BlocklistLimitPercent is the percentage of every limit it gets.
BlocklistURLs []string
BlocklistRefresh time.Duration
BlocklistAction ReputationAction
BlocklistAction string
BlocklistLimitPercent int64
// DNSBLZones are the DNSBL zones clients are asked about
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
@@ -154,7 +154,7 @@ type WAF struct {
AbuseIPDBKey string
AbuseIPDBMinScore int64
AbuseIPDBDailyBudget int
DNSBLAbuseIPDBAction ReputationAction
DNSBLAbuseIPDBAction string
DNSBLAbuseIPDBLimitPercent int64
IPReputationCacheTTL time.Duration
DNSBLAbuseIPDBTimeout time.Duration
@@ -237,7 +237,7 @@ type WAF struct {
// reads none.
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
CoreRuleSetMode CoreRuleSetMode
CoreRuleSetMode string
CoreRuleSetParanoiaLevel int
CoreRuleSetScoreThreshold int
CoreRuleSetDisabledRules []int
@@ -316,66 +316,20 @@ type WAF struct {
settings []slog.Attr
}
// BytesCounted is which body bytes count toward the byte limits and the
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
type BytesCounted string
// The values of SWW_WAF_BYTES_COUNTED.
const (
// BytesCountedResponse counts the response's body bytes.
BytesCountedResponse BytesCounted = "response"
// BytesCountedRequest counts the request's body bytes.
BytesCountedRequest BytesCounted = "request"
// BytesCountedBoth counts the body bytes of both.
BytesCountedBoth BytesCounted = "both"
)
// LookupSource is where each client's AS number and country are looked up,
// a value of SWW_WAF_LOOKUP_SOURCE.
type LookupSource string
// The values of SWW_WAF_LOOKUP_SOURCE.
const (
// LookupSourceGeoJS asks GeoJS.
LookupSourceGeoJS LookupSource = "geojs"
// LookupSourceFile looks clients up in the lookup database, the file
// SWW_WAF_LOOKUP_DB_PATH names.
LookupSourceFile LookupSource = "file"
// LookupSourceOff looks no client up.
LookupSourceOff LookupSource = "off"
)
// ReputationAction is what is done with a client a reputation source
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
type ReputationAction string
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
const (
// ReputationActionDeny refuses the client's requests.
ReputationActionDeny ReputationAction = "deny"
// ReputationActionLimit gives the client a percentage of every limit,
// written limit:<percent> in the setting.
ReputationActionLimit ReputationAction = "limit"
// ReputationActionLog only notes the hit in the log line and raises an
// alert.
ReputationActionLog ReputationAction = "log"
)
// CoreRuleSetMode is what the Core Rule Set does, a value of
// SWW_WAF_CORE_RULE_SET_MODE.
type CoreRuleSetMode string
// The values of SWW_WAF_CORE_RULE_SET_MODE.
const (
// CoreRuleSetModeOff runs no request through the Core Rule Set.
CoreRuleSetModeOff CoreRuleSetMode = "off"
CoreRuleSetModeOff = off
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
CoreRuleSetModeDetect = "detect"
// CoreRuleSetModeBlock refuses a match with 403.
CoreRuleSetModeBlock CoreRuleSetMode = "block"
CoreRuleSetModeBlock = "block"
)
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
const fileSource = "file"
const (
day = 24 * time.Hour
ipv4Bits = 32
@@ -604,13 +558,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
),
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
BytesCounted: env.bytesCounted(
"SWW_WAF_BYTES_COUNTED",
BytesCountedBoth,
),
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
LookupSource: env.lookupSource(
"SWW_WAF_LOOKUP_SOURCE",
LookupSourceGeoJS,
"geojs",
),
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
ASNAndCountryLookupTimeout: env.milliseconds(
@@ -807,9 +758,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.InstanceName, cfg.LogRemoteURL != nil)
// The app gets the headers by default, unless no client is looked up.
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
strconv.FormatBool(cfg.LookupSource != off))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
"SWW_WAF_BLOCKLIST_ACTION", "deny")
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
@@ -907,12 +858,9 @@ func (e *environment) count(name, defaultValue string) int64 {
// bytesCounted reads the setting that is which body bytes count toward the
// byte limits and the byte anomaly thresholds: response, request or both.
func (e *environment) bytesCounted(
name string, defaultValue BytesCounted,
) BytesCounted {
value := BytesCounted(e.value(name, string(defaultValue)))
if value != BytesCountedResponse && value != BytesCountedRequest &&
value != BytesCountedBoth {
func (e *environment) bytesCounted(name, defaultValue string) string {
value := e.value(name, defaultValue)
if value != "response" && value != "request" && value != "both" {
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
}
@@ -957,10 +905,8 @@ func (e *environment) trapPaths(name string) []string {
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
// detect or block.
func (e *environment) coreRuleSetMode(
name string, defaultValue CoreRuleSetMode,
) CoreRuleSetMode {
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
mode := e.value(name, defaultValue)
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
mode != CoreRuleSetModeBlock {
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
@@ -1110,24 +1056,20 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
// action reads a setting that is what is done with a client a blocklist
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
// as limit and the percentage.
func (e *environment) action(
name, defaultValue string,
) (ReputationAction, int64) {
value := ReputationAction(e.value(name, defaultValue))
if value == ReputationActionDeny || value == ReputationActionLog {
func (e *environment) action(name, defaultValue string) (string, int64) {
value := e.value(name, defaultValue)
if value == "deny" || value == "log" {
return value, 0
}
percentText, isLimit := strings.CutPrefix(
string(value), string(ReputationActionLimit)+":",
)
percentText, isLimit := strings.CutPrefix(value, "limit:")
percent, err := ParsePercent(percentText)
if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
}
return ReputationActionLimit, percent
return "limit", percent
}
// zones reads the setting that is the list of DNSBL zones. It is empty by
@@ -1178,12 +1120,9 @@ func (e *environment) routeWatchURL(name string) string {
// lookupSource reads the setting that is where clients are looked up:
// geojs, file, or off.
func (e *environment) lookupSource(
name string, defaultValue LookupSource,
) LookupSource {
source := LookupSource(e.value(name, string(defaultValue)))
if source != LookupSourceGeoJS && source != LookupSourceFile &&
source != LookupSourceOff {
func (e *environment) lookupSource(name, defaultValue string) string {
source := e.value(name, defaultValue)
if source != "geojs" && source != fileSource && source != off {
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
}
@@ -1195,12 +1134,12 @@ func (e *environment) lookupSource(
// one source at a time.
func (e *environment) checkLookupDBPath(cfg *WAF) {
switch {
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
errNeedsDBPath,
))
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
cfg.LookupSource,
@@ -1225,7 +1164,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
}
}
if cfg.LookupSource != LookupSourceOff {
if cfg.LookupSource != off {
return
}
+5 -6
View File
@@ -614,7 +614,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
},
{
environment{
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
coreRuleSetMode: config.CoreRuleSetModeDetect,
coreRuleSetParanoiaLevel: "4",
coreRuleSetScoreThreshold: "10",
coreRuleSetDisabledRules: "942100, 920350",
@@ -2033,9 +2033,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
}
for _, tc := range []struct {
value string
action config.ReputationAction
percent int64
value, action string
percent int64
}{
{actionDeny, actionDeny, 0},
{actionLog, actionLog, 0},
@@ -2168,7 +2167,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
// want are the zones, resolver, action and percent config.WAF gives.
want []string
wantResolver string
wantAction config.ReputationAction
wantAction string
wantPercent int64
}{
{
@@ -3050,7 +3049,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
logLevel: "info",
ruleFilesDir: "/etc/sww/rules.d",
ruleFilesEnabled: "true",
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
coreRuleSetMode: config.CoreRuleSetModeBlock,
coreRuleSetParanoiaLevel: "1",
coreRuleSetScoreThreshold: "5",
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
-10
View File
@@ -7,16 +7,6 @@ import (
"strings"
)
// The headers rproxy sets on the requests it passes to the app. Go writes every
// header name in this form, as it sends it and as it receives it, so
// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever
// their case; header names are not case-sensitive.
const (
requestIDHeader = "X-Request-ID"
asnHeader = "X-Client-Asn"
countryHeader = "X-Client-Country"
)
// peerAddress is the address of the request's TCP peer, normally traefik, without
// any IPv6 zone, which waf refuses.
func peerAddress(r *http.Request) netip.Addr {
+3 -3
View File
@@ -20,7 +20,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// post is the start of a POST whose headers end with what follows it.
@@ -492,7 +492,7 @@ func TestASettingOfOffSetsNoLimit(t *testing.T) {
strings.NewReader("a body"))),
)
if got := get(t, addr, waf.HealthPath); got.status != http.StatusOK {
if got := get(t, addr, wafprotocol.HealthPath); got.status != http.StatusOK {
t.Errorf("answered %+v, want rproxy's own 200", got)
}
})
@@ -1471,7 +1471,7 @@ func TestAVisitorThatStopsALittleShortGetsAnAnswerRProxyGivesAtOnce(
{"a deny", post, http.StatusForbidden},
{
"the health check",
"GET " + waf.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
"GET " + wafprotocol.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
http.StatusOK,
},
} {
+13 -13
View File
@@ -12,11 +12,11 @@ import (
"strings"
"testing"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// questionBodyMaxBytes is the most of a body a question carries.
const questionBodyMaxBytes = waf.QuestionBodyMaxBytes
const questionBodyMaxBytes = wafprotocol.QuestionBodyMaxBytes
// chunked is the encoding of a body whose length the visitor did not announce.
const chunked = "chunked"
@@ -42,9 +42,9 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
req.Header.Set(forwardedForHeader, "198.51.100.1")
req.Header.Set(forwardedProtoHeader, "gopher")
req.Header.Set(requestIDHeader, "the-visitor's")
req.Header.Set(waf.ClientHeader, "198.51.100.2")
req.Header.Set(waf.PeerHeader, "198.51.100.3")
req.Header.Set(waf.ContentLengthHeader, "12345")
req.Header.Set(wafprotocol.ClientHeader, "198.51.100.2")
req.Header.Set(wafprotocol.PeerHeader, "198.51.100.3")
req.Header.Set(wafprotocol.ContentLengthHeader, "12345")
// Go's client then sends no User-Agent, so that one in the question would be
// Go's own.
req.Header["User-Agent"] = nil
@@ -73,8 +73,8 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
want.Set(forwardedForHeader, "198.51.100.1")
want.Set(forwardedProtoHeader, "http")
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
want.Set(waf.ClientHeader, localhost)
want.Set(waf.PeerHeader, localhost)
want.Set(wafprotocol.ClientHeader, localhost)
want.Set(wafprotocol.PeerHeader, localhost)
if !maps.EqualFunc(asked.header, want, slices.Equal) {
t.Errorf(
@@ -124,8 +124,8 @@ func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) {
wantGreeting(t, visit(t, req))
asked := standIn.question(t)
if asked.header.Get(waf.ClientHeader) != tc.client ||
asked.header.Get(waf.PeerHeader) != localhost ||
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
asked.header.Get(wafprotocol.PeerHeader) != localhost ||
asked.header.Get(forwardedForHeader) != visitor ||
asked.header.Get(forwardedProtoHeader) != tc.proto {
t.Errorf("the question's headers are %v", asked.header)
@@ -191,7 +191,7 @@ func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) {
req.Header[forwardedForHeader] = tc.header
wantGreeting(t, visit(t, req))
if got := standIn.question(t).header.Get(waf.ClientHeader); got !=
if got := standIn.question(t).header.Get(wafprotocol.ClientHeader); got !=
tc.client {
t.Errorf("the question names %s, want %s", got, tc.client)
}
@@ -282,7 +282,7 @@ func wantQuestionBody(
)
}
values := asked.header.Values(waf.ContentLengthHeader)
values := asked.header.Values(wafprotocol.ContentLengthHeader)
if announced {
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
@@ -586,8 +586,8 @@ func TestAQuestionNamesNoZone(t *testing.T) {
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
asked := standIn.question(t)
if asked.header.Get(waf.ClientHeader) != tc.client ||
asked.header.Get(waf.PeerHeader) != tc.peer {
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
asked.header.Get(wafprotocol.PeerHeader) != tc.peer {
t.Errorf("the question's headers are %v", asked.header)
}
+11 -13
View File
@@ -18,17 +18,13 @@ import (
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
// soon as it arrives.
const flushAfterEachWrite time.Duration = -1
// reportStatusMax is the highest status waf takes in a report. An app can
// send a higher one, up to 999, and the visitor gets that as it is.
const reportStatusMax = 599
// plainText is the Content-Type of the answers rproxy gives itself.
const plainText = "text/plain; charset=utf-8"
@@ -154,7 +150,7 @@ func (rq *request) readFirstPartOfBody() bool {
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
held, err := io.ReadAll(
io.LimitReader(rq.body, waf.QuestionBodyMaxBytes),
io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
)
// From here on, the request timer ends a read of the rest of the body, so that
// the answer can name the side rproxy was waiting on.
@@ -228,16 +224,17 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
setPathAsSent(pr.Out.URL, pr.In)
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
pr.Out.Header.Set(requestIDHeader, rq.id)
pr.Out.Header.Del(asnHeader)
pr.Out.Header.Del(countryHeader)
pr.Out.Header.Set(wafprotocol.RequestIDHeader, rq.id)
// Del removes a visitor's own, whatever their case.
pr.Out.Header.Del(wafprotocol.ASNHeader)
pr.Out.Header.Del(wafprotocol.CountryHeader)
if rq.asn != "" {
pr.Out.Header.Set(asnHeader, rq.asn)
pr.Out.Header.Set(wafprotocol.ASNHeader, rq.asn)
}
if rq.country != "" {
pr.Out.Header.Set(countryHeader, rq.country)
pr.Out.Header.Set(wafprotocol.CountryHeader, rq.country)
}
}
@@ -470,9 +467,10 @@ func (rq *request) finish(ctx context.Context) {
return
}
report := &waf.Report{
// An app can send a status up to 999, and the visitor gets it as it is.
report := &wafprotocol.Report{
RequestID: rq.id,
Status: min(rq.out.status, reportStatusMax),
Status: min(rq.out.status, wafprotocol.ReportStatusMax),
RequestBytes: requestBytes,
ResponseBytes: responseBytes,
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
+10 -12
View File
@@ -24,7 +24,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// How rproxy keeps connections to the app and to waf open between
@@ -47,10 +47,6 @@ const (
idleTimeout = 120 * time.Second
)
// ownEndpointPrefix starts the path of every request for sww itself, which
// never reaches the app.
const ownEndpointPrefix = "/_sww/"
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
// them. It answers any other method 405 itself.
// The WebDAV ones are for Nextcloud.
@@ -92,7 +88,9 @@ func New(params Params) *Server {
transport: newTransport(),
wafTransport: newWAFTransport(),
reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
Scheme: backend.Scheme,
Host: backend.Host,
Path: wafprotocol.ReportPath,
}).String(),
}
@@ -176,7 +174,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer rq.readRestOfBody()
defer rq.finish(r.Context())
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath {
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
rq.answerHealth()
return
@@ -209,9 +207,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
rq.noteAnswer(answer)
switch rq.disposition {
case waf.DispositionAllow:
case wafprotocol.DispositionAllow:
rq.passToApp(r.Context())
case waf.DispositionClose:
case wafprotocol.DispositionClose:
// This panic has Go's server close the connection without an answer,
// and log nothing; the deferred finish still runs.
panic(http.ErrAbortHandler)
@@ -221,8 +219,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// passToApp passes the request on to the app, unless a limit or its path keeps
// it from the app. waf answers a path under ownEndpointPrefix itself, so
// such a request gets here only when
// it from the app. waf answers a path under wafprotocol.OwnEndpointPrefix
// itself, so such a request gets here only when
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
// ctx is the request's own context.
func (rq *request) passToApp(ctx context.Context) {
@@ -231,7 +229,7 @@ func (rq *request) passToApp(ctx context.Context) {
switch {
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
rq.answer(http.StatusRequestEntityTooLarge)
case strings.HasPrefix(rq.in.URL.Path, ownEndpointPrefix):
case strings.HasPrefix(rq.in.URL.Path, wafprotocol.OwnEndpointPrefix):
rq.answer(http.StatusNotFound)
default:
rq.forward(ctx)
+18 -18
View File
@@ -23,7 +23,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -56,7 +56,7 @@ const (
// The headers the tests set and look at.
const (
dispositionHeader = waf.DispositionHeader
dispositionHeader = wafprotocol.DispositionHeader
forwardedForHeader = "X-Forwarded-For"
forwardedProtoHeader = "X-Forwarded-Proto"
requestIDHeader = "X-Request-ID"
@@ -72,9 +72,9 @@ const (
// What the stand-in for waf answers.
const (
dispositionAllow = waf.DispositionAllow
dispositionDeny = waf.DispositionDeny
dispositionClose = waf.DispositionClose
dispositionAllow = wafprotocol.DispositionAllow
dispositionDeny = wafprotocol.DispositionDeny
dispositionClose = wafprotocol.DispositionClose
// deniedBody is the body of a deny.
deniedBody = "refused by waf"
)
@@ -175,7 +175,7 @@ type WAF struct {
*httptest.Server
questions chan question
reports chan waf.Report
reports chan wafprotocol.Report
}
// startWAF starts a stand-in for waf.
@@ -184,11 +184,11 @@ func startWAF(t *testing.T, answer http.HandlerFunc) *WAF {
s := &WAF{
questions: make(chan question, 10),
reports: make(chan waf.Report, 10),
reports: make(chan wafprotocol.Report, 10),
}
s.Server = httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) == "" {
if r.Header.Get(wafprotocol.ClientHeader) == "" {
s.takeReport(t, w, r)
return
@@ -217,11 +217,11 @@ func (s *WAF) takeReport(
) {
t.Helper()
var report waf.Report
var report wafprotocol.Report
err := json.NewDecoder(r.Body).Decode(&report)
if err != nil || r.Method != http.MethodPost ||
r.URL.Path != waf.ReportPath {
r.URL.Path != wafprotocol.ReportPath {
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
}
@@ -242,7 +242,7 @@ func (s *WAF) wantNoReport(t *testing.T) {
}
// report waits for a report.
func (s *WAF) report(t *testing.T) waf.Report {
func (s *WAF) report(t *testing.T) wafprotocol.Report {
t.Helper()
select {
@@ -251,7 +251,7 @@ func (s *WAF) report(t *testing.T) waf.Report {
case <-time.After(waitLimit):
t.Fatal("no report")
return waf.Report{}
return wafprotocol.Report{}
}
}
@@ -279,7 +279,7 @@ func startWAFAnsweringReports(
backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" {
if r.Header.Get(wafprotocol.ClientHeader) != "" {
answer(w, r)
return
@@ -811,7 +811,7 @@ func TestAnAnswerRProxyHoldsWholeIsSentBeforeItsLogLine(t *testing.T) {
}{
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
{"the health check", "GET " + waf.HealthPath + host, "ok\n", http.StatusOK},
{"the health check", "GET " + wafprotocol.HealthPath + host, "ok\n", http.StatusOK},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
@@ -1576,19 +1576,19 @@ func TestHealthCheckIsAnsweredWhileWAFIsDown(t *testing.T) {
app := startApp(t, greet)
addr, out := startRProxy(t, nowhere, app.URL, nil)
got := get(t, addr, waf.HealthPath)
got := get(t, addr, wafprotocol.HealthPath)
if got.status != http.StatusOK || got.body != "ok\n" {
t.Errorf("answered %+v, want ok", got)
}
line := out.requestLine(t)
if line["path"] != waf.HealthPath || line["disposition"] != nil {
if line["path"] != wafprotocol.HealthPath || line["disposition"] != nil {
t.Errorf("line %v, want one with no disposition", line)
}
// With waf up too, it is not asked.
addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
get(t, addr, waf.HealthPath)
get(t, addr, wafprotocol.HealthPath)
if len(standIn.questions) != 0 {
t.Error("waf was asked about the health check")
@@ -1749,7 +1749,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
release := make(chan struct{})
backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" {
if r.Header.Get(wafprotocol.ClientHeader) != "" {
allow(w, r)
return
+34 -82
View File
@@ -1,9 +1,7 @@
package rproxy
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -12,7 +10,8 @@ import (
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// answerBodyMaxBytes is the most of the body of waf's answer that is read.
@@ -41,15 +40,12 @@ type wafAnswer struct {
// notes nothing, and no report is due for it: waf holds no decision to report
// on.
func (rq *request) noteAnswer(answer *wafAnswer) {
disposition := answer.header.Get(waf.DispositionHeader)
switch disposition {
case waf.DispositionAllow, waf.DispositionDeny,
waf.DispositionClose:
rq.disposition = disposition
read, valid := wafprotocol.ReadAnswer(answer.header)
if valid {
rq.disposition = read.Disposition
rq.answered = true
rq.asn = answer.header.Get(asnHeader)
rq.country = answer.header.Get(countryHeader)
rq.asn = read.ASN
rq.country = read.Country
}
}
@@ -80,7 +76,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
rq.wafNoAnswer = true
if rq.disposition == waf.DispositionAllow {
if rq.disposition == wafprotocol.DispositionAllow {
rq.passToApp(ctx)
} else {
rq.answer(http.StatusServiceUnavailable)
@@ -120,7 +116,21 @@ func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
defer cancel()
question, err := rq.newQuestion(ctx)
var held []byte
if rq.body != nil {
held = rq.body.held
}
question, err := wafprotocol.NewQuestion(ctx, wafprotocol.QuestionParams{
Visitor: rq.in,
PathAsSent: rules.Path(rq.in),
Body: held,
WAFBackend: rq.h.config.WAFBackend,
Client: rq.client,
Peer: rq.peer,
RequestID: rq.id,
Scheme: scheme(rq.in, rq.peerTrusted),
})
if err != nil {
return nil, err
}
@@ -177,69 +187,13 @@ func canBeAsked(r *http.Request) bool {
len(r.Header.Values("User-Agent")) <= 1
}
// newQuestion returns the question about the request, as "The question" in
// SPEC.md gives it: the visitor's request as the visitor sent it, with the part
// of the body it holds and rproxy's headers.
func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) {
in := rq.in
var held []byte
if rq.body != nil {
held = rq.body.held
}
question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
ctx,
in.Method,
rq.h.config.WAFBackend.String(),
bytes.NewReader(held),
)
if err != nil {
return nil, fmt.Errorf("make the question: %w", err)
}
backend := question.URL
target := *in.URL
target.Scheme = backend.Scheme
target.Host = backend.Host
setPathAsSent(&target, in)
question.URL = &target
question.Host = in.Host
// The length NewRequestWithContext set is that of what is held. This line makes
// a body whose length the visitor did not announce go chunked, as "The question"
// in SPEC.md requires.
question.TransferEncoding = in.TransferEncoding
question.Header = in.Header.Clone()
question.Header.Set(waf.ClientHeader, rq.client.String())
question.Header.Set(waf.PeerHeader, rq.peer.String())
question.Header.Del(waf.ContentLengthHeader)
announced := in.Header.Get("Content-Length")
if announced != "" {
question.Header.Set(waf.ContentLengthHeader, announced)
}
question.Header.Set(requestIDHeader, rq.id)
question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted))
// Go's client adds a User-Agent of its own to a request without the key.
_, sent := question.Header["User-Agent"]
if !sent {
question.Header["User-Agent"] = nil
}
return question, nil
}
// answerWith gives the visitor waf's answer, less its disposition.
func (rq *request) answerWith(answer *wafAnswer) {
rq.prepareOwnAnswer(rq.requestDeadline())
header := rq.out.Header()
maps.Copy(header, answer.header)
header.Del(waf.DispositionHeader)
header.Del(wafprotocol.DispositionHeader)
// Go's server adds a Content-Type it guesses from the body to a response whose
// headers have no Content-Type key. The key with no value sends none.
@@ -253,7 +207,10 @@ func (rq *request) answerWith(answer *wafAnswer) {
// sendReport sends waf report on a goroutine of its own, so that the
// visitor does not wait for it. The report goes on after ctx, the request's own
// context, has ended.
func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
func (h *handler) sendReport(
ctx context.Context,
report *wafprotocol.Report,
) {
h.reportsMu.Lock()
defer h.reportsMu.Unlock()
@@ -271,23 +228,18 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
// postReport posts report to waf, and waits for its answer for up to
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
body, err := json.Marshal(report)
if err != nil {
return fmt.Errorf("encode the report: %w", err)
}
func (h *handler) postReport(
ctx context.Context,
report *wafprotocol.Report,
) error {
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
defer cancel()
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
ctx, http.MethodPost, h.reportURL, bytes.NewReader(body))
req, err := wafprotocol.NewReportRequest(ctx, h.reportURL, report)
if err != nil {
return fmt.Errorf("make the request: %w", err)
return err
}
req.Header.Set("Content-Type", "application/json")
res, err := h.wafTransport.RoundTrip(req)
if err != nil {
return fmt.Errorf("send the report: %w", err)
+18 -30
View File
@@ -1284,18 +1284,18 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
// A second ban, made while the first waits to be written, puts the
// write off no further, and is written with it.
first, _ := params.Ledger.Ban(
first, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5 * time.Second)
second, _ := params.Ledger.Ban(
second, _ := params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.10/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
time.Sleep(5*time.Second - time.Nanosecond)
@@ -1358,16 +1358,16 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
run(t, load(t, params).Run)
// A ban, and bans.json written with it.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay)
synctest.Wait()
// A second ban is to be written BansWriteDelay later. Just before
// then, an admin saves bans.json with the first ban lifted and
// another added.
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Grounds{Cause: bans.CauseLimit})
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
midnight(), bans.Notes{})
time.Sleep(params.BansWriteDelay - time.Nanosecond)
synctest.Wait()
edit(t, dir, bansJSON, permanentBansJSON)
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
params.Limiter.Count(
netip.MustParsePrefix("203.0.113.9/32"),
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
t.Fatalf("mkdir: %v", err)
}
params.Ledger.Ban(
params.Ledger.BanForLimit(
netip.MustParsePrefix("203.0.113.9/32"),
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
err = files.WriteAll()
@@ -2135,11 +2135,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
// The ban that liftedBansJSON lifts, before it is lifted.
netblock := netip.MustParsePrefix(liftedClient + "/32")
params.Ledger.Ban(
netblock,
midnight(),
bans.Grounds{Cause: bans.CauseLimit},
)
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
if !banned {
@@ -2514,15 +2510,11 @@ func fill(params state.Params) {
client := netip.MustParsePrefix("203.0.113.9/32")
params.Ledger.Load([]bans.Ban{permanentBan()})
params.Ledger.Ban(client, now, bans.Grounds{
Cause: bans.CauseLimit,
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
params.Ledger.BanForLimit(client, now, bans.Notes{
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
})
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Grounds{
Cause: bans.CauseAttack,
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
})
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
bans.Notes{RuleID: "env-file", Target: "path"})
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
@@ -2708,11 +2700,7 @@ func wantLiftedBanKept(
}
// Were the lifted ban counted, the next would last three hours.
ban, _ := ledger.Ban(
netblock,
afterLifting(),
bans.Grounds{Cause: bans.CauseLimit},
)
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
if ban.Expires.Sub(ban.Start) != time.Hour {
t.Errorf(
"the next ban lasts %s, want 1h",
+2 -2
View File
@@ -13,7 +13,7 @@ import (
"github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// healthCheckTimeout bounds the whole health check.
@@ -116,7 +116,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
health := "http://" + net.JoinHostPort(
"127.0.0.1",
port,
) + waf.HealthPath
) + wafprotocol.HealthPath
req, err := http.NewRequestWithContext(
ctx,
+13 -10
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/sww"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -164,7 +164,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
// One request's three lines carry one id, rproxy's with waf's answer.
line := firstOut.line(t, "path", bannedProbe)
if line["disposition"] != waf.DispositionDeny ||
if line["disposition"] != wafprotocol.DispositionDeny ||
line["client_ip"] != placed {
t.Errorf("rproxy's line is %v", line)
}
@@ -249,7 +249,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
line := out.line(t, "path", "/after-stop")
if line["disposition"] != waf.DispositionAllow ||
if line["disposition"] != wafprotocol.DispositionAllow ||
line["waf_backend_no_answer"] != true {
t.Errorf("rproxy's line is %v", line)
}
@@ -437,7 +437,7 @@ func TestAChunkedBodyStaysABodyAtWAF(t *testing.T) {
const smuggledID = "smuggled-request"
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
waf.ClientHeader + ": " + otherClient + "\r\n" +
wafprotocol.ClientHeader + ": " + otherClient + "\r\n" +
"X-Request-ID: " + smuggledID + "\r\n\r\n"
status := sendRaw(
@@ -605,10 +605,11 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
protocols := make(chan string, 10)
backend := httptest.NewUnstartedServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" {
if r.Header.Get(wafprotocol.ClientHeader) != "" {
protocols <- r.Proto
w.Header().Set(waf.DispositionHeader, waf.DispositionDeny)
w.Header().
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionDeny)
w.WriteHeader(http.StatusForbidden)
}
}))
@@ -690,8 +691,9 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
release := make(chan struct{})
backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(waf.ClientHeader) != "" {
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
if r.Header.Get(wafprotocol.ClientHeader) != "" {
w.Header().
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
return
}
@@ -802,7 +804,8 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
w.Header().
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
}))
t.Cleanup(backend.Close)
@@ -935,7 +938,7 @@ func TestMainRunsRProxy(t *testing.T) {
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
"http://"+addr+waf.HealthPath, http.NoBody)
"http://"+addr+wafprotocol.HealthPath, http.NoBody)
if err != nil {
t.Fatalf("new request: %v", err)
}
+3 -5
View File
@@ -23,7 +23,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/sww"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
status := run(t.Context(), map[string]string{
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
"SWW_WAF_CORE_RULE_SET_MODE": string(
config.CoreRuleSetModeOff,
),
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
}, out)
if status != 1 {
@@ -1813,7 +1811,7 @@ func statusFrom(t *testing.T, url, from string) int {
}
req.Header.Set("X-Forwarded-For", from)
req.Header.Set(waf.ClientHeader, from)
req.Header.Set(wafprotocol.ClientHeader, from)
req.Header.Set("X-Request-ID", rand.Text())
transport := &http.Transport{}
+4 -29
View File
@@ -2,7 +2,6 @@ package sww
import (
"context"
"fmt"
"io"
"log/slog"
"net"
@@ -176,35 +175,11 @@ func newRuleFiles(
return ruleFiles, nil
}
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
crs, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
crs, err := waf.NewCoreRuleSet(cfg)
if err != nil {
return nil, startError{
"cannot use the Core Rule Set",
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
}
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
return nil, startError{"cannot use the Core Rule Set", err}
}
return crs, nil
@@ -217,7 +192,7 @@ func newLookupFile(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*lookup.File, error) {
if cfg.LookupSource != config.LookupSourceFile {
if cfg.LookupSource != "file" {
return nil, nil //nolint:nilnil // no lookup database, and no error
}
+14 -8
View File
@@ -20,7 +20,7 @@ import (
_ "time/tzdata"
"sneak.berlin/go/smallwebwaf/internal/sww"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// commandForMain, while set in the environment, has this test binary run Main
@@ -106,7 +106,7 @@ func TestMainRunsWAF(t *testing.T) {
starting := out.lineBeforeExit(t, "starting", exited)
addr, _ := starting["address"].(string)
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
// The files are written as sww stops.
err = cmd.Process.Signal(syscall.SIGTERM)
@@ -217,7 +217,7 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
t.Fatalf("new request: %v", err)
}
req.Header.Set(waf.ClientHeader, placed)
req.Header.Set(wafprotocol.ClientHeader, placed)
req.Header.Set("X-Request-ID", rand.Text())
transport := &http.Transport{}
@@ -232,11 +232,11 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
_ = res.Body.Close()
if res.StatusCode != status ||
res.Header.Get(waf.DispositionHeader) != disposition {
res.Header.Get(wafprotocol.DispositionHeader) != disposition {
t.Errorf(
"answered %d with disposition %q, want %d with %q",
res.StatusCode,
res.Header.Get(waf.DispositionHeader),
res.Header.Get(wafprotocol.DispositionHeader),
status,
disposition,
)
@@ -287,12 +287,18 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
}
runWAF(t, env, func(addr string) {
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
// The default rule file bans a probe for this.
ask(t, addr, "/.env", http.StatusForbidden, waf.DispositionDeny)
ask(
t,
addr,
"/.env",
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
})
runWAF(t, env, func(addr string) {
ask(t, addr, "/", http.StatusForbidden, waf.DispositionDeny)
ask(t, addr, "/", http.StatusForbidden, wafprotocol.DispositionDeny)
})
}
+1 -1
View File
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
)
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil
}
+2 -2
View File
@@ -37,10 +37,10 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
"",
map[string]string{metricsToken: token},
)
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(otherClient+"/32"),
clk.Now(),
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
before := server.Ledger.Snapshot()
+5 -3
View File
@@ -130,9 +130,11 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
// A ban for a clear sign of attack, which a request under it would make
// permanent.
group := netip.MustParsePrefix(ipv6Group)
attackBan, _ := server.Ledger.Ban(group, start, bans.Grounds{
Cause: bans.CauseAttack, Notes: bans.Notes{RuleID: "probe"},
})
attackBan, _ := server.Ledger.BanForAttack(
group,
start,
bans.Notes{RuleID: "probe"},
)
// The third request breaks the limit, and so does the fourth, within the
// cooldown, which raises nothing. The probe is a clear sign of attack.
+63 -25
View File
@@ -26,6 +26,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -84,7 +85,7 @@ func TestSixteenThousandCookiesAreRefusedBeforeTheCoreRuleSetReadsThem(
got := do(t, req)
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
wantDisposition(t, got, tooLarge, wafprotocol.DispositionDeny)
line := s.out.requestLine(t)
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
@@ -164,9 +165,14 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck(
line := s.out.requestLine(t)
if tc.status == http.StatusOK {
wantDisposition(t, got, tc.status, waf.DispositionAllow)
wantDisposition(
t,
got,
tc.status,
wafprotocol.DispositionAllow,
)
} else {
wantDisposition(t, got, tc.status, waf.DispositionDeny)
wantDisposition(t, got, tc.status, wafprotocol.DispositionDeny)
}
wantLine(t, line, tc.status, tc.action)
@@ -219,9 +225,14 @@ func TestAHeaderLineOverTheLimitIsRefusedWith431(t *testing.T) {
got := do(t, req)
if tc.refused {
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
wantDisposition(
t,
got,
tooLarge,
wafprotocol.DispositionDeny,
)
} else {
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
}
})
}
@@ -244,10 +255,7 @@ func TestRequestLineRefusalNamesItsHeuristicLengthAndLimit(t *testing.T) {
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
t.Parallel()
repeated := make([]string, headerFieldLimit)
for i := range repeated {
repeated[i] = "1"
}
repeated := slices.Repeat([]string{"1"}, headerFieldLimit)
for _, tc := range []struct {
name string
@@ -294,7 +302,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
line := s.out.requestLine(t)
if tc.refused {
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
wantDisposition(
t,
got,
tooLarge,
wafprotocol.DispositionDeny,
)
wantLine(
t,
line,
@@ -305,7 +318,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
return
}
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
got,
http.StatusOK,
wafprotocol.DispositionAllow,
)
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
if line.DurationCoreRuleSetMS == nil {
@@ -332,7 +350,12 @@ func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) {
req := s.question(siteA, client, "/")
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
wantDisposition(
t,
do(t, req),
tooLarge,
wafprotocol.DispositionDeny,
)
})
}
}
@@ -348,7 +371,12 @@ func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) {
req := s.question(siteA, client, "/")
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
}
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
@@ -364,7 +392,7 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
line := s.out.requestLine(t)
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
@@ -386,14 +414,14 @@ func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
for range 2 {
req := s.question(siteA, client, "/")
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
wantDisposition(t, do(t, req), tooLarge, wafprotocol.DispositionDeny)
}
wantDisposition(
t,
s.ask(client, "/"),
http.StatusOK,
waf.DispositionAllow,
wafprotocol.DispositionAllow,
)
}
@@ -466,7 +494,12 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
req := s.question(siteA, client, tc.path)
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
wantDisposition(
t,
do(t, req),
tooLarge,
wafprotocol.DispositionDeny,
)
wantLine(
t,
s.out.requestLine(t),
@@ -503,11 +536,16 @@ func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
over := s.question(siteA, client, "/")
over.Header.Set(cookieHeader, cookies(cookieLimit+1))
wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny)
wantDisposition(t, do(t, over), tooLarge, wafprotocol.DispositionDeny)
within := s.question(siteA, client, "/")
within.Header.Set(cookieHeader, cookies(cookieLimit))
wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
do(t, within),
http.StatusOK,
wafprotocol.DispositionAllow,
)
lines := s.out.requestLines(t, 2)
@@ -564,7 +602,7 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
t,
s.ask(fromDE, "/"),
http.StatusForbidden,
waf.DispositionDeny,
wafprotocol.DispositionDeny,
)
line := s.out.requestLine(t)
@@ -601,7 +639,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
t,
s.ask(client, "/"),
http.StatusForbidden,
waf.DispositionDeny,
wafprotocol.DispositionDeny,
)
line := s.out.requestLine(t)
@@ -630,7 +668,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
t.Errorf(
@@ -672,7 +710,7 @@ func askDirectly(
http.NoBody,
)
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
req.Header.Set(waf.ClientHeader, client)
req.Header.Set(wafprotocol.ClientHeader, client)
req.Header.Set(requestIDHeader, "the-id")
change(req)
@@ -686,10 +724,10 @@ func askDirectly(
// banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
server.Ledger.Ban(
server.Ledger.BanForLimit(
netip.MustParsePrefix(client+"/32"),
now,
bans.Grounds{Cause: bans.CauseLimit},
bans.Notes{},
)
}
+57 -77
View File
@@ -86,14 +86,13 @@ func (rq *request) countedBytes() int64 {
response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted {
case config.BytesCountedResponse:
case "response":
return response
case config.BytesCountedRequest:
case "request":
return request
case config.BytesCountedBoth:
default: // both
return response + request
}
return response + request
}
// banForLimit bans the client's netblock at now for a broken limit, the
@@ -103,8 +102,9 @@ func (rq *request) countedBytes() int64 {
// one whose bytes broke a byte limit, the refusal for one that broke the
// refusal limit. The ban's notes give the client's limit percentage for a
// rate limit or a byte limit; the refusal limit is not lowered. The ban sets
// the client's counters back to zero. In observe mode, banClient says what
// happens instead.
// the client's counters back to zero. In observe mode it makes no ban and
// sets nothing back, and raises the alert for the ban it would have made,
// if that alert would be sent.
func (rq *request) banForLimit(
now time.Time,
hit ratelimit.Hit,
@@ -122,11 +122,23 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
}
notes := bans.Notes{
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Kind: hit.Kind,
Limit: hit.Limit,
Window: hit.Window,
Count: hit.Count,
Reputation: rq.reputation,
Request: rq.noted(now, status),
Requests: rq.netblockRequests(netblock),
}
switch hit.Kind {
@@ -136,54 +148,47 @@ func (rq *request) banForLimit(
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
}
rq.banClient(
now,
status,
bans.Grounds{Cause: bans.CauseLimit, Notes: notes},
)
if rq.h.config.Observe {
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
return
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
rq.alertBan(ban)
}
}
// banForAttack bans the client's netblock at now for a clear sign of
// attack, which notes name: the ban rule that matched, or the trap path
// asked for. It returns the action for that ban. In observe mode, banClient
// says what happens instead.
// asked for. It fills in the rest of the notes. In observe mode it makes
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
return rq.banClient(
now,
rq.h.config.BanResponse,
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
)
}
// banClient bans the client's netblock at now on grounds, and returns the
// action for that ban. It adds to the notes what the request tells of the
// client, with status, what the client was sent. A ban for a broken limit
// sets the client's counters back to zero.
//
// In observe mode it makes no ban and sets nothing back. It raises the
// alert for the ban it would have made, if that alert would be sent.
func (rq *request) banClient(
now time.Time,
status int,
grounds bans.Grounds,
) string {
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
)
}
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, rq.h.config.BanResponse)
notes.Requests = rq.netblockRequests(netblock)
if rq.h.config.Observe {
if !rq.wouldAlertBan(netblock, now, grounds.Cause) {
return banAction(
rq.h.ledger.WouldBePermanent(netblock, now, grounds.Cause),
)
}
grounds.Notes = rq.withClientNotes(
grounds.Notes,
now,
status,
netblock,
)
ban, wouldBan := rq.h.ledger.WouldBan(netblock, now, grounds)
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
if wouldBan {
rq.alertBan(ban)
}
@@ -191,13 +196,7 @@ func (rq *request) banClient(
return banAction(ban.Permanent())
}
grounds.Notes = rq.withClientNotes(grounds.Notes, now, status, netblock)
ban, made := rq.h.ledger.Ban(netblock, now, grounds)
if grounds.Cause == bans.CauseLimit {
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
}
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
rq.line.BanExpires = banExpires(ban)
if made {
@@ -207,25 +206,6 @@ func (rq *request) banClient(
return banAction(ban.Permanent())
}
// withClientNotes is notes with what the request tells of the client
// added: its AS, country and reputation, the request at now with status,
// and how many requests netblock, the client's, has sent.
func (rq *request) withClientNotes(
notes bans.Notes,
now time.Time,
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = rq.netblockRequests(netblock)
return notes
}
// wouldAlertBan reports whether the alert for a ban on netblock for cause
// made at now would be sent. In observe mode the ban the request would
// have made is worked out only then, at most once per
+2 -3
View File
@@ -63,8 +63,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
}
blocklisted := percentage{percent: whole}
if rq.blocklisted &&
cfg.BlocklistAction == config.ReputationActionLimit {
if rq.blocklisted && cfg.BlocklistAction == "limit" {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION",
@@ -73,7 +72,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
cfg.DNSBLAbuseIPDBAction == "limit" {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
+2 -1
View File
@@ -19,6 +19,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// The biased thresholds.
@@ -589,7 +590,7 @@ func serveFromDE(
t.Helper()
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
req.Header.Set(waf.ClientHeader, fromDE)
req.Header.Set(wafprotocol.ClientHeader, fromDE)
req.Header.Set(requestIDHeader, "the-id")
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
+1 -3
View File
@@ -7,7 +7,6 @@ import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -42,6 +41,5 @@ func (blocklists) check(
// client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
config.ReputationActionDeny
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
}
+2 -4
View File
@@ -7,6 +7,7 @@ import (
"slices"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// peerAddress is the address of the request's TCP peer.
@@ -19,14 +20,11 @@ func peerAddress(r *http.Request) netip.Addr {
return addrPort.Addr().Unmap()
}
// requestIDHeader carries the request's id, from rproxy.
const requestIDHeader = "X-Request-ID"
// requestID is the request's id: the one a question gives, or for any other
// request a new random one.
func requestID(r *http.Request, isQuestion bool) string {
if isQuestion {
return r.Header.Get(requestIDHeader)
return r.Header.Get(wafprotocol.RequestIDHeader)
}
return rand.Text()
+2 -2
View File
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
}
threshold := cfg.CoreRuleSetScoreThreshold
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect)
detail["mode"] = config.CoreRuleSetModeDetect
case rq.h.config.Observe:
detail["mode"] = "observe"
}
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
@@ -395,9 +395,9 @@ func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) {
})
wantDisposition(t, do(t, s.question(siteA, client, fileName)),
http.StatusOK, waf.DispositionAllow)
http.StatusOK, wafprotocol.DispositionAllow)
wantDisposition(t, do(t, s.question(siteB, client, fileName)),
http.StatusForbidden, waf.DispositionDeny)
http.StatusForbidden, wafprotocol.DispositionDeny)
lines := s.out.requestLines(t, 2)
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
+1 -2
View File
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
}
if !rq.dnsblListed ||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil
}
+8 -3
View File
@@ -5,7 +5,7 @@ import (
"testing"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
@@ -25,8 +25,13 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
)
for range healthChecks {
got := s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "",
http.StatusOK)
got := s.ownEndpointRequest(
"",
http.MethodGet,
wafprotocol.HealthPath,
"",
http.StatusOK,
)
if string(got.body) != "ok\n" ||
got.header.Get("Content-Type") != contentType {
+2 -1
View File
@@ -11,6 +11,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
@@ -118,7 +119,7 @@ func TestOwnEndpointsAreNotInTheHistory(t *testing.T) {
s.ownEndpointRequest(
"",
http.MethodGet,
waf.HealthPath,
wafprotocol.HealthPath,
"",
http.StatusOK,
)
+2 -25
View File
@@ -3,7 +3,6 @@ package waf
import (
"context"
"log/slog"
"net/http"
"net/netip"
"strings"
"time"
@@ -16,15 +15,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)
// The headers in which an answer of allow gives rproxy the client's AS number
// and country, for the app, while SWW_WAF_ADD_LOOKUP_HEADERS is true. Go writes
// every header name in this form, as it sends it and as it receives it, so
// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive.
const (
asnHeader = "X-Client-Asn"
countryHeader = "X-Client-Country"
)
// lookupResult is what the lookup gave for a request: whether its client was
// looked up, the answer, and whether the request skipped the wait for GeoJS.
type lookupResult struct {
@@ -49,8 +39,7 @@ func (rq *request) startLookUp(
) <-chan lookupResult {
result := make(chan lookupResult, 1)
if rq.h.config.LookupSource == config.LookupSourceOff ||
!canBePlaced(rq.client) {
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
result <- lookupResult{}
return result
@@ -58,7 +47,7 @@ func (rq *request) startLookUp(
client := clientGroup(rq.h.config, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
result <- lookupResult{
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
}
@@ -167,18 +156,6 @@ func GeoJSFailed(
m.GeoJSFailures.Inc()
}
// setLookupHeaders sets the headers that give the client's AS number and
// country, leaving out one that is unknown.
func setLookupHeaders(header http.Header, asn, country string) {
if asn != "" {
header.Set(asnHeader, asn)
}
if country != "" {
header.Set(countryHeader, country)
}
}
// canBePlaced reports whether a lookup can place addr: private, loopback
// and link-local addresses have no AS number or country.
func canBePlaced(addr netip.Addr) bool {
+2 -1
View File
@@ -16,6 +16,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -66,7 +67,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
{http.MethodPost, waf.HealthPath, "", http.StatusNotFound},
{http.MethodPost, wafprotocol.HealthPath, "", http.StatusNotFound},
} {
got := s.ownEndpointRequest(
tc.authorization,
+37 -10
View File
@@ -6,6 +6,7 @@ package waf
import (
"cmp"
"fmt"
"io"
"log/slog"
"net/http"
@@ -28,14 +29,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/rules"
)
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
// never sees a path under it.
const ownEndpointPrefix = "/_sww/"
// HealthPath is the health endpoint of waf and of rproxy, which the
// container's health check asks.
const HealthPath = "/_sww/healthz"
// MetricsPath is where the metrics are, for a request that carries
// SWW_WAF_METRICS_TOKEN.
const MetricsPath = "/_sww/metrics"
@@ -82,8 +75,8 @@ type Params struct {
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
// CoreRuleSet is what inspects requests while
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
@@ -173,6 +166,40 @@ func newHandler(params Params) *handler {
}
}
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, fmt.Errorf(
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
err,
)
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return coreRuleSet, nil
}
// handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request.
type handler struct {
+13 -34
View File
@@ -20,7 +20,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
@@ -505,20 +504,24 @@ func newWAFWith(
}
}
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params := waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
// The tests of the limit set it back, and hold the Core Rule Set
// with a stand-in.
BesideLookupTimeLimit: time.Hour,
}
addCoreRuleSet(t, &params)
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
@@ -527,30 +530,6 @@ func newWAFWith(
return server, out, alertQueue
}
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
func addCoreRuleSet(t *testing.T, params *waf.Params) {
t.Helper()
cfg := params.Config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params.CoreRuleSet = coreRuleSet
}
// addCounters adds to params the metrics, the table of clients, the ledger
// and the anomaly counters, as sww builds them from params' settings. The
// metrics read the ledger, the table of clients and the rules.
+4
View File
@@ -7,6 +7,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
)
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
// refuses the requests of a client a source lists.
const deny = "deny"
// noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does,
// with reason.
+1 -1
View File
@@ -421,7 +421,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return geojs.Answer{}, false
}
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
return rq.lookupAnswer, true
}
+2 -2
View File
@@ -13,7 +13,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -248,7 +248,7 @@ func TestTimingsAreInOrder(t *testing.T) {
wantStatus(t, do(t, req), http.StatusForbidden)
refused := out.endedLines(t, 2)[1]
wantStatus(t, get(t, server.Addr, waf.HealthPath), http.StatusOK)
wantStatus(t, get(t, server.Addr, wafprotocol.HealthPath), http.StatusOK)
health := out.endedLines(t, 3)[2]
// A request passed to the app has every timing; one refused, none of
+54 -226
View File
@@ -1,75 +1,21 @@
package waf
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/netip"
"strconv"
"strings"
"time"
"unicode/utf8"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// The headers of rproxy's question and of waf's answer, as
// "waf" in SPEC.md gives them.
const (
// ClientHeader marks a request as rproxy's question, and gives the
// visitor's address.
ClientHeader = "X-Sww-Client"
// PeerHeader gives the address of rproxy's own TCP peer.
PeerHeader = "X-Sww-Peer"
// ContentLengthHeader gives the body length the visitor announced.
ContentLengthHeader = "X-Sww-Content-Length"
// DispositionHeader gives waf's answer.
DispositionHeader = "X-Sww-Disposition"
)
// The values of DispositionHeader.
const (
// DispositionAllow tells rproxy to pass the request to the app.
DispositionAllow = "allow"
// DispositionDeny tells rproxy to give the visitor the answer's status,
// headers and body.
DispositionDeny = "deny"
// DispositionClose tells rproxy to close the visitor's connection
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
// waf closing the question's connection instead would read as
// waf being down, and rproxy would then let the request through.
DispositionClose = "close"
)
// ReportPath is where rproxy sends the report of each request waf
// answered.
const ReportPath = "/_sww/report"
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
const QuestionBodyMaxBytes = 64 << 10
const (
// maxAwaitingReport is the most answered requests held until their
// reports come. Past it, the request answered longest ago is dropped.
maxAwaitingReport = 10000
// maxRequestIDBytes is the longest id a question may give.
maxRequestIDBytes = 128
// reportBodyMaxBytes is the most of a report's body that is read. It is
// far above any report's.
reportBodyMaxBytes = 1 << 20
// maxHTTPStatus is the highest status a report may give.
maxHTTPStatus = 599
// maxReportedBytes is the most bytes a report may give for each way, 1
// PiB, so that adding them up cannot overflow.
maxReportedBytes = 1 << 50
// maxReportedMilliseconds is the longest duration a report may give,
// about 31 years, so that it fits a time.Duration.
maxReportedMilliseconds = 1e12
// wafTimeout bounds how long a request to waf takes to
// arrive, and its answer to leave, and wafIdleTimeout how long a
// kept-open connection waits for its next request.
@@ -77,26 +23,6 @@ const (
wafIdleTimeout = 120 * time.Second
)
var (
errReportFieldMissing = errors.New("a field is missing")
errReportValue = errors.New("a value is out of range")
)
// Report is the JSON object rproxy posts to ReportPath once the visitor's
// answer has ended, its fields named as the request log's of the same
// meaning. RequestID is the question's id. Status is what the visitor got, 0
// if nothing. The durations are in milliseconds.
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Report struct {
RequestID string `json:"request_id"`
Status int `json:"status"`
RequestBytes int64 `json:"request_bytes"`
ResponseBytes int64 `json:"response_bytes"`
DurationTotalMS float64 `json:"duration_total_ms"`
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
}
// New returns the server `sww waf` runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
// passes nothing to an app. rproxy applies the size and time limits to the
@@ -132,10 +58,10 @@ func New(params Params) *Server {
}
// ServeHTTP handles one request to waf: a request carrying
// ClientHeader is rproxy's question, and any other is for waf
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
// itself.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(r.Header.Values(ClientHeader)) > 0 {
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
h.answerQuestion(w, r)
return
@@ -162,19 +88,16 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
// carries, and answers rproxy what to do with it. A denied request's counting
// is finished here, and an allowed one's by its report.
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
client, body, ok := h.readQuestion(w, r)
question, ok := h.readQuestion(w, r)
if !ok {
return
}
// One that cannot be read leaves the log line's peer_ip invalid.
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
rq := h.startRequest(
w,
asTheVisitorSentIt(r, body),
client.Unmap(),
peer.Unmap(),
question.Visitor,
question.Client,
question.Peer,
true,
)
// The last deferred runs first, so that what is held has the finished log
@@ -187,7 +110,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
rq.line.WAFLine = requestlog.WAFLineDecision
refused := rq.check(r.Context())
rq.checked = time.Now()
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
underOwnPrefix := strings.HasPrefix(
r.URL.Path,
wafprotocol.OwnEndpointPrefix,
)
if refused == nil && !underOwnPrefix {
// In observe mode, what enforce mode would have refused counts here.
@@ -213,7 +139,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
@@ -221,45 +150,19 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
)
}
// readQuestion returns the visitor's address and the body a question carries,
// and whether waf takes the question. It answers a question it does not
// take with 400 and no disposition. rproxy passes that 400 to the visitor,
// and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does not apply. The id of
// a question waf takes is held from here on, without a decision yet, so
// that another question with that id is refused while this one is being
// handled.
// readQuestion reads a question, and reports whether waf takes it. It answers
// a question it does not take with 400 and no disposition. rproxy passes that
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
// not apply. The id of a question waf takes is held from here on, without a
// decision yet, so that another question with that id is refused while this
// one is being handled.
//
// The body is read before any check, since Go's server does not send an answer
// until it has read the body. A question whose body stops arriving gets no
// answer, and is not looked at.
// A question whose body stops arriving gets no answer, and is not looked at.
func (h *handler) readQuestion(
w http.ResponseWriter, r *http.Request,
) (netip.Addr, []byte, bool) {
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
if err != nil || client.Zone() != "" {
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
return netip.Addr{}, nil, false
}
id := r.Header.Get(requestIDHeader)
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
return netip.Addr{}, nil, false
}
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
)
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
h.refuseQuestion(w, "the body is over 64 KiB")
return netip.Addr{}, nil, false
}
if err != nil {
) (wafprotocol.Question, bool) {
question, err := wafprotocol.ReadQuestion(w, r)
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
h.processLog.Warn(
"a question's body stopped arriving",
"error",
@@ -270,17 +173,23 @@ func (h *handler) readQuestion(
panic(http.ErrAbortHandler)
}
held, _ := h.awaitingReport.ContainsOrAdd(id, nil)
if err != nil {
h.refuseQuestion(w, err.Error())
return question, false
}
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
if held {
h.refuseQuestion(
w,
requestIDHeader+" is the id of a request waf holds",
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
)
return netip.Addr{}, nil, false
return question, false
}
return client, body, true
return question, true
}
// refuseQuestion answers a question waf does not take, for reason.
@@ -289,48 +198,18 @@ func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
http.Error(w, reason, http.StatusBadRequest)
}
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
// log line are to see it: without the headers rproxy adds for waf,
// with body, which has been read of r, and with the length the visitor
// announced, as -1 when it announced none. The question's own length covers
// only the part of the body it carries.
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
sent := r.Clone(r.Context())
announced := sent.Header.Get(ContentLengthHeader)
sent.Header.Del(ClientHeader)
sent.Header.Del(PeerHeader)
sent.Header.Del(ContentLengthHeader)
sent.Header.Del("Content-Length")
sent.ContentLength = -1
length, err := strconv.ParseInt(announced, 10, 64)
if err == nil && length >= 0 {
sent.ContentLength = length
sent.Header.Set("Content-Length", announced)
}
sent.Body = http.NoBody
if len(body) > 0 {
sent.Body = io.NopCloser(bytes.NewReader(body))
}
return sent
}
// allow answers a question with DispositionAllow and, while
// allow answers a question with wafprotocol.DispositionAllow and, while
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
func (rq *request) allow() {
rq.allowed = true
header := rq.out.Header()
header.Set(DispositionHeader, DispositionAllow)
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
if rq.h.config.AddLookupHeaders {
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
answer.ASN = rq.line.ASN
answer.Country = rq.line.Country
}
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
rq.out.WriteHeader(http.StatusOK)
}
@@ -348,13 +227,19 @@ func (rq *request) deny(r refusal) {
if r.status == 0 {
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
// status stays 0: the visitor is sent nothing.
rq.out.Header().Set(DispositionHeader, DispositionClose)
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
)
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
return
}
rq.out.Header().Set(DispositionHeader, DispositionDeny)
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(rq.out, http.StatusText(r.status), r.status)
}
@@ -362,7 +247,7 @@ func (rq *request) deny(r refusal) {
// runs on it: the address is on the private network, where the peers are
// rproxy and the admin's tools, not visitors.
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
h.takeReport(w, r)
return
@@ -373,7 +258,7 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
rq := h.startRequest(w, r, peer, peer, false)
defer rq.finish()
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
rq.answerHealth()
return
@@ -382,11 +267,11 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
rq.answerAdmin()
}
// takeReport answers POST ReportPath: it finishes the request the report
// names, if it holds that request's decision, and ignores the report
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
// report names, if it holds that request's decision, and ignores the report
// otherwise.
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
report, err := readReport(w, r)
report, err := wafprotocol.ReadReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
@@ -404,63 +289,6 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// readReport reads a report from r's body, which must be one JSON object with
// all of Report's fields, in the ranges SPEC.md gives.
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
var report Report
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
)
if err != nil {
return report, fmt.Errorf("read the body: %w", err)
}
var fields map[string]json.RawMessage
err = json.Unmarshal(body, &fields)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
for _, name := range []string{
"request_id", "status", "request_bytes", "response_bytes",
"duration_total_ms", "duration_upstream_total_ms",
} {
value, there := fields[name]
if !there || string(value) == "null" {
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
}
}
err = json.Unmarshal(body, &report)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
return report, report.checkRanges()
}
// checkRanges returns an error for a value of the report that cannot be true.
func (report *Report) checkRanges() error {
switch {
case report.Status != 0 &&
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
return fmt.Errorf("%w: status", errReportValue)
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
return fmt.Errorf("%w: request_bytes", errReportValue)
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
return fmt.Errorf("%w: response_bytes", errReportValue)
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_total_ms", errReportValue)
case report.DurationUpstreamTotalMS < 0 ||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
}
return nil
}
// finishWith finishes a request with what its report says, and writes the
// request's last log line, which holds its decision's fields and the report's.
// For an allowed request it first runs the heuristics of onReport, and adds the
@@ -470,7 +298,7 @@ func (report *Report) checkRanges() error {
func (h *handler) finishWith(
ctx context.Context,
held *decision,
report Report,
report wafprotocol.Report,
) {
rq := &request{
decision: *held,
+195 -86
View File
@@ -22,6 +22,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -106,8 +107,8 @@ func (s *WAF) questionWith(
req := newRequest(s.t, method, s.addr, path, body)
req.Host = host
req.Header.Set(waf.ClientHeader, from)
req.Header.Set(waf.PeerHeader, rproxyPeer)
req.Header.Set(wafprotocol.ClientHeader, from)
req.Header.Set(wafprotocol.PeerHeader, rproxyPeer)
req.Header.Set(requestIDHeader, rand.Text())
return req
@@ -134,13 +135,18 @@ func (s *WAF) allowed(from string) string {
s.t.Helper()
req := s.question(siteA, from, "/")
wantDisposition(s.t, do(s.t, req), http.StatusOK, waf.DispositionAllow)
wantDisposition(
s.t,
do(s.t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
return req.Header.Get(requestIDHeader)
}
// report sends rproxy's report and returns the answer.
func (s *WAF) report(report waf.Report) answer {
func (s *WAF) report(report wafprotocol.Report) answer {
s.t.Helper()
body, err := json.Marshal(report)
@@ -155,8 +161,11 @@ func (s *WAF) report(report waf.Report) answer {
func (s *WAF) reportBody(body string) answer {
s.t.Helper()
return do(s.t, newRequest(s.t, http.MethodPost, s.addr, waf.ReportPath,
strings.NewReader(body)))
return do(
s.t,
newRequest(s.t, http.MethodPost, s.addr, wafprotocol.ReportPath,
strings.NewReader(body)),
)
}
// passedOn asks about a request from the visitor at from on siteA, and reports
@@ -164,7 +173,7 @@ func (s *WAF) reportBody(body string) answer {
func (s *WAF) passedOn(from string) {
s.t.Helper()
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: s.allowed(from),
Status: http.StatusOK,
RequestBytes: bodyBytes,
@@ -210,7 +219,7 @@ func wantDisposition(
t.Helper()
wantStatus(t, got, status)
if answered := got.header.Get(waf.DispositionHeader); answered !=
if answered := got.header.Get(wafprotocol.DispositionHeader); answered !=
disposition {
t.Errorf(
"answered with disposition %q, want %q",
@@ -272,10 +281,10 @@ func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) {
req := s.question(siteA, client, "/page?q=1&token=visible")
req.Header.Set(requestIDHeader, "the-id")
req.Header.Set("X-Forwarded-Proto", "https")
req.Header.Set(waf.ContentLengthHeader, "1234")
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
wantBody(t, got, "")
if id := got.header.Get(requestIDHeader); id != "" {
@@ -308,7 +317,12 @@ func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) {
s.allowed("2001:db8::1")
got := s.ask("2001:db8::2", "/")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
}
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
@@ -317,9 +331,14 @@ func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
s := startWAF(t, "", map[string]string{denyNets: client})
req := s.question(siteA, "::ffff:"+client, "/")
req.Header.Set(waf.PeerHeader, "::ffff:"+rproxyPeer)
req.Header.Set(wafprotocol.PeerHeader, "::ffff:"+rproxyPeer)
wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
do(t, req),
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
line := s.out.requestLine(t)
if line.ClientIP != client || line.PeerIP != rproxyPeer {
@@ -339,13 +358,13 @@ func TestMalformedQuestionGets400AndAWarningAndNoLine(t *testing.T) {
change func(*http.Request)
}{
{"client not an address", func(req *http.Request) {
req.Header.Set(waf.ClientHeader, "not an address")
req.Header.Set(wafprotocol.ClientHeader, "not an address")
}},
{"client empty", func(req *http.Request) {
req.Header.Set(waf.ClientHeader, "")
req.Header.Set(wafprotocol.ClientHeader, "")
}},
{"client address with a zone", func(req *http.Request) {
req.Header.Set(waf.ClientHeader, "2001:db8::1%eth0")
req.Header.Set(wafprotocol.ClientHeader, "2001:db8::1%eth0")
}},
{"id absent", func(req *http.Request) {
req.Header.Del(requestIDHeader)
@@ -390,7 +409,12 @@ func TestLongestValidIDIsTaken(t *testing.T) {
req := s.question(siteA, client, "/")
req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
}
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
@@ -399,16 +423,18 @@ func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
s := startWAF(t, "", nil)
over := strings.NewReader(
strings.Repeat("x", waf.QuestionBodyMaxBytes+1),
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes+1),
)
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
wantDisposition(t, got, http.StatusBadRequest, "")
wantProcessMessages(t, s, "a question is refused", 1)
wantNoRequestLines(t, s)
whole := strings.NewReader(strings.Repeat("x", waf.QuestionBodyMaxBytes))
whole := strings.NewReader(
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes),
)
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
}
func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
@@ -424,12 +450,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
wantProcessMessages(t, s, "a question is refused", 1)
// A denied request's id is held as well.
denied := s.question(siteA, otherClient, waf.HealthPath)
denied := s.question(siteA, otherClient, wafprotocol.HealthPath)
wantDisposition(
t,
do(t, denied),
http.StatusNotFound,
waf.DispositionDeny,
wafprotocol.DispositionDeny,
)
again = s.question(siteA, client, "/")
@@ -437,7 +463,9 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
wantDisposition(t, do(t, again), http.StatusBadRequest, "")
// The report ends the holding, and the refused questions changed nothing.
got := s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
got := s.report(
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent)
if history := s.history(); history.ResponseBytes != answerBytes {
@@ -446,7 +474,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
again = s.question(siteA, client, "/")
again.Header.Set(requestIDHeader, id)
wantDisposition(t, do(t, again), http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
do(t, again),
http.StatusOK,
wafprotocol.DispositionAllow,
)
}
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
@@ -481,7 +514,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
// A report for a question not yet answered is ignored, and that question's
// id is taken.
got := s.report(waf.Report{RequestID: id, ResponseBytes: 1000})
got := s.report(wafprotocol.Report{RequestID: id, ResponseBytes: 1000})
wantStatus(t, got, http.StatusNoContent)
// This would wait on the same lookup were the id not taken.
@@ -496,7 +529,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
select {
case got := <-answered:
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
case <-time.After(waitLimit):
t.Fatal("the question got no answer")
}
@@ -515,7 +548,9 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
}
// The true report comes after, and is the one that counts.
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
got = s.report(
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent)
if history := s.history(); history.ResponseBytes != answerBytes {
@@ -533,17 +568,17 @@ func TestPeerAbsentOrUnreadableIsAnsweredAndLoggedInvalid(t *testing.T) {
s := startWAF(t, "", nil)
req := s.question(siteA, client, "/")
req.Header.Del(waf.PeerHeader)
req.Header.Del(wafprotocol.PeerHeader)
if peer != "" {
req.Header.Set(waf.PeerHeader, peer)
req.Header.Set(wafprotocol.PeerHeader, peer)
}
wantDisposition(
t,
do(t, req),
http.StatusOK,
waf.DispositionAllow,
wafprotocol.DispositionAllow,
)
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
@@ -571,12 +606,12 @@ func TestContentLengthUnreadableOrNegativeCountsAsAbsent(t *testing.T) {
})
req := s.question(siteA, client, "/")
req.Header.Set(waf.ContentLengthHeader, announced)
req.Header.Set(wafprotocol.ContentLengthHeader, announced)
wantDisposition(
t,
do(t, req),
http.StatusOK,
waf.DispositionAllow,
wafprotocol.DispositionAllow,
)
if line := s.out.requestLine(t); line.ContentLength != 0 {
@@ -595,13 +630,18 @@ func TestHeaderSentTwiceUsesItsFirstValue(t *testing.T) {
s := startWAF(t, "", nil)
req := s.question(siteA, client, "/")
req.Header.Add(waf.ClientHeader, otherClient)
req.Header.Add(waf.PeerHeader, "172.18.0.3")
req.Header.Add(wafprotocol.ClientHeader, otherClient)
req.Header.Add(wafprotocol.PeerHeader, "172.18.0.3")
req.Header.Set(requestIDHeader, "first")
req.Header.Add(requestIDHeader, "second")
req.Header.Set(waf.ContentLengthHeader, "1234")
req.Header.Add(waf.ContentLengthHeader, "5678")
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
req.Header.Add(wafprotocol.ContentLengthHeader, "5678")
wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
line := s.out.requestLine(t)
if line.ClientIP != client || line.PeerIP != rproxyPeer ||
@@ -630,7 +670,7 @@ func sendQuestionWithPartOfItsBody(t *testing.T, conn net.Conn) {
_, err := fmt.Fprintf(conn,
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
"Content-Length: 100\r\n\r\npart",
siteA, waf.ClientHeader, client, requestIDHeader)
siteA, wafprotocol.ClientHeader, client, requestIDHeader)
if err != nil {
t.Fatalf("send the question: %v", err)
}
@@ -699,7 +739,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
t,
do(t, again),
http.StatusOK,
waf.DispositionAllow,
wafprotocol.DispositionAllow,
)
})
}
@@ -717,7 +757,7 @@ func TestQuestionDeniedGetsTheBanResponse(t *testing.T) {
})
got := s.ask(client, "/")
wantDisposition(t, got, status, waf.DispositionDeny)
wantDisposition(t, got, status, wafprotocol.DispositionDeny)
wantBody(t, got, http.StatusText(status)+"\n")
wantLine(
t,
@@ -737,7 +777,7 @@ func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) {
})
got := s.ask(client, "/")
wantDisposition(t, got, http.StatusOK, waf.DispositionClose)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionClose)
wantBody(t, got, "")
// The visitor is sent nothing.
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
@@ -761,12 +801,22 @@ func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse(
})
got := s.ask(client, "/blocked")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
wantLine(t, s.out.requestLine(t), http.StatusForbidden,
requestlog.ActionBlockedByBlockRule)
got = s.ask(client, "/.env")
wantDisposition(t, got, http.StatusTooManyRequests, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusTooManyRequests,
wafprotocol.DispositionDeny,
)
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
requestlog.ActionTemporarilyBanned)
}
@@ -781,10 +831,15 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
})
for _, path := range []string{
waf.HealthPath, waf.MetricsPath, waf.ReportPath,
wafprotocol.HealthPath, waf.MetricsPath, wafprotocol.ReportPath,
} {
got := s.ask(client, path)
wantDisposition(t, got, http.StatusNotFound, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusNotFound,
wafprotocol.DispositionDeny,
)
}
wantLine(
@@ -795,7 +850,12 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
)
got := s.ask(otherClient, waf.MetricsPath)
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
}
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
@@ -804,7 +864,7 @@ func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
got := s.ask(client, "/")
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
line := s.out.requestLine(t)
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
@@ -822,7 +882,7 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheRefusalLimit(
for range 2 {
got := s.ask(client, "/blocked")
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
}
lines := s.out.requestLines(t, 2)
@@ -846,7 +906,12 @@ func TestDeniedQuestionCountsForTheRefusalLimit(t *testing.T) {
for range 2 {
got := s.ask(client, "/blocked")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
}
lines := s.out.requestLines(t, 2)
@@ -882,7 +947,12 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
})
got := s.ask(fromDE, "/")
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
got,
http.StatusOK,
wafprotocol.DispositionAllow,
)
asn, country := got.header.Get(
"X-Client-Asn",
@@ -916,7 +986,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
}
// 100 bytes are over the limit of 99.
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
})
@@ -940,7 +1010,12 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
}
got = s.ask(client, "/")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
}
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
@@ -958,12 +1033,12 @@ func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
t,
s.ask(fromKP, "/"),
http.StatusOK,
waf.DispositionAllow,
wafprotocol.DispositionAllow,
)
s.passedOn(fromDE)
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
waf.DispositionDeny)
wafprotocol.DispositionDeny)
}
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
@@ -989,7 +1064,7 @@ func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
)
}
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
})
@@ -1021,7 +1096,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
t,
do(t, denied),
http.StatusForbidden,
waf.DispositionDeny,
wafprotocol.DispositionDeny,
)
counters := s.server.Anomalies.Snapshot()
@@ -1039,7 +1114,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
wantMetric(t, s.metrics(), series, 1)
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: denied.Header.Get(
requestIDHeader,
), Status: http.StatusForbidden,
@@ -1082,7 +1157,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
1,
) // the scrape itself
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusBadGateway,
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
@@ -1100,7 +1175,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
// check needs a server that has not been scraped yet.
unscraped := startWAF(t, "", map[string]string{metricsToken: token})
got = unscraped.report(waf.Report{
got = unscraped.report(wafprotocol.Report{
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
DurationTotalMS: 5,
})
@@ -1118,7 +1193,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: "no-such-request", ResponseBytes: 1000,
})
wantStatus(t, got, http.StatusNoContent)
@@ -1137,7 +1212,9 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
// A second report of the same request does not count it twice.
id := s.allowed(client)
for range 2 {
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
got = s.report(
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent)
}
@@ -1347,7 +1424,7 @@ func TestBytesAtTheirMostEachWayBanRatherThanOverflow(t *testing.T) {
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: s.allowed(client), Status: http.StatusOK,
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
})
@@ -1441,9 +1518,14 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields(
req.Header.Set("X-Forwarded-Proto", "https")
id := req.Header.Get(requestIDHeader)
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
wantDisposition(
t,
do(t, req),
http.StatusOK,
wafprotocol.DispositionAllow,
)
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: id, Status: http.StatusCreated,
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
@@ -1490,7 +1572,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
0, requestlog.ActionClientDenied,
},
{"own prefix", nil, waf.HealthPath, http.StatusNotFound,
{"own prefix", nil, wafprotocol.HealthPath, http.StatusNotFound,
requestlog.ActionOwnEndpoint},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -1502,7 +1584,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
id := req.Header.Get(requestIDHeader)
do(t, req)
got := s.report(waf.Report{
got := s.report(wafprotocol.Report{
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
})
wantStatus(t, got, http.StatusNoContent)
@@ -1542,13 +1624,13 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
t,
readAnswer(res),
http.StatusOK,
waf.DispositionAllow,
wafprotocol.DispositionAllow,
)
}
// The table is full again with next and those, without oldest.
got := s.report(
waf.Report{RequestID: oldest, ResponseBytes: answerBytes},
wafprotocol.Report{RequestID: oldest, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent)
@@ -1559,7 +1641,9 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
)
}
got = s.report(waf.Report{RequestID: next, ResponseBytes: answerBytes})
got = s.report(
wafprotocol.Report{RequestID: next, ResponseBytes: answerBytes},
)
wantStatus(t, got, http.StatusNoContent)
if counted := s.history().Requests; counted != 1 {
@@ -1580,10 +1664,20 @@ func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) {
)
got := do(t, s.question(siteA, client, "/.env"))
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
got = do(t, s.question(siteB, client, "/"))
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
lines := s.out.requestLines(t, 2)
if lines[0].Host != siteA || lines[1].Host != siteB {
@@ -1621,7 +1715,12 @@ func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) {
s.allowed(client)
got := do(t, s.question(siteB, client, "/"))
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
}
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
@@ -1636,13 +1735,13 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
path, authorization string
want int
}{
{waf.HealthPath, "", http.StatusOK},
{wafprotocol.HealthPath, "", http.StatusOK},
{waf.MetricsPath, bearer, http.StatusOK},
{waf.MetricsPath, "", http.StatusUnauthorized},
{waf.BansPath, adminBearer, http.StatusOK},
{waf.BansPath, "", http.StatusUnauthorized},
{waf.ClientsPath + client, adminBearer, http.StatusOK},
{waf.ReportPath, "", http.StatusNotFound},
{wafprotocol.ReportPath, "", http.StatusNotFound},
{"/_sww/other", "", http.StatusNotFound},
{"/", "", http.StatusNotFound},
} {
@@ -1659,10 +1758,10 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
t,
do(t, req),
http.StatusNotFound,
waf.DispositionDeny,
wafprotocol.DispositionDeny,
)
} else {
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
wantDisposition(t, do(t, req), http.StatusOK, wafprotocol.DispositionAllow)
}
}
}
@@ -1672,11 +1771,11 @@ func TestOwnEndpointsLinesDoNotSayWhichOfTheTwoTheyAre(t *testing.T) {
s := startWAF(t, "", nil)
wantStatus(t, get(t, s.addr, waf.HealthPath), http.StatusOK)
wantStatus(t, get(t, s.addr, wafprotocol.HealthPath), http.StatusOK)
line := s.out.requestLine(t)
if _, there := line.fields["waf_line"]; there ||
line.Path != waf.HealthPath {
line.Path != wafprotocol.HealthPath {
t.Errorf("the health check's log line is %+v", line.fields)
}
}
@@ -1688,7 +1787,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
req := s.questionWith(http.MethodPost, siteA, client, "/",
strings.NewReader("a body"))
req.Header.Set(waf.ContentLengthHeader, "1073741824")
req.Header.Set(wafprotocol.ContentLengthHeader, "1073741824")
req.Header.Set(forwardedFor, otherClient)
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
@@ -1698,7 +1797,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
}
got := do(t, req)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
line := s.out.requestLine(t)
if line.ClientIP != client || line.ContentLength != 1<<30 {
@@ -1776,7 +1875,7 @@ carried header:Content-Length block ^6$
strings.NewReader("a body"))
if tc.announced != "" {
req.Header.Set(waf.ContentLengthHeader, tc.announced)
req.Header.Set(wafprotocol.ContentLengthHeader, tc.announced)
}
wantStatus(t, do(t, req), tc.want)
@@ -1794,19 +1893,24 @@ func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) {
req := s.questionWith(http.MethodPost, siteA, client, "/",
strings.NewReader(body))
req.Header.Set("Content-Type", contentType)
req.Header.Set(waf.ContentLengthHeader, "100000")
req.Header.Set(wafprotocol.ContentLengthHeader, "100000")
return do(t, req)
}
got := ask(formData, "id=1'%20OR%20'1'='1")
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
// The first 8 KiB of a longer JSON body cannot be parsed, which is no
// attack, so the Core Rule Set does not read it.
cut := `{"text":"` + strings.Repeat("x", 8<<10)
got = ask("application/json", cut)
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
}
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
@@ -1815,7 +1919,12 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
got := s.ask(client, sqlInjection)
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
wantDisposition(
t,
got,
http.StatusForbidden,
wafprotocol.DispositionDeny,
)
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 ||
@@ -1836,7 +1945,7 @@ func TestReputationHitAlertNamesTheSite(t *testing.T) {
loadLists(t, s.server, map[string][]string{dropURL: {client}})
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
waf.DispositionDeny)
wafprotocol.DispositionDeny)
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
+46
View File
@@ -0,0 +1,46 @@
package wafprotocol
import "net/http"
// Answer is what waf's answer to a question tells rproxy in its headers.
// The status, the other headers and the body of an answer of deny are what
// the visitor gets.
type Answer struct {
// Disposition is what rproxy does with the request: DispositionAllow,
// DispositionDeny or DispositionClose.
Disposition string
// ASN and Country are the client's AS number and country, each empty
// when the answer does not give it.
ASN, Country string
}
// SetAnswerHeaders sets the headers of waf's answer that tell rproxy answer,
// leaving out an ASN or Country that is empty.
func SetAnswerHeaders(header http.Header, answer Answer) {
header.Set(DispositionHeader, answer.Disposition)
if answer.ASN != "" {
header.Set(ASNHeader, answer.ASN)
}
if answer.Country != "" {
header.Set(CountryHeader, answer.Country)
}
}
// ReadAnswer returns what the headers of waf's answer tell rproxy, and
// whether they give a valid disposition.
func ReadAnswer(header http.Header) (Answer, bool) {
disposition := header.Get(DispositionHeader)
switch disposition {
case DispositionAllow, DispositionDeny, DispositionClose:
return Answer{
Disposition: disposition,
ASN: header.Get(ASNHeader),
Country: header.Get(CountryHeader),
}, true
}
return Answer{}, false
}
+189
View File
@@ -0,0 +1,189 @@
package wafprotocol
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/netip"
"net/url"
"strconv"
"unicode/utf8"
)
// maxRequestIDBytes is the longest id a question may give.
const maxRequestIDBytes = 128
// ErrBodyStoppedArriving is the error ReadQuestion wraps when the question's
// body stops arriving. Every other error it returns is why waf refuses the
// question.
var ErrBodyStoppedArriving = errors.New("the body stopped arriving")
// Why waf refuses a question. Each is also the body of the refusal.
var (
errClientNotAnAddress = errors.New(
ClientHeader + " is not an address without a zone",
)
errRequestIDNotValid = errors.New(RequestIDHeader + " is not a valid id")
errBodyTooLarge = errors.New("the body is over 64 KiB")
)
// QuestionParams are what NewQuestion needs.
type QuestionParams struct {
// Visitor is the visitor's request.
Visitor *http.Request
// PathAsSent is Visitor's path as the visitor sent it, which the question
// sends unchanged.
PathAsSent string
// Body is the part of Visitor's body the question carries, at most
// QuestionBodyMaxBytes.
Body []byte
// WAFBackend is waf's address. The question takes its scheme and host.
WAFBackend *url.URL
// Client is the visitor's address, and Peer that of rproxy's own TCP
// peer.
Client, Peer netip.Addr
// RequestID is the request's id.
RequestID string
// Scheme is how the visitor reached rproxy, or the trusted proxy in front
// of it.
Scheme string
}
// Question is a question as waf reads it.
type Question struct {
// Client is the visitor's address.
Client netip.Addr
// Peer is the address of rproxy's own TCP peer. It is not valid when the
// question gives none that can be read.
Peer netip.Addr
// RequestID is the request's id.
RequestID string
// Visitor is the visitor's request, as asTheVisitorSentIt gives it.
Visitor *http.Request
}
// NewQuestion returns the question about a visitor's request, as "The
// question" in SPEC.md gives it: the request as the visitor sent it, with the
// part of the body it carries and rproxy's headers.
func NewQuestion(
ctx context.Context,
params QuestionParams,
) (*http.Request, error) {
in := params.Visitor
question, err := http.NewRequestWithContext(
ctx,
in.Method,
params.WAFBackend.String(),
bytes.NewReader(params.Body),
)
if err != nil {
return nil, fmt.Errorf("make the question: %w", err)
}
backend := question.URL
target := *in.URL
target.Scheme = backend.Scheme
target.Host = backend.Host
target.Opaque = params.PathAsSent
question.URL = &target
question.Host = in.Host
// The length NewRequestWithContext set is that of the body carried. This
// line makes a body whose length the visitor did not announce go chunked,
// as "The question" in SPEC.md requires.
question.TransferEncoding = in.TransferEncoding
question.Header = in.Header.Clone()
question.Header.Set(ClientHeader, params.Client.String())
question.Header.Set(PeerHeader, params.Peer.String())
question.Header.Del(ContentLengthHeader)
announced := in.Header.Get("Content-Length")
if announced != "" {
question.Header.Set(ContentLengthHeader, announced)
}
question.Header.Set(RequestIDHeader, params.RequestID)
question.Header.Set("X-Forwarded-Proto", params.Scheme)
// Go's client adds a User-Agent of its own to a request without the key.
_, sent := question.Header["User-Agent"]
if !sent {
question.Header["User-Agent"] = nil
}
return question, nil
}
// ReadQuestion reads the question r, with the body it carries. An error
// wrapping ErrBodyStoppedArriving means the body stopped arriving. Any other
// error is why waf refuses the question, and the question's body is then not
// read. The body is read whole, since Go's server does not send an answer
// until it has read the body.
func ReadQuestion(w http.ResponseWriter, r *http.Request) (Question, error) {
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
if err != nil || client.Zone() != "" {
return Question{}, errClientNotAnAddress
}
id := r.Header.Get(RequestIDHeader)
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
return Question{}, errRequestIDNotValid
}
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
)
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
return Question{}, errBodyTooLarge
}
if err != nil {
return Question{}, fmt.Errorf("%w: %w", ErrBodyStoppedArriving, err)
}
// One that cannot be read leaves the log line's peer_ip invalid.
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
return Question{
Client: client.Unmap(),
Peer: peer.Unmap(),
RequestID: id,
Visitor: asTheVisitorSentIt(r, body),
}, nil
}
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
// log line are to see it: without the headers rproxy adds for waf,
// with body, which has been read of r, and with the length the visitor
// announced, as -1 when it announced none. The question's own length covers
// only the part of the body it carries.
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
sent := r.Clone(r.Context())
announced := sent.Header.Get(ContentLengthHeader)
sent.Header.Del(ClientHeader)
sent.Header.Del(PeerHeader)
sent.Header.Del(ContentLengthHeader)
sent.Header.Del("Content-Length")
sent.ContentLength = -1
length, err := strconv.ParseInt(announced, 10, 64)
if err == nil && length >= 0 {
sent.ContentLength = length
sent.Header.Set("Content-Length", announced)
}
sent.Body = http.NoBody
if len(body) > 0 {
sent.Body = io.NopCloser(bytes.NewReader(body))
}
return sent
}
+123
View File
@@ -0,0 +1,123 @@
package wafprotocol
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
)
const (
// reportBodyMaxBytes is the most of a report's body that is read. It is
// far above any report's.
reportBodyMaxBytes = 1 << 20
// maxReportedBytes is the most bytes a report may give for each way, 1
// PiB, so that adding them up cannot overflow.
maxReportedBytes = 1 << 50
// maxReportedMilliseconds is the longest duration a report may give,
// about 31 years, so that it fits a time.Duration.
maxReportedMilliseconds = 1e12
)
var (
errReportFieldMissing = errors.New("a field is missing")
errReportValue = errors.New("a value is out of range")
)
// Report is the JSON object rproxy posts to ReportPath once the visitor's
// answer has ended, its fields named as the request log's of the same
// meaning. RequestID is the question's id. Status is what the visitor got, 0
// if nothing. The durations are in milliseconds.
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Report struct {
RequestID string `json:"request_id"`
Status int `json:"status"`
RequestBytes int64 `json:"request_bytes"`
ResponseBytes int64 `json:"response_bytes"`
DurationTotalMS float64 `json:"duration_total_ms"`
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
}
// NewReportRequest returns the request that posts report to reportURL, which
// is waf's ReportPath.
func NewReportRequest(
ctx context.Context,
reportURL string,
report *Report,
) (*http.Request, error) {
body, err := json.Marshal(report)
if err != nil {
return nil, fmt.Errorf("encode the report: %w", err)
}
req, err := http.NewRequestWithContext(
ctx, http.MethodPost, reportURL, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("make the request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
return req, nil
}
// ReadReport reads a report from r's body, which must be one JSON object with
// all of Report's fields, in the ranges SPEC.md gives.
func ReadReport(w http.ResponseWriter, r *http.Request) (Report, error) {
var report Report
body, err := io.ReadAll(
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
)
if err != nil {
return report, fmt.Errorf("read the body: %w", err)
}
var fields map[string]json.RawMessage
err = json.Unmarshal(body, &fields)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
for _, name := range []string{
"request_id", "status", "request_bytes", "response_bytes",
"duration_total_ms", "duration_upstream_total_ms",
} {
value, there := fields[name]
if !there || string(value) == "null" {
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
}
}
err = json.Unmarshal(body, &report)
if err != nil {
return report, fmt.Errorf("decode the body: %w", err)
}
return report, report.checkRanges()
}
// checkRanges returns an error for a value of the report that cannot be true.
func (report *Report) checkRanges() error {
switch {
case report.Status != 0 &&
(report.Status < http.StatusContinue || report.Status > ReportStatusMax):
return fmt.Errorf("%w: status", errReportValue)
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
return fmt.Errorf("%w: request_bytes", errReportValue)
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
return fmt.Errorf("%w: response_bytes", errReportValue)
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_total_ms", errReportValue)
case report.DurationUpstreamTotalMS < 0 ||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
}
return nil
}
+61
View File
@@ -0,0 +1,61 @@
// Package wafprotocol holds the three messages rproxy and waf send each
// other, as "`waf`" in SPEC.md gives them. rproxy asks waf a question about
// each visitor's request, and waf answers it. Once the visitor's answer has
// ended, rproxy sends waf a report. This package writes and reads each
// message, and declares every name and limit both sides must agree on.
package wafprotocol
// The headers of the question and of the answer. Go writes every header name
// in this form, as it sends it and as it receives it, so X-Client-ASN arrives
// as X-Client-Asn; header names are not case-sensitive.
const (
// ClientHeader marks a request as rproxy's question, and gives the
// visitor's address.
ClientHeader = "X-Sww-Client"
// PeerHeader gives the address of rproxy's own TCP peer.
PeerHeader = "X-Sww-Peer"
// ContentLengthHeader gives the body length the visitor announced.
ContentLengthHeader = "X-Sww-Content-Length"
// RequestIDHeader gives the request's id, which rproxy makes. rproxy
// also sets it on the request it passes to the app.
RequestIDHeader = "X-Request-ID"
// DispositionHeader gives waf's answer.
DispositionHeader = "X-Sww-Disposition"
// ASNHeader and CountryHeader give the client's AS number and country in
// an answer of allow. rproxy passes them to the app.
ASNHeader = "X-Client-Asn"
CountryHeader = "X-Client-Country"
)
// The values of DispositionHeader.
const (
// DispositionAllow tells rproxy to pass the request to the app.
DispositionAllow = "allow"
// DispositionDeny tells rproxy to give the visitor the answer's status,
// headers and body.
DispositionDeny = "deny"
// DispositionClose tells rproxy to close the visitor's connection
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
// waf closing the question's connection instead would read as
// waf being down, and rproxy would then let the request through.
DispositionClose = "close"
)
// OwnEndpointPrefix starts the path of every request for sww itself: waf's
// own endpoints and rproxy's health endpoint. The app never sees a path under
// it.
const OwnEndpointPrefix = "/_sww/"
// HealthPath is the health endpoint of waf and of rproxy, which the
// container's health check asks.
const HealthPath = "/_sww/healthz"
// ReportPath is where rproxy sends the report of each request waf
// answered.
const ReportPath = "/_sww/report"
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
const QuestionBodyMaxBytes = 64 << 10
// ReportStatusMax is the highest status a report may give.
const ReportStatusMax = 599
+204
View File
@@ -0,0 +1,204 @@
package wafprotocol_test
import (
"io"
"net/http"
"net/http/httptest"
"net/netip"
"net/url"
"strings"
"testing"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// requestID is the id the tests' questions and reports give.
const requestID = "an-id"
// questionRead is what the server startQuestionReader starts read of a
// question: the question, and the body it carries.
type questionRead struct {
question wafprotocol.Question
body string
}
// startQuestionReader starts a server that reads each question it gets with
// ReadQuestion. It returns the server's URL and the channel what it reads goes
// to.
func startQuestionReader(t *testing.T) (*url.URL, <-chan questionRead) {
t.Helper()
read := make(chan questionRead, 1)
server := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
question, err := wafprotocol.ReadQuestion(w, r)
if err != nil {
t.Errorf("read the question: %v", err)
}
var body []byte
if question.Visitor != nil {
body, _ = io.ReadAll(question.Visitor.Body)
}
read <- questionRead{question, string(body)}
},
))
t.Cleanup(server.Close)
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("parse the server's URL: %v", err)
}
return serverURL, read
}
// TestQuestionReadsAsWritten sends a question NewQuestion writes over HTTP,
// and checks that ReadQuestion gives back the visitor's request and what
// rproxy added.
func TestQuestionReadsAsWritten(t *testing.T) {
t.Parallel()
const (
target = "/a%2Fb?q=1"
body = "name=value"
held = "name"
)
client := netip.MustParseAddr("2001:db8::1")
peer := netip.MustParseAddr("192.0.2.1")
backend, read := startQuestionReader(t)
visitor := httptest.NewRequestWithContext(
t.Context(),
http.MethodPost,
"http://site.example.com"+target,
strings.NewReader(body),
)
visitor.Header.Set("Content-Length", "10")
visitor.Header.Set("User-Agent", "a visitor")
visitor.Header.Set(wafprotocol.PeerHeader, "the visitor's own")
question, err := wafprotocol.NewQuestion(
t.Context(),
wafprotocol.QuestionParams{
Visitor: visitor,
PathAsSent: "/a%2Fb",
Body: []byte(held),
WAFBackend: backend,
Client: client,
Peer: peer,
RequestID: requestID,
Scheme: "https",
},
)
if err != nil {
t.Fatalf("write the question: %v", err)
}
res, err := http.DefaultTransport.RoundTrip(question)
if err != nil {
t.Fatalf("send the question: %v", err)
}
_ = res.Body.Close()
got := <-read
if got.question.Visitor == nil {
t.FailNow()
}
if got.question.Client != client || got.question.Peer != peer ||
got.question.RequestID != requestID {
t.Errorf("read %+v", got.question)
}
sent := got.question.Visitor
if sent.Host != "site.example.com" || sent.RequestURI != target ||
sent.ContentLength != int64(len(body)) || got.body != held {
t.Errorf(
"read %s at %s, of length %d, carrying %q",
sent.RequestURI, sent.Host, sent.ContentLength, got.body,
)
}
for name, want := range map[string]string{
"Content-Length": "10",
"User-Agent": "a visitor",
"X-Forwarded-Proto": "https",
wafprotocol.RequestIDHeader: requestID,
wafprotocol.ClientHeader: "",
wafprotocol.PeerHeader: "",
wafprotocol.ContentLengthHeader: "",
} {
if sent.Header.Get(name) != want {
t.Errorf("%s is %q, want %q", name, sent.Header.Get(name), want)
}
}
}
// TestAnswerReadsAsWritten checks that ReadAnswer gives back what
// SetAnswerHeaders sets, and that headers without a valid disposition give
// nothing.
func TestAnswerReadsAsWritten(t *testing.T) {
t.Parallel()
for _, want := range []wafprotocol.Answer{
{Disposition: wafprotocol.DispositionAllow, ASN: "64496", Country: "DE"},
{Disposition: wafprotocol.DispositionAllow, Country: "DE"},
{Disposition: wafprotocol.DispositionDeny},
{Disposition: wafprotocol.DispositionClose},
} {
header := http.Header{}
wafprotocol.SetAnswerHeaders(header, want)
got, valid := wafprotocol.ReadAnswer(header)
if !valid || got != want {
t.Errorf("read %+v (valid: %t), want %+v", got, valid, want)
}
}
header := http.Header{}
header.Set(wafprotocol.DispositionHeader, "maybe")
header.Set(wafprotocol.ASNHeader, "64496")
got, valid := wafprotocol.ReadAnswer(header)
if valid || got != (wafprotocol.Answer{}) {
t.Errorf("read %+v (valid: %t), want nothing", got, valid)
}
}
// TestReportReadsAsWritten checks that ReadReport gives back the report
// NewReportRequest posts.
func TestReportReadsAsWritten(t *testing.T) {
t.Parallel()
want := wafprotocol.Report{
RequestID: requestID,
Status: http.StatusOK,
RequestBytes: 10,
ResponseBytes: 20,
DurationTotalMS: 1.5,
DurationUpstreamTotalMS: 0.5,
}
req, err := wafprotocol.NewReportRequest(
t.Context(),
"http://waf.example.com"+wafprotocol.ReportPath,
&want,
)
if err != nil {
t.Fatalf("write the report: %v", err)
}
if req.Method != http.MethodPost ||
req.URL.Path != wafprotocol.ReportPath {
t.Errorf("the report is %s %s", req.Method, req.URL.Path)
}
got, err := wafprotocol.ReadReport(httptest.NewRecorder(), req)
if err != nil || got != want {
t.Errorf("read %+v (%v), want %+v", got, err, want)
}
}