Rule files, and bans for a clear sign of attack (closes #24)

Every *.rules file in SWWAF_RULES_DIR is read at start and again when
one changes, and each request is checked against the rules after the
rate limits: log notes a match, block refuses with 403, ban refuses and
bans the netblock for SWWAF_ATTACK_BAN_DURATION, made permanent by its
next request or its next clear sign of attack. bans.json gains each
ban's cause, the request log rule_ids and rule_blocked, the metrics
rule matches and rules loaded. The image ships 00-default.rules.

Judgement call: a header sent twice is matched with its values joined
by ", ".
Judgement call: SWWAF_MAX_BAN_DURATION does not cap a ban for an attack.
Not in this unit: offences for rule matches, with the error burst.

Model: opus-5-5

README.md

@@ -103,11 +103,11 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of
   window and the requests counted in it, the request that broke it, the client's
   country when it was looked up, the netblock's requests since it was first
   seen, how many of them the ban has refused, and how many bans the netblock had
-  before, for a broken limit, for a clear sign of attack and without a cause. At
-  most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; past that,
-  the earliest ban of the netblock that has gone longest without a request is
-  dropped first. `bans.json` shows the bans and their notes, a restart lifts
-  none, and you add or lift a ban by editing it (see "State files" below).
+  before. At most `SWWAF_MAX_BANS` bans are kept, past, active and permanent;
+  past that, the earliest ban of the netblock that has gone longest without a
+  request is dropped first. `bans.json` shows the bans and their notes, a
+  restart lifts none, and you add or lift a ban by editing it (see "State files"
+  below).
 - Checks each request against the rules of the rule files (see "Rule files"
   below) after the rate limits, and before its body is read. A `log` rule that
   matches is noted in the log line; a `block` rule refuses the request with

internal/bans/bans.go

@@ -101,19 +101,8 @@ type Notes struct {
 	// far. Both go up with each request the ban refuses.
 	Requests int64 `json:"requests"`
 	Refused  int64 `json:"refused"`
-	// EarlierBans is how many bans the netblock had before this one, by
-	// cause.
-	EarlierBans EarlierBans `json:"earlier_bans"`
-}
-
-// EarlierBans counts a netblock's bans before a ban, by cause.
-//
-//nolint:tagliatelle // the state files use snake_case, as the request log does
-type EarlierBans struct {
-	Limit  int `json:"limit"`
-	Attack int `json:"attack"`
-	// WithoutCause counts the bans an admin added without a cause.
-	WithoutCause int `json:"without_cause"`
+	// EarlierBans is how many bans the netblock had before this one.
+	EarlierBans int `json:"earlier_bans"`
 }
 
 // Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -359,7 +348,10 @@ func (l *Ledger) ban(
 		}
 
 		held = *bans
-		notes.EarlierBans = earlierBans(held)
+
+		// The netblock's first ban held counts the bans it had before that
+		// one, since dropped to make room, and each ban held adds one.
+		notes.EarlierBans = (*bans)[0].Notes.EarlierBans + len(*bans)
 	}
 
 	notes.Request = notes.Request.cut()
@@ -378,27 +370,6 @@ func (l *Ledger) ban(
 	return ban
 }
 
-// earlierBans returns how many bans a netblock with the bans held, oldest
-// first, has had, by cause: the first ban held counts the bans the
-// netblock had before that one, since dropped to make room, and each ban
-// held adds one.
-func earlierBans(held []Ban) EarlierBans {
-	earlier := held[0].Notes.EarlierBans
-
-	for _, ban := range held {
-		switch ban.Cause {
-		case CauseLimit:
-			earlier.Limit++
-		case CauseAttack:
-			earlier.Attack++
-		default:
-			earlier.WithoutCause++
-		}
-	}
-
-	return earlier
-}
-
 // markChanged has Changed receive a value, unless one is waiting already.
 func (l *Ledger) markChanged() {
 	select {

internal/bans/bans_test.go

@@ -24,9 +24,8 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
 		ban := ledger.BanForLimit(netblock, now, bans.Notes{})
 
 		length := time.Duration(hours) * time.Hour
-		if !ban.Expires.Equal(now.Add(length)) ||
-			ban.Notes.EarlierBans != (bans.EarlierBans{Limit: i}) {
-			t.Fatalf("ban %d lasts %s with earlier bans %+v, want %d hours and %d for a limit",
+		if !ban.Expires.Equal(now.Add(length)) || ban.Notes.EarlierBans != i {
+			t.Fatalf("ban %d lasts %s with %d earlier bans, want %d hours and %d",
 				i+1, ban.Expires.Sub(now), ban.Notes.EarlierBans, hours, i)
 		}
 
@@ -67,9 +66,8 @@ func TestRepeatWindowRunsOut(t *testing.T) {
 			first := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
 			second := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{})
 
-			if second.Expires.Sub(second.Start) != tc.want ||
-				second.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
-				t.Errorf("second ban lasts %s with earlier bans %+v, want %s and 1 for a limit",
+			if second.Expires.Sub(second.Start) != tc.want || second.Notes.EarlierBans != 1 {
+				t.Errorf("second ban lasts %s with %d earlier bans, want %s and 1",
 					second.Expires.Sub(second.Start), second.Notes.EarlierBans, tc.want)
 			}
 		})
@@ -237,10 +235,9 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
 	second := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
 
 	held := ledger.Bans(netblock)
-	if len(held) != 1 || held[0] != second ||
-		held[0].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
-		t.Errorf("the ledger holds %+v, want only the second ban, "+
-			"with 1 earlier ban for a limit", held)
+	if len(held) != 1 || held[0] != second || held[0].Notes.EarlierBans != 1 {
+		t.Errorf("the ledger holds %+v, want only the second ban, with 1 earlier ban",
+			held)
 	}
 }
 
@@ -304,13 +301,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
 		t.Fatal("the ban did not end")
 	}
 
-	// Its notes show the earlier ban for an attack that makes it permanent,
-	// beside the one for a limit.
 	third := ledger.BanForAttack(netblock, second.Expires.Add(30*day), bans.Notes{})
-	if !third.Permanent() ||
-		third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
+	if !third.Permanent() || third.Notes.EarlierBans != 2 {
 		t.Errorf("the next ban for an attack is %+v, want a permanent one, "+
-			"with 1 earlier ban for a limit and 1 for an attack", third)
+			"with 2 earlier bans", third)
 	}
 }
 

internal/bans/snapshot_test.go

@@ -82,9 +82,8 @@ func TestLoadedBansCarryOn(t *testing.T) {
 	}
 
 	again := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
-	if again.Expires.Sub(again.Start) != 3*time.Hour ||
-		again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
-		t.Errorf("the next ban lasts %s with earlier bans %+v, want 3h and 1 for a limit",
+	if again.Expires.Sub(again.Start) != 3*time.Hour || again.Notes.EarlierBans != 1 {
+		t.Errorf("the next ban lasts %s with %d earlier bans, want 3h and 1",
 			again.Expires.Sub(again.Start), again.Notes.EarlierBans)
 	}
 }
@@ -179,8 +178,7 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
 		Netblock: netblock,
 		Start:    midnight(),
 		Expires:  midnight().Add(9 * time.Hour),
-		Cause:    bans.CauseLimit,
-		Notes:    bans.Notes{EarlierBans: bans.EarlierBans{Limit: 2}},
+		Notes:    bans.Notes{EarlierBans: 2},
 	}
 	admins := bans.Ban{
 		Netblock: netblock,
@@ -193,13 +191,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
 
 	// Once both have ended, a limit broken within the repeat window bans
 	// for three times the 9 hours, and the notes count the two bans
-	// before the 9-hour one and it, for a limit, and the admin's, without
-	// a cause.
+	// before the 9-hour one, it, and the admin's.
 	ban := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{})
-	if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
-		ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, WithoutCause: 1}) {
-		t.Errorf("the next ban lasts %s with earlier bans %+v, "+
-			"want 27h, 3 for a limit and 1 without a cause",
+	if ban.Expires.Sub(ban.Start) != 27*time.Hour || ban.Notes.EarlierBans != 4 {
+		t.Errorf("the next ban lasts %s with %d earlier bans, want 27h and 4",
 			ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans)
 	}
 }

internal/proxy/bans_test.go

@@ -296,7 +296,7 @@ func TestBanNotes(t *testing.T) {
 			// refused under the ban.
 			Requests:    4,
 			Refused:     2,
-			EarlierBans: bans.EarlierBans{},
+			EarlierBans: 0,
 		},
 	}
 
@@ -313,8 +313,8 @@ func TestBanNotes(t *testing.T) {
 	s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
 
 	got = ledger.Bans(netblock)
-	if len(got) != 2 || got[1].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
-		t.Errorf("bans %+v, want two, the second with one earlier ban for a limit", got)
+	if len(got) != 2 || got[1].Notes.EarlierBans != 1 {
+		t.Errorf("bans %+v, want two, the second with one earlier ban", got)
 	}
 }
 

internal/rules/rules.go

@@ -327,7 +327,7 @@ func isTarget(target string) bool {
 // them once percent-decoded, so that an encoded probe cannot slip past.
 func (rule Rule) matches(r *http.Request) bool {
 	if rule.Target == "uri" {
-		uri := pathAndQuery(r)
+		uri := r.URL.RequestURI()
 
 		return rule.regex.MatchString(uri) || rule.regex.MatchString(decodeOnce(uri))
 	}
@@ -337,18 +337,14 @@ func (rule Rule) matches(r *http.Request) bool {
 
 // value returns what a rule with target, other than uri, is matched
 // against in r: the path and the query as the client sent them, before
-// any decoding or re-encoding, split at the first ?, and a header's values
-// joined by ", ", as HTTP joins those of a header sent more than once.
+// any decoding, and a header's values joined by ", ", as HTTP joins those
+// of a header sent more than once.
 func value(target string, r *http.Request) string {
 	switch target {
 	case "path":
-		path, _, _ := strings.Cut(pathAndQuery(r), "?")
-
-		return path
+		return r.URL.EscapedPath()
 	case "query":
-		_, query, _ := strings.Cut(pathAndQuery(r), "?")
-
-		return query
+		return r.URL.RawQuery
 	case "method":
 		return r.Method
 	case "host":
@@ -362,27 +358,6 @@ func value(target string, r *http.Request) string {
 	}
 }
 
-// pathAndQuery returns the path and the query of r as the client sent
-// them: the target of its request line, r.RequestURI, of which a target
-// in absolute form, http://host/path as a client sends it to a proxy,
-// gives what follows the host. r.URL is not used: when the path holds a
-// character it escapes, such as \ or a non-ASCII byte, it decodes the
-// whole path and escapes it again, so that \ becomes %5C and %2e a dot.
-func pathAndQuery(r *http.Request) string {
-	if !r.URL.IsAbs() {
-		return r.RequestURI
-	}
-
-	_, afterScheme, _ := strings.Cut(r.RequestURI, "://")
-
-	start := strings.IndexAny(afterScheme, "/?")
-	if start < 0 {
-		return ""
-	}
-
-	return afterScheme[start:]
-}
-
 // header returns the values of r's header name joined by ", ", or "" if
 // r has no such header.
 func header(r *http.Request, name string) string {

internal/rules/rules_test.go

@@ -95,31 +95,6 @@ func TestEachTargetMatchesWhatItNames(t *testing.T) {
 	}
 }
 
-func TestPathMatchedAsTheClientSentIt(t *testing.T) {
-	t.Parallel()
-
-	// Each path holds a character Go's URL type would escape again, \ or
-	// a non-ASCII byte, and each rule is written for the path as sent.
-	for _, tc := range []struct {
-		rule string // its target, action and regex
-		sent string // the path and query the client sent
-	}{
-		{`path log ^/\.\.\\\.\.\\windows\\win\.ini$`, `/..\..\windows\win.ini`},
-		{`path log ^/%2e%2e\\%2e%2e\\windows\\win\.ini$`, `/%2e%2e\%2e%2e\windows\win.ini`},
-		{`path log ^/café$`, "/café?x=1"},
-		{`uri log ^/%2e%2e\\%2e%2e\\boot\.ini\?x=1$`, `/%2e%2e\%2e%2e\boot.ini?x=1`},
-	} {
-		files := load(t, ruleFiles{testFile: "as-sent " + tc.rule + "\n"})
-
-		// The target in origin form, as traefik sends it, and in absolute
-		// form, as a client sends it to a proxy.
-		for _, target := range []string{tc.sent, "http://app.example" + tc.sent} {
-			r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
-			wantMatched(t, files, r, "as-sent")
-		}
-	}
-}
-
 func TestMatchingStopsAtTheFirstRuleThatRefuses(t *testing.T) {
 	t.Parallel()
 

internal/state/state_test.go

@@ -63,11 +63,7 @@ const permanentBansJSON = `{
                 },
                 "requests": 1500,
                 "refused": 3,
-                "earlier_bans": {
-                    "limit": 3,
-                    "attack": 1,
-                    "without_cause": 1
-                }
+                "earlier_bans": 5
             }
         }
     ]
@@ -958,7 +954,7 @@ func permanentBan() bans.Ban {
 			},
 			Requests:    1500,
 			Refused:     3,
-			EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, WithoutCause: 1},
+			EarlierBans: 5,
 		},
 	}
 }