Rule files, and bans for a clear sign of attack (closes #24)

Every *.rules file in SWWAF_RULES_DIR is read at start and on each
change, and each request is checked against the rules after the rate
limits: log notes a match, block refuses with 403, ban refuses and bans
the netblock for SWWAF_ATTACK_BAN_DURATION, made permanent by its next
request or clear sign of attack. path, query and uri are matched as the
request line sent them. bans.json gains each ban's cause, and ban notes
count earlier bans by cause. The image ships 00-default.rules.

Judgement call: a header sent twice is matched with its values joined
by ", ".
Judgement call: SWWAF_MAX_BAN_DURATION does not cap a ban for an attack.
Not in this unit: offences for rule matches, with the error burst.

Model: opus-5-5

README.md

@@ -103,11 +103,11 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of
   window and the requests counted in it, the request that broke it, the client's
   country when it was looked up, the netblock's requests since it was first
   seen, how many of them the ban has refused, and how many bans the netblock had
-  before. At most `SWWAF_MAX_BANS` bans are kept, past, active and permanent;
+  before, for a broken limit, for a clear sign of attack and without a cause. At
-  past that, the earliest ban of the netblock that has gone longest without a
+  most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; past that,
-  request is dropped first. `bans.json` shows the bans and their notes, a
+  the earliest ban of the netblock that has gone longest without a request is
-  restart lifts none, and you add or lift a ban by editing it (see "State files"
+  dropped first. `bans.json` shows the bans and their notes, a restart lifts
-  below).
+  none, and you add or lift a ban by editing it (see "State files" below).
 - Checks each request against the rules of the rule files (see "Rule files"
   below) after the rate limits, and before its body is read. A `log` rule that
   matches is noted in the log line; a `block` rule refuses the request with

internal/bans/bans.go

@@ -101,8 +101,19 @@ type Notes struct {
 	// far. Both go up with each request the ban refuses.
 	Requests int64 `json:"requests"`
 	Refused  int64 `json:"refused"`
-	// EarlierBans is how many bans the netblock had before this one.
+	// EarlierBans is how many bans the netblock had before this one, by
-	EarlierBans int `json:"earlier_bans"`
+	// cause.
+	EarlierBans EarlierBans `json:"earlier_bans"`
+}
+
+// EarlierBans counts a netblock's bans before a ban, by cause.
+//
+//nolint:tagliatelle // the state files use snake_case, as the request log does
+type EarlierBans struct {
+	Limit  int `json:"limit"`
+	Attack int `json:"attack"`
+	// WithoutCause counts the bans an admin added without a cause.
+	WithoutCause int `json:"without_cause"`
 }
 
 // Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -348,10 +359,7 @@ func (l *Ledger) ban(
 		}
 
 		held = *bans
-
+		notes.EarlierBans = earlierBans(held)
-		// The netblock's first ban held counts the bans it had before that
-		// one, since dropped to make room, and each ban held adds one.
-		notes.EarlierBans = (*bans)[0].Notes.EarlierBans + len(*bans)
 	}
 
 	notes.Request = notes.Request.cut()
@@ -370,6 +378,27 @@ func (l *Ledger) ban(
 	return ban
 }
 
+// earlierBans returns how many bans a netblock with the bans held, oldest
+// first, has had, by cause: the first ban held counts the bans the
+// netblock had before that one, since dropped to make room, and each ban
+// held adds one.
+func earlierBans(held []Ban) EarlierBans {
+	earlier := held[0].Notes.EarlierBans
+
+	for _, ban := range held {
+		switch ban.Cause {
+		case CauseLimit:
+			earlier.Limit++
+		case CauseAttack:
+			earlier.Attack++
+		default:
+			earlier.WithoutCause++
+		}
+	}
+
+	return earlier
+}
+
 // markChanged has Changed receive a value, unless one is waiting already.
 func (l *Ledger) markChanged() {
 	select {

internal/bans/bans_test.go

@@ -24,8 +24,9 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
 		ban := ledger.BanForLimit(netblock, now, bans.Notes{})
 
 		length := time.Duration(hours) * time.Hour
-		if !ban.Expires.Equal(now.Add(length)) || ban.Notes.EarlierBans != i {
+		if !ban.Expires.Equal(now.Add(length)) ||
-			t.Fatalf("ban %d lasts %s with %d earlier bans, want %d hours and %d",
+			ban.Notes.EarlierBans != (bans.EarlierBans{Limit: i}) {
+			t.Fatalf("ban %d lasts %s with earlier bans %+v, want %d hours and %d for a limit",
 				i+1, ban.Expires.Sub(now), ban.Notes.EarlierBans, hours, i)
 		}
 
@@ -66,8 +67,9 @@ func TestRepeatWindowRunsOut(t *testing.T) {
 			first := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
 			second := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{})
 
-			if second.Expires.Sub(second.Start) != tc.want || second.Notes.EarlierBans != 1 {
+			if second.Expires.Sub(second.Start) != tc.want ||
-				t.Errorf("second ban lasts %s with %d earlier bans, want %s and 1",
+				second.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+				t.Errorf("second ban lasts %s with earlier bans %+v, want %s and 1 for a limit",
 					second.Expires.Sub(second.Start), second.Notes.EarlierBans, tc.want)
 			}
 		})
@@ -235,9 +237,10 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
 	second := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
 
 	held := ledger.Bans(netblock)
-	if len(held) != 1 || held[0] != second || held[0].Notes.EarlierBans != 1 {
+	if len(held) != 1 || held[0] != second ||
-		t.Errorf("the ledger holds %+v, want only the second ban, with 1 earlier ban",
+		held[0].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
-			held)
+		t.Errorf("the ledger holds %+v, want only the second ban, "+
+			"with 1 earlier ban for a limit", held)
 	}
 }
 
@@ -301,10 +304,13 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
 		t.Fatal("the ban did not end")
 	}
 
+	// Its notes show the earlier ban for an attack that makes it permanent,
+	// beside the one for a limit.
 	third := ledger.BanForAttack(netblock, second.Expires.Add(30*day), bans.Notes{})
-	if !third.Permanent() || third.Notes.EarlierBans != 2 {
+	if !third.Permanent() ||
+		third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
 		t.Errorf("the next ban for an attack is %+v, want a permanent one, "+
-			"with 2 earlier bans", third)
+			"with 1 earlier ban for a limit and 1 for an attack", third)
 	}
 }
 

internal/bans/snapshot_test.go

@@ -82,8 +82,9 @@ func TestLoadedBansCarryOn(t *testing.T) {
 	}
 
 	again := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
-	if again.Expires.Sub(again.Start) != 3*time.Hour || again.Notes.EarlierBans != 1 {
+	if again.Expires.Sub(again.Start) != 3*time.Hour ||
-		t.Errorf("the next ban lasts %s with %d earlier bans, want 3h and 1",
+		again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+		t.Errorf("the next ban lasts %s with earlier bans %+v, want 3h and 1 for a limit",
 			again.Expires.Sub(again.Start), again.Notes.EarlierBans)
 	}
 }
@@ -178,7 +179,8 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
 		Netblock: netblock,
 		Start:    midnight(),
 		Expires:  midnight().Add(9 * time.Hour),
-		Notes:    bans.Notes{EarlierBans: 2},
+		Cause:    bans.CauseLimit,
+		Notes:    bans.Notes{EarlierBans: bans.EarlierBans{Limit: 2}},
 	}
 	admins := bans.Ban{
 		Netblock: netblock,
@@ -191,10 +193,13 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
 
 	// Once both have ended, a limit broken within the repeat window bans
 	// for three times the 9 hours, and the notes count the two bans
-	// before the 9-hour one, it, and the admin's.
+	// before the 9-hour one and it, for a limit, and the admin's, without
+	// a cause.
 	ban := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{})
-	if ban.Expires.Sub(ban.Start) != 27*time.Hour || ban.Notes.EarlierBans != 4 {
+	if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
-		t.Errorf("the next ban lasts %s with %d earlier bans, want 27h and 4",
+		ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, WithoutCause: 1}) {
+		t.Errorf("the next ban lasts %s with earlier bans %+v, "+
+			"want 27h, 3 for a limit and 1 without a cause",
 			ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans)
 	}
 }

internal/proxy/bans_test.go

@@ -296,7 +296,7 @@ func TestBanNotes(t *testing.T) {
 			// refused under the ban.
 			Requests:    4,
 			Refused:     2,
-			EarlierBans: 0,
+			EarlierBans: bans.EarlierBans{},
 		},
 	}
 
@@ -313,8 +313,8 @@ func TestBanNotes(t *testing.T) {
 	s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
 
 	got = ledger.Bans(netblock)
-	if len(got) != 2 || got[1].Notes.EarlierBans != 1 {
+	if len(got) != 2 || got[1].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
-		t.Errorf("bans %+v, want two, the second with one earlier ban", got)
+		t.Errorf("bans %+v, want two, the second with one earlier ban for a limit", got)
 	}
 }
 

internal/rules/rules.go

@@ -327,7 +327,7 @@ func isTarget(target string) bool {
 // them once percent-decoded, so that an encoded probe cannot slip past.
 func (rule Rule) matches(r *http.Request) bool {
 	if rule.Target == "uri" {
-		uri := r.URL.RequestURI()
+		uri := pathAndQuery(r)
 
 		return rule.regex.MatchString(uri) || rule.regex.MatchString(decodeOnce(uri))
 	}
@@ -337,14 +337,18 @@ func (rule Rule) matches(r *http.Request) bool {
 
 // value returns what a rule with target, other than uri, is matched
 // against in r: the path and the query as the client sent them, before
-// any decoding, and a header's values joined by ", ", as HTTP joins those
+// any decoding or re-encoding, split at the first ?, and a header's values
-// of a header sent more than once.
+// joined by ", ", as HTTP joins those of a header sent more than once.
 func value(target string, r *http.Request) string {
 	switch target {
 	case "path":
-		return r.URL.EscapedPath()
+		path, _, _ := strings.Cut(pathAndQuery(r), "?")
+
+		return path
 	case "query":
-		return r.URL.RawQuery
+		_, query, _ := strings.Cut(pathAndQuery(r), "?")
+
+		return query
 	case "method":
 		return r.Method
 	case "host":
@@ -358,6 +362,27 @@ func value(target string, r *http.Request) string {
 	}
 }
 
+// pathAndQuery returns the path and the query of r as the client sent
+// them: the target of its request line, r.RequestURI, of which a target
+// in absolute form, http://host/path as a client sends it to a proxy,
+// gives what follows the host. r.URL is not used: when the path holds a
+// character it escapes, such as \ or a non-ASCII byte, it decodes the
+// whole path and escapes it again, so that \ becomes %5C and %2e a dot.
+func pathAndQuery(r *http.Request) string {
+	if !r.URL.IsAbs() {
+		return r.RequestURI
+	}
+
+	_, afterScheme, _ := strings.Cut(r.RequestURI, "://")
+
+	start := strings.IndexAny(afterScheme, "/?")
+	if start < 0 {
+		return ""
+	}
+
+	return afterScheme[start:]
+}
+
 // header returns the values of r's header name joined by ", ", or "" if
 // r has no such header.
 func header(r *http.Request, name string) string {

internal/rules/rules_test.go

@@ -95,6 +95,31 @@ func TestEachTargetMatchesWhatItNames(t *testing.T) {
 	}
 }
 
+func TestPathMatchedAsTheClientSentIt(t *testing.T) {
+	t.Parallel()
+
+	// Each path holds a character Go's URL type would escape again, \ or
+	// a non-ASCII byte, and each rule is written for the path as sent.
+	for _, tc := range []struct {
+		rule string // its target, action and regex
+		sent string // the path and query the client sent
+	}{
+		{`path log ^/\.\.\\\.\.\\windows\\win\.ini$`, `/..\..\windows\win.ini`},
+		{`path log ^/%2e%2e\\%2e%2e\\windows\\win\.ini$`, `/%2e%2e\%2e%2e\windows\win.ini`},
+		{`path log ^/café$`, "/café?x=1"},
+		{`uri log ^/%2e%2e\\%2e%2e\\boot\.ini\?x=1$`, `/%2e%2e\%2e%2e\boot.ini?x=1`},
+	} {
+		files := load(t, ruleFiles{testFile: "as-sent " + tc.rule + "\n"})
+
+		// The target in origin form, as traefik sends it, and in absolute
+		// form, as a client sends it to a proxy.
+		for _, target := range []string{tc.sent, "http://app.example" + tc.sent} {
+			r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+			wantMatched(t, files, r, "as-sent")
+		}
+	}
+}
+
 func TestMatchingStopsAtTheFirstRuleThatRefuses(t *testing.T) {
 	t.Parallel()
 

internal/state/state_test.go

@@ -63,7 +63,11 @@ const permanentBansJSON = `{
                 },
                 "requests": 1500,
                 "refused": 3,
-                "earlier_bans": 5
+                "earlier_bans": {
+                    "limit": 3,
+                    "attack": 1,
+                    "without_cause": 1
+                }
             }
         }
     ]
@@ -954,7 +958,7 @@ func permanentBan() bans.Ban {
 			},
 			Requests:    1500,
 			Refused:     3,
-			EarlierBans: 5,
+			EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, WithoutCause: 1},
 		},
 	}
 }