Rule files, and bans for a clear sign of attack (closes #24)

Every *.rules file in SWWAF_RULES_DIR not named with a leading dot is
read at start, and again 2 seconds after the directory's last change.
Each request is checked against the rules after the rate limits: log
notes a match, block refuses with 403, ban refuses and bans the netblock
for SWWAF_ATTACK_BAN_DURATION, made permanent by its next request or
attack. path, query and uri are matched as the request line sent them;
header:Host and header:Transfer-Encoding are refused. Bans gain a cause.
The image ships 00-default.rules.

Judgement call: a header sent twice is matched with its values joined
by ", ".
Judgement call: SWWAF_MAX_BAN_DURATION does not cap a ban for an attack.
Not in this unit: offences for rule matches, with the error burst.

Model: opus-5-5

internal/proxy/rulefiles_test.go

@@ -0,0 +1,232 @@
+package proxy_test
+
+import (
+	"net/http"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// testRules are the rules most tests here load: a block rule for
+// /blocked and a ban rule for /.env.
+const testRules = `
+blocked  path  block  ^/blocked$
+probe    path  ban    ^/\.env$
+`
+
+func TestEachRuleAction(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server := startWithClock(t, "", map[string]string{
+		rulesDir:    writeRules(t, "noted path log ^/\n"+testRules),
+		banResponse: "429",
+	})
+	start := clk.Now()
+
+	// A log rule notes its match, and lets the request through.
+	line := s.get(client, http.StatusOK, requestlog.ActionForward)
+	wantRuleIDs(t, line, "noted")
+
+	// A block rule refuses with 403, whatever SWWAF_BAN_RESPONSE is, and
+	// bans no one.
+	line = s.request(client, "/blocked", http.StatusForbidden,
+		requestlog.ActionRuleBlocked)
+	wantRuleIDs(t, line, "noted", "blocked")
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+
+	// A ban rule refuses with SWWAF_BAN_RESPONSE, and bans the client for
+	// seven days, the default.
+	line = s.request(client, "/.env", http.StatusTooManyRequests, requestlog.ActionBanned)
+	wantRuleIDs(t, line, "noted", "probe")
+
+	if line.BanExpires != requestlog.FormatTime(start.Add(7*24*time.Hour)) {
+		t.Errorf("log line has ban_expires %q, want seven days on", line.BanExpires)
+	}
+
+	netblock := netip.MustParsePrefix(client + "/32")
+	want := bans.Ban{
+		Netblock: netblock,
+		Start:    start,
+		Expires:  start.Add(7 * 24 * time.Hour),
+		Cause:    bans.CauseAttack,
+		Notes: bans.Notes{
+			RuleID: "probe",
+			Target: "path",
+			Request: bans.Request{
+				Time:      start,
+				Method:    http.MethodGet,
+				Host:      appHost,
+				Path:      "/.env",
+				Status:    http.StatusTooManyRequests,
+				UserAgent: userAgent,
+			},
+			// The four requests up to and including the probe.
+			Requests: 4,
+		},
+	}
+
+	got := server.Ledger.Bans(netblock)
+	if len(got) != 1 || got[0] != want {
+		t.Fatalf("bans\n%+v\nwant\n%+v", got, want)
+	}
+
+	// The next request is refused under the ban, without being checked
+	// against the rules, and makes the ban permanent.
+	clk.advance(time.Hour)
+
+	line = s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
+	wantRuleIDs(t, line)
+
+	if line.BanExpires != permanent {
+		t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires)
+	}
+
+	clk.advance(365 * 24 * time.Hour)
+	s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
+}
+
+func TestNextClearSignOfAttackAfterABanBansPermanently(t *testing.T) {
+	t.Parallel()
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		rulesDir:          writeRules(t, testRules),
+		attackBanDuration: "1h",
+	})
+
+	// The first probe bans for SWWAF_ATTACK_BAN_DURATION.
+	line := s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	if line.BanExpires != requestlog.FormatTime(clk.Now().Add(time.Hour)) {
+		t.Errorf("log line has ban_expires %q, want an hour on", line.BanExpires)
+	}
+
+	// Once that ban has run out without a request, the client is served,
+	// and its next probe bans it for good.
+	clk.advance(time.Hour)
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+
+	line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	if line.BanExpires != permanent {
+		t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires)
+	}
+}
+
+func TestRulesComeAfterTheOtherChecks(t *testing.T) {
+	t.Parallel()
+
+	const (
+		allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS
+		exempt  = "192.0.2.50" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
+	)
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		rulesDir:            writeRules(t, testRules),
+		allowNets:           allowed,
+		rateLimitExemptNets: exempt,
+		rateLimitPerMinute:  "1",
+	})
+
+	// A client in SWWAF_ALLOW_NETS is not checked.
+	line := s.request(allowed, "/.env", http.StatusOK, requestlog.ActionForward)
+	wantRuleIDs(t, line)
+
+	// A probe over the rate limit breaks the limit before any rule sees
+	// it.
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionRateLimited)
+	wantRuleIDs(t, line)
+
+	limitBan := server.Ledger.Bans(netip.MustParsePrefix(client + "/32"))
+	if len(limitBan) != 1 || limitBan[0].Cause != bans.CauseLimit {
+		t.Errorf("bans %+v, want one for a broken limit", limitBan)
+	}
+
+	// A client the rate limits do not apply to is still checked.
+	s.get(exempt, http.StatusOK, requestlog.ActionForward)
+	s.request(exempt, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+}
+
+func TestObserveModeLogsWhatTheRulesWouldDo(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		rulesDir: writeRules(t, testRules),
+		mode:     observe,
+	})
+
+	line := s.request(client, "/blocked", http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionRuleBlocked)
+	wantRuleIDs(t, line, "blocked")
+
+	line = s.request(client, "/.env", http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionBanned)
+	wantRuleIDs(t, line, "probe")
+
+	if line.BanExpires != "" {
+		t.Errorf("log line has ban_expires %q, want none", line.BanExpires)
+	}
+
+	// No ban was made.
+	line = s.get(client, http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, "")
+
+	if got := server.Ledger.Snapshot(); len(got) != 0 {
+		t.Errorf("bans %+v, want none", got)
+	}
+}
+
+func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) {
+	t.Parallel()
+
+	const scraper = "192.0.2.200"
+
+	s, _, _ := startWithClock(t, "", map[string]string{
+		rulesDir:     writeRules(t, testRules),
+		metricsToken: token,
+	})
+
+	s.request(client, "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked)
+	s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	metrics := s.scrape(scraper)
+	wantMetric(t, metrics,
+		`smallwebwaf_rule_matches_total{action="block",rule_id="blocked"}`, 1)
+	wantMetric(t, metrics,
+		`smallwebwaf_rule_matches_total{action="ban",rule_id="probe"}`, 1)
+	wantMetric(t, metrics, "smallwebwaf_rules_loaded", 2)
+	wantMetric(t, metrics,
+		`smallwebwaf_requests_total{action="rule_blocked",status_class="4xx"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="attack"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 0)
+	wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1)
+}
+
+// writeRules writes content as a rule file into a new directory, and
+// returns the directory.
+func writeRules(t *testing.T, content string) string {
+	t.Helper()
+
+	dir := t.TempDir()
+
+	err := os.WriteFile(filepath.Join(dir, "test.rules"), []byte(content), 0o600)
+	if err != nil {
+		t.Fatalf("write the rule file: %v", err)
+	}
+
+	return dir
+}
+
+// wantRuleIDs checks the request log line's rule_ids.
+func wantRuleIDs(t *testing.T, line logLine, want ...string) {
+	t.Helper()
+
+	if !slices.Equal(line.RuleIDs, want) {
+		t.Errorf("log line has rule_ids %v, want %v", line.RuleIDs, want)
+	}
+}