Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h). The last good copy is kept
whole in reputation.json and used while a fetch fails and across restarts.
SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log
line names the lists, each raises reputation_hit, and a failed fetch raises
source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and
counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a restart fetches only lists whose copy is due.
Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

internal/state/state_test.go

@@ -27,15 +27,19 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/reputation"
 	"sneak.berlin/go/smallwebwaf/internal/state"
 )
 
 const (
 	// The state files.
-	bansJSON    = "bans.json"
-	clientsJSON = "clients.json"
-	lookupsJSON = "lookups.json"
-	alertsJSON  = "alerts.json"
+	bansJSON       = "bans.json"
+	clientsJSON    = "clients.json"
+	lookupsJSON    = "lookups.json"
+	reputationJSON = "reputation.json"
+	alertsJSON     = "alerts.json"
+	// blocklistURL is the blocklist the tests' lists name.
+	blocklistURL = "https://lists.example/drop.txt"
 	// The AS number and AS name the tests' clients are looked up in.
 	asn    = "AS64496"
 	asName = "Example Net"
@@ -204,6 +208,24 @@ const filledAlertsJSON = `{
 }
 `
 
+// filledReputationJSON is reputation.json holding the copy of the
+// blocklist fill puts in, with its comment line.
+const filledReputationJSON = `{
+    "version": 1,
+    "lists": [
+        {
+            "url": "https://lists.example/drop.txt",
+            "fetched": "2026-10-05T23:00:00Z",
+            "lines": [
+                "; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL",
+                "203.0.113.0/24 ; SBL1",
+                "2001:db8::/32 ; SBL2"
+            ]
+        }
+    ]
+}
+`
+
 func TestFilesWrittenAndReadBack(t *testing.T) {
 	t.Parallel()
 
@@ -230,6 +252,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
 	wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot())
 	wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot())
 
+	if got, want := after.Lists.Snapshot(), before.Lists.Snapshot(); !reflect.DeepEqual(
+		got, want) {
+		t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
+	}
+
 	if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
 		got, want) {
 		t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -238,12 +265,32 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
 	wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot())
 
 	// Each one-per-line file lists its entries by client, and nothing
-	// but the four files is left in the directory.
+	// but the five files is left in the directory.
 	wantEntries(t, filepath.Join(dir, clientsJSON), "clients",
 		"192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64")
 	wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups",
 		"192.0.2.1/32", "203.0.113.9/32")
-	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
+}
+
+func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	params := newParams(dir)
+	fill(params)
+
+	files := load(t, params)
+
+	err := files.WriteAll()
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	got := readFile(t, filepath.Join(dir, reputationJSON))
+	if got != filledReputationJSON {
+		t.Errorf("reputation.json\n%s\nwant\n%s", got, filledReputationJSON)
+	}
 }
 
 func TestAlertsJSONIsIndentedWithTheCooldownsTheHourAndTheAlertsWaiting(t *testing.T) {
@@ -368,8 +415,9 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 
 	held := params.Alerts.Snapshot()
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
-		len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
-		len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
+		len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
+		len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
+		held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -428,6 +476,13 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
 			`: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` +
 				`or watch`,
 		},
+		{
+			"a copy of a list with a line that does not read", reputationJSON,
+			`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
+				`"fetched": "2026-10-06T00:00:00Z", "lines": ["; DROP", "203.0.113.300"]}]}`,
+			`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
+				`such as 192.0.2.0/24`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -523,6 +578,42 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	}
 }
 
+func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	const fetched = `"fetched": "2026-10-06T00:00:00Z"`
+
+	for _, tc := range []struct {
+		name, content string
+		// want is what the error says after the file's path.
+		want string
+	}{
+		{
+			"a copy of a list without its URL",
+			`{"version": 1, "lists": [{` + fetched + `, "lines": []}]}`,
+			`: entry 1 has no "url"`,
+		},
+		{
+			"a copy of a list without the time it was fetched",
+			`{"version": 1, "lists": [{"url": "` + blocklistURL + `", "lines": []}]}`,
+			`: entry 1 has no "fetched"`,
+		},
+		{
+			// An empty list has no lines, which is not having none.
+			"a copy of a list without its lines",
+			`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` + fetched +
+				`, "lines": []}, {"url": "https://lists.example/tor.txt", ` + fetched + `}]}`,
+			`: entry 2 has no "lines"`,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			wantRefused(t, reputationJSON, tc.content, tc.want)
+		})
+	}
+}
+
 func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -626,7 +717,9 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
 func TestUnknownVersionStopsTheStart(t *testing.T) {
 	t.Parallel()
 
-	for _, file := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
+	for _, file := range []string{
+		bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
+	} {
 		for _, content := range []string{`{"version": 2}`, `{}`} {
 			t.Run(file+" "+content, func(t *testing.T) {
 				t.Parallel()
@@ -728,8 +821,9 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
 
 			time.Sleep(time.Nanosecond)
 			synctest.Wait()
-			wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
-			removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+			wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
+			removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON,
+				reputationJSON)
 		}
 	})
 }
@@ -948,7 +1042,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) {
 		t.Errorf("bans.json is now %v (%v), want the socket", info, err)
 	}
 
-	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
 	wantWriteFailed(t, params, bansJSON)
 }
 
@@ -1021,6 +1115,16 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(),
 		[]lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}})
 
+	edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
+		`"fetched": "2026-10-06T00:00:00Z", "lines": ["198.51.100.7"]}]}`)
+	wantTakenIn(t, lines, dir, reputationJSON)
+
+	listedBy := params.Lists.ListedBy(client.Addr())
+	if !slices.Equal(listedBy, []string{blocklistURL}) {
+		t.Errorf("%s taken in lists %s on %v, want on the blocklist", reputationJSON,
+			client.Addr(), listedBy)
+	}
+
 	// A netblock with bits past its length is read as the netblock it is
 	// in.
 	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1248,7 +1352,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 	edit(t, dir, bansJSON, broken)
 	edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`)
 	wantTakenIn(t, lines, dir, clientsJSON)
-	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
 
 	// The next write sets it aside, logged with where the error is, and
 	// writes bans.json again from what smallwebwaf still holds.
@@ -1272,7 +1376,8 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 		t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")
 	}
 
-	wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON,
+		reputationJSON)
 
 	if got := readFile(t, path+".bad"); got != broken {
 		t.Errorf("bans.json.bad holds\n%s\nwant the edit", got)
@@ -1401,9 +1506,10 @@ func midnight() time.Time {
 }
 
 // newParams returns Params for the state files in dir, with parts that
-// hold nothing yet. GeoJS is never asked, and the alerts, at most two an
-// hour, are never sent. The anomaly counters count the scopes fill
-// counts, with thresholds fill does not reach.
+// hold nothing yet. GeoJS is never asked, the lists, of which there is one
+// blocklist, are never fetched, and the alerts, at most two an hour, are
+// never sent. The anomaly counters count the scopes fill counts, with
+// thresholds fill does not reach.
 func newParams(dir string) state.Params {
 	discard := slog.New(slog.DiscardHandler)
 	m := metrics.New(1, "app")
@@ -1431,6 +1537,10 @@ func newParams(dir string) state.Params {
 		GeoJS: lookup.New(lookup.Params{
 			Now: midnight, ProcessLog: discard, Metrics: m,
 		}),
+		Lists: reputation.New(reputation.Params{
+			BlocklistURLs: []string{blocklistURL}, Refresh: 24 * time.Hour,
+			Now: midnight, ProcessLog: discard, Alerts: queue,
+		}),
 		Alerts: queue,
 		Anomalies: anomaly.New(anomaly.Params{
 			Net:            anomaly.Thresholds{RequestsPerMinute: 1000},
@@ -1455,8 +1565,9 @@ func office() netip.Prefix {
 
 // fill puts a permanent ban an admin made, a ban for a broken limit and
 // one for a clear sign of attack, clients with counts and histories,
-// GeoJS answers, and alerts and anomaly counters, as filledAlertsJSON
-// holds them, into the parts of params.
+// GeoJS answers, a copy of the blocklist, as filledReputationJSON holds
+// it, and alerts and anomaly counters, as filledAlertsJSON holds them,
+// into the parts of params.
 func fill(params state.Params) {
 	now := midnight()
 	client := netip.MustParsePrefix("203.0.113.9/32")
@@ -1489,6 +1600,18 @@ func fill(params state.Params) {
 		},
 	})
 
+	err := params.Lists.Load([]reputation.List{{
+		URL: blocklistURL, Fetched: now.Add(-time.Hour),
+		Lines: []string{
+			"; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL",
+			"203.0.113.0/24 ; SBL1",
+			"2001:db8::/32 ; SBL2",
+		},
+	}})
+	if err != nil {
+		panic(err) // the copy reads
+	}
+
 	// An alert waiting, a repeat of it the cooldown holds back, another
 	// alert waiting, and one past the two an hour, for the hour's summary.
 	ban := alerts.Alert{