Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands
for its IPv4 address or netblock. reputation.json keeps each list's last
try, failed or not, even one cut off by a stop, which a restart waits on
as a running instance does, and its last good copy, whole, used while a
fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed
client; the log line names the lists, each raises reputation_hit, and a
failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched
the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

internal/state/state_test.go

@@ -27,15 +27,20 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/reputation"
 	"sneak.berlin/go/smallwebwaf/internal/state"
 )
 
 const (
 	// The state files.
-	bansJSON    = "bans.json"
-	clientsJSON = "clients.json"
-	lookupsJSON = "lookups.json"
-	alertsJSON  = "alerts.json"
+	bansJSON       = "bans.json"
+	clientsJSON    = "clients.json"
+	lookupsJSON    = "lookups.json"
+	reputationJSON = "reputation.json"
+	alertsJSON     = "alerts.json"
+	// blocklistURL and torURL are the blocklists the tests' lists name.
+	blocklistURL = "https://lists.example/drop.txt"
+	torURL       = "https://lists.example/tor.txt"
 	// The AS number and AS name the tests' clients are looked up in.
 	asn    = "AS64496"
 	asName = "Example Net"
@@ -204,6 +209,29 @@ const filledAlertsJSON = `{
 }
 `
 
+// filledReputationJSON is reputation.json holding the blocklists' last
+// tries and the copy of one, with its comment line, as fill puts them in.
+const filledReputationJSON = `{
+    "version": 1,
+    "lists": [
+        {
+            "url": "https://lists.example/drop.txt",
+            "tried": "2026-10-06T00:00:00Z",
+            "fetched": "2026-10-05T23:00:00Z",
+            "lines": [
+                "; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL",
+                "203.0.113.0/24 ; SBL1",
+                "2001:db8::/32 ; SBL2"
+            ]
+        },
+        {
+            "url": "https://lists.example/tor.txt",
+            "tried": "2026-10-06T00:00:00Z"
+        }
+    ]
+}
+`
+
 func TestFilesWrittenAndReadBack(t *testing.T) {
 	t.Parallel()
 
@@ -230,6 +258,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
 	wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot())
 	wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot())
 
+	if got, want := after.Lists.Snapshot(), before.Lists.Snapshot(); !reflect.DeepEqual(
+		got, want) {
+		t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
+	}
+
 	if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
 		got, want) {
 		t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -238,12 +271,32 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
 	wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot())
 
 	// Each one-per-line file lists its entries by client, and nothing
-	// but the four files is left in the directory.
+	// but the five files is left in the directory.
 	wantEntries(t, filepath.Join(dir, clientsJSON), "clients",
 		"192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64")
 	wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups",
 		"192.0.2.1/32", "203.0.113.9/32")
-	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
+}
+
+func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	params := newParams(dir)
+	fill(params)
+
+	files := load(t, params)
+
+	err := files.WriteAll()
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	got := readFile(t, filepath.Join(dir, reputationJSON))
+	if got != filledReputationJSON {
+		t.Errorf("reputation.json\n%s\nwant\n%s", got, filledReputationJSON)
+	}
 }
 
 func TestAlertsJSONIsIndentedWithTheCooldownsTheHourAndTheAlertsWaiting(t *testing.T) {
@@ -368,8 +421,9 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 
 	held := params.Alerts.Snapshot()
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
-		len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
-		len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
+		len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
+		len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
+		held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -428,6 +482,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
 			`: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` +
 				`or watch`,
 		},
+		{
+			"a copy of a list with a line that does not read", reputationJSON,
+			`{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` +
+				`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` +
+				`"lines": ["; DROP", "203.0.113.300"]}]}`,
+			`: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` +
+				`such as 192.0.2.0/24`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -523,6 +585,51 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	}
 }
 
+func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	const (
+		drop    = `"url": "` + blocklistURL + `", `
+		tried   = `"tried": "2026-10-06T00:00:00Z", `
+		fetched = `"fetched": "2026-10-06T00:00:00Z"`
+	)
+
+	for _, tc := range []struct {
+		name, content string
+		// want is what the error says after the file's path.
+		want string
+	}{
+		{
+			"a list without its URL",
+			`{"version": 1, "lists": [{` + tried + fetched + `, "lines": []}]}`,
+			`: entry 1 has no "url"`,
+		},
+		{
+			"a list without the time it was last tried",
+			`{"version": 1, "lists": [{` + drop + fetched + `, "lines": []}]}`,
+			`: entry 1 has no "tried"`,
+		},
+		{
+			"a copy of a list without the time it was fetched",
+			`{"version": 1, "lists": [{` + drop + tried + `"lines": []}]}`,
+			`: entry 1 has no "fetched"`,
+		},
+		{
+			// An empty list has no lines, which is not having none.
+			"a copy of a list without its lines",
+			`{"version": 1, "lists": [{` + drop + tried + fetched + `, "lines": []}, ` +
+				`{"url": "` + torURL + `", ` + tried + fetched + `}]}`,
+			`: entry 2 has no "lines"`,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			wantRefused(t, reputationJSON, tc.content, tc.want)
+		})
+	}
+}
+
 func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -626,7 +733,9 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
 func TestUnknownVersionStopsTheStart(t *testing.T) {
 	t.Parallel()
 
-	for _, file := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
+	for _, file := range []string{
+		bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON,
+	} {
 		for _, content := range []string{`{"version": 2}`, `{}`} {
 			t.Run(file+" "+content, func(t *testing.T) {
 				t.Parallel()
@@ -728,8 +837,9 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
 
 			time.Sleep(time.Nanosecond)
 			synctest.Wait()
-			wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
-			removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+			wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
+			removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON,
+				reputationJSON)
 		}
 	})
 }
@@ -948,7 +1058,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) {
 		t.Errorf("bans.json is now %v (%v), want the socket", info, err)
 	}
 
-	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
 	wantWriteFailed(t, params, bansJSON)
 }
 
@@ -1021,6 +1131,17 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(),
 		[]lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}})
 
+	edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
+		`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
+		`"lines": ["198.51.100.7"]}]}`)
+	wantTakenIn(t, lines, dir, reputationJSON)
+
+	listedBy := params.Lists.ListedBy(client.Addr())
+	if !slices.Equal(listedBy, []string{blocklistURL}) {
+		t.Errorf("%s taken in lists %s on %v, want on the blocklist", reputationJSON,
+			client.Addr(), listedBy)
+	}
+
 	// A netblock with bits past its length is read as the netblock it is
 	// in.
 	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1248,7 +1369,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 	edit(t, dir, bansJSON, broken)
 	edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`)
 	wantTakenIn(t, lines, dir, clientsJSON)
-	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON)
 
 	// The next write sets it aside, logged with where the error is, and
 	// writes bans.json again from what smallwebwaf still holds.
@@ -1272,7 +1393,8 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 		t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")
 	}
 
-	wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON,
+		reputationJSON)
 
 	if got := readFile(t, path+".bad"); got != broken {
 		t.Errorf("bans.json.bad holds\n%s\nwant the edit", got)
@@ -1401,9 +1523,10 @@ func midnight() time.Time {
 }
 
 // newParams returns Params for the state files in dir, with parts that
-// hold nothing yet. GeoJS is never asked, and the alerts, at most two an
-// hour, are never sent. The anomaly counters count the scopes fill
-// counts, with thresholds fill does not reach.
+// hold nothing yet. GeoJS is never asked, the lists, two blocklists, are
+// never fetched, and the alerts, at most two an hour, are
+// never sent. The anomaly counters count the scopes fill counts, with
+// thresholds fill does not reach.
 func newParams(dir string) state.Params {
 	discard := slog.New(slog.DiscardHandler)
 	m := metrics.New(1, "app")
@@ -1431,6 +1554,10 @@ func newParams(dir string) state.Params {
 		GeoJS: lookup.New(lookup.Params{
 			Now: midnight, ProcessLog: discard, Metrics: m,
 		}),
+		Lists: reputation.New(reputation.Params{
+			BlocklistURLs: []string{blocklistURL, torURL}, Refresh: 24 * time.Hour,
+			Now: midnight, ProcessLog: discard, Alerts: queue,
+		}),
 		Alerts: queue,
 		Anomalies: anomaly.New(anomaly.Params{
 			Net:            anomaly.Thresholds{RequestsPerMinute: 1000},
@@ -1455,8 +1582,9 @@ func office() netip.Prefix {
 
 // fill puts a permanent ban an admin made, a ban for a broken limit and
 // one for a clear sign of attack, clients with counts and histories,
-// GeoJS answers, and alerts and anomaly counters, as filledAlertsJSON
-// holds them, into the parts of params.
+// GeoJS answers, the blocklists' last tries and the copy of one, as
+// filledReputationJSON holds them, and alerts and anomaly counters, as
+// filledAlertsJSON holds them, into the parts of params.
 func fill(params state.Params) {
 	now := midnight()
 	client := netip.MustParsePrefix("203.0.113.9/32")
@@ -1489,6 +1617,20 @@ func fill(params state.Params) {
 		},
 	})
 
+	// The copy of drop.txt was fetched an hour ago, and the fetches of it
+	// and of tor.txt tried since failed.
+	err := params.Lists.Load([]reputation.List{{
+		URL: blocklistURL, Tried: now, Fetched: now.Add(-time.Hour),
+		Lines: []string{
+			"; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL",
+			"203.0.113.0/24 ; SBL1",
+			"2001:db8::/32 ; SBL2",
+		},
+	}, {URL: torURL, Tried: now}})
+	if err != nil {
+		panic(err) // the copy reads
+	}
+
 	// An alert waiting, a repeat of it the cooldown holds back, another
 	// alert waiting, and one past the two an hour, for the hour's summary.
 	ban := alerts.Alert{