Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands
for its IPv4 address or netblock. reputation.json keeps each list's last
try, failed or not, even one cut off by a stop, which a restart waits on
as a running instance does, and its last good copy, whole, used while a
fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed
client; the log line names the lists, each raises reputation_hit, and a
failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched
the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

internal/smallwebwaf/smallwebwaf.go

@@ -200,6 +200,7 @@ func loadStateFiles(
 		Ledger:          server.Ledger,
 		Limiter:         server.Limiter,
 		GeoJS:           server.GeoJS,
+		Lists:           server.Lists,
 		Alerts:          alertQueue,
 		Anomalies:       server.Anomalies,
 		Now:             now,
@@ -267,8 +268,9 @@ func startSending(
 
 // serve serves requests on listener, writes the state files as they are
 // due, takes in an admin's edits of them, reads the rule files again as
-// they change, and the lookup database when it is replaced, and sends the
-// alerts, until ctx is done. Then it gives the requests in progress
+// they change, and the lookup database when it is replaced, fetches the
+// lists the settings name by URL as they are due, and sends the alerts,
+// until ctx is done. Then it gives the requests in progress
 // shutdownTimeout to finish, and writes every state file, alerts.json with
 // the alerts still waiting.
 func serve(
@@ -293,6 +295,7 @@ func serve(
 			server.LookupFile.Watch(writing)
 		}
 	})
+	listsFetched := inBackground(func() { server.Lists.Run(writing) })
 	alertsSent := inBackground(func() { alertQueue.Run(writing) })
 
 	select {
@@ -336,6 +339,7 @@ func serve(
 	<-watched
 	<-rulesWatched
 	<-lookupFileWatched
+	<-listsFetched
 	<-alertsSent
 
 	err = files.WriteAll()

internal/smallwebwaf/smallwebwaf_test.go

@@ -38,6 +38,7 @@ const (
 	stateWriteDelay      = "SWWAF_STATE_WRITE_DELAY"
 	stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
 	rateLimitPerDay      = "SWWAF_RATE_LIMIT_PER_DAY"
+	rateLimitExemptNets  = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
 	rulesDir             = "SWWAF_RULES_DIR"
 	lookupSource         = "SWWAF_LOOKUP_SOURCE"
 	lookupDBPath         = "SWWAF_LOOKUP_DB_PATH"
@@ -522,7 +523,7 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
 		// The requests sent until a replacement takes effect, and those
 		// for the metrics, must not break a rate limit, whose ban would
 		// refuse them too.
-		"SWWAF_RATE_LIMIT_EXEMPT_NETS": placed + "," + localhost,
+		rateLimitExemptNets: placed + "," + localhost,
 	}
 	began := time.Now()
 	// Each replacement is written beside the file and renamed over it, as
@@ -573,6 +574,111 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
 	}
 }
 
+func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T) {
+	t.Parallel()
+
+	const (
+		token    = "0123456789abcdef0123456789abcdef"
+		torPath  = "/tor.txt"
+		dropPath = "/drop.txt"
+		// copyright is the DROP list's date and copyright line.
+		copyright = "; Spamhaus DROP List 2026/10/07 - (c) 2026 The Spamhaus Project SLL"
+	)
+
+	failing, torFetches := new(atomic.Bool), new(atomic.Int32)
+	lists := map[string]string{
+		torPath:  "198.51.100.0/24\n",
+		dropPath: copyright + "\n" + placed + " ; SBL1\n",
+	}
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == torPath {
+				torFetches.Add(1)
+			}
+
+			if failing.Load() {
+				w.WriteHeader(http.StatusServiceUnavailable)
+
+				return
+			}
+
+			_, _ = io.WriteString(w, lists[r.URL.Path])
+		}))
+	t.Cleanup(server.Close)
+
+	torURL, dropURL := server.URL+torPath, server.URL+dropPath
+	dir := t.TempDir()
+	env := map[string]string{
+		listenAddr:             localhost + ":0",
+		upstreamURL:            startApp(t),
+		stateDir:               dir,
+		rulesDir:               t.TempDir(),
+		trustedProxies:         localhost + "/32",
+		metricsToken:           token,
+		instanceName:           instance,
+		"SWWAF_BLOCKLIST_URLS": torURL,
+		// The requests sent until the list takes effect, and those for the
+		// metrics, must not break a rate limit, whose ban would refuse them
+		// too.
+		rateLimitExemptNets: placed + "," + localhost,
+	}
+	failures := `smallwebwaf_reputation_failures_total{instance="fsn1app1/gitea",` +
+		`source="` + torURL + `"} `
+
+	// tor.txt cannot be fetched at first, which is counted.
+	failing.Store(true)
+	runUntilStopped(t, env, func(url string) {
+		metricsWith(t, url+"_smallwebwaf/metrics", token, failures+"1")
+	})
+
+	// Restarted with drop.txt named after it and the server answering, tor.txt
+	// waits SWWAF_BLOCKLIST_REFRESH after its failed try, kept in reputation.json,
+	// while drop.txt, never tried, is fetched at once. Lists are fetched in the
+	// order named, so once drop.txt refuses the client, tor.txt has had its turn.
+	failing.Store(false)
+
+	env["SWWAF_BLOCKLIST_URLS"] = torURL + "," + dropURL
+	out := runUntilStopped(t, env, func(url string) {
+		for statusFrom(t, url, placed) != http.StatusForbidden {
+			time.Sleep(pollInterval)
+		}
+	})
+	wantDeniedByList(t, out.line(t, "action", "denied"), dropURL)
+
+	if fetches := torFetches.Load(); fetches != 1 {
+		t.Errorf("tor.txt fetched %d times, want once, before the restart", fetches)
+	}
+
+	// After another restart, with the server failing, the copy of drop.txt
+	// kept in reputation.json, its copyright line included, refuses the
+	// client from the first request.
+	failing.Store(true)
+
+	out = runUntilStopped(t, env, func(url string) {
+		wantStatus(t, url, placed, http.StatusForbidden)
+	})
+	wantDeniedByList(t, out.line(t, "type", "request"), dropURL)
+
+	path := filepath.Join(dir, "reputation.json")
+
+	kept, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
+	if err != nil || !strings.Contains(string(kept), `"`+copyright+`"`) {
+		t.Errorf("reputation.json holds\n%s\nwant the copy with %q (%v)", kept, copyright,
+			err)
+	}
+}
+
+// wantDeniedByList checks that the request log line is of a request the
+// blocklist at listURL refused.
+func wantDeniedByList(t *testing.T, line map[string]any, listURL string) {
+	t.Helper()
+
+	reputation, _ := line["reputation"].([]any)
+	if line["action"] != "denied" || len(reputation) != 1 || reputation[0] != listURL {
+		t.Errorf("request log line %v, want one denied for %s", line, listURL)
+	}
+}
+
 func TestLookupDatabaseThatCannotBeReadStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -1014,7 +1120,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) {
 		"SWWAF_REQUEST_MAX_BYTES":               "100M",
 		"SWWAF_RESPONSE_MAX_BYTES":              "5G",
 		"SWWAF_ALLOW_NETS":                      "",
-		"SWWAF_RATE_LIMIT_EXEMPT_NETS":          "",
+		rateLimitExemptNets:                     "",
 		"SWWAF_DENY_NETS":                       "",
 		"SWWAF_RATE_LIMIT_PER_MINUTE":           "1000",
 		"SWWAF_RATE_LIMIT_PER_HOUR":             "10000",