Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands
for its IPv4 address or netblock. reputation.json keeps each list's last
try, failed or not, which a restart waits on as a running instance does,
and its last good copy, whole, used while a fetch fails.
SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log
line names the lists, each raises reputation_hit, and a failed fetch raises
source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and
counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

internal/config/config_test.go

@@ -57,6 +57,10 @@ const (
 	asnBytesPercent         = "SWWAF_ASN_BYTES_PERCENT"
 	countryBytesPercent     = "SWWAF_COUNTRY_BYTES_PERCENT"
 	unknownLimitPercent     = "SWWAF_UNKNOWN_LIMIT_PERCENT"
+	asnLimitPercentURL      = "SWWAF_ASN_LIMIT_PERCENT_URL"
+	blocklistURLs           = "SWWAF_BLOCKLIST_URLS"
+	blocklistRefresh        = "SWWAF_BLOCKLIST_REFRESH"
+	blocklistAction         = "SWWAF_BLOCKLIST_ACTION"
 	banResponse             = "SWWAF_BAN_RESPONSE"
 	limitBanDuration        = "SWWAF_LIMIT_BAN_DURATION"
 	limitBanRepeatWindow    = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
@@ -953,6 +957,7 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
 		deniedCountries:                "kp",
 		allowedCountries:               "de",
 		addLookupHeaders:               enabled,
+		asnLimitPercentURL:             asnURL,
 		asnLimitPercent:                "AS64496:50",
 		countryLimitPercent:            "cn:25",
 		asnBytesPercent:                "AS64496:50",
@@ -978,11 +983,13 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
 
 	// Set empty, the lists need nothing looked up, and nor does
 	// SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly
-	// threshold per AS number that is off, or any other anomaly threshold.
+	// threshold per AS number that is off, any other anomaly threshold, or
+	// a blocklist.
 	env := environment{
 		lookupSource: off, deniedCountries: "", allowedCountries: "",
 		asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "",
-		countryBytesPercent: "", unknownLimitPercent: "100",
+		countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "",
+		blocklistURLs: dropURL,
 	}
 	for _, name := range anomalyThresholds() {
 		env[name] = "1000"
@@ -1205,6 +1212,116 @@ func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) {
 	}
 }
 
+// dropURL and torURL are blocklists, and asnURL a file of AS:percent
+// lines.
+const (
+	dropURL = "https://www.spamhaus.org/drop/drop.txt"
+	torURL  = "https://lists.example/tor-exits.txt"
+	asnURL  = "https://lists.example/asn.txt"
+)
+
+// The actions of SWWAF_BLOCKLIST_ACTION, as Config gives them.
+const (
+	actionDeny  = "deny"
+	actionLimit = "limit"
+	actionLog   = "log"
+)
+
+func TestReputationSettingsAsSet(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{})
+	if len(cfg.BlocklistURLs) != 0 || cfg.BlocklistRefresh != 24*time.Hour ||
+		cfg.BlocklistAction != actionDeny || cfg.ASNLimitPercentURL != "" {
+		t.Errorf("%s, %s, %s and %s gave %v, %s, %s and %q by default, "+
+			"want none, 24h, deny and none", blocklistURLs, blocklistRefresh,
+			blocklistAction, asnLimitPercentURL, cfg.BlocklistURLs, cfg.BlocklistRefresh,
+			cfg.BlocklistAction, cfg.ASNLimitPercentURL)
+	}
+
+	for _, tc := range []struct {
+		value, action string
+		percent       int64
+	}{
+		{actionDeny, actionDeny, 0},
+		{actionLog, actionLog, 0},
+		{"limit:25", actionLimit, 25},
+		{"limit:0", actionLimit, 0},
+	} {
+		// An hour, the shortest refresh allowed.
+		cfg := fromEnvironment(t, environment{
+			blocklistURLs: dropURL + ", " + torURL, blocklistRefresh: "1h",
+			blocklistAction: tc.value, asnLimitPercentURL: asnURL,
+		})
+
+		if !slices.Equal(cfg.BlocklistURLs, []string{dropURL, torURL}) ||
+			cfg.BlocklistRefresh != time.Hour || cfg.BlocklistAction != tc.action ||
+			cfg.BlocklistLimitPercent != tc.percent || cfg.ASNLimitPercentURL != asnURL {
+			t.Errorf("%s=%s gave %v, %s, %s, %d and %s", blocklistAction, tc.value,
+				cfg.BlocklistURLs, cfg.BlocklistRefresh, cfg.BlocklistAction,
+				cfg.BlocklistLimitPercent, cfg.ASNLimitPercentURL)
+		}
+	}
+}
+
+func TestInvalidReputationSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
+	t.Parallel()
+
+	const (
+		notURL = " is not an http or https URL without a user or a fragment, " +
+			"such as https://www.spamhaus.org/drop/drop.txt"
+		notAnHour = " is not a duration of 1h or more, such as 24h"
+		notAction = " is not deny, limit:<percent> such as limit:25, or log"
+	)
+
+	for _, tc := range []struct{ name, value, want string }{
+		{
+			blocklistURLs, "ftp://lists.example/drop.txt",
+			`"ftp://lists.example/drop.txt"` + notURL,
+		},
+		{blocklistURLs, "lists.example/drop.txt", `"lists.example/drop.txt"` + notURL},
+		{
+			blocklistURLs, "https://me:secret@lists.example/drop.txt",
+			`"https://me:secret@lists.example/drop.txt"` + notURL,
+		},
+		{
+			blocklistURLs, dropURL + "," + torURL + "," + dropURL,
+			`"` + dropURL + `" is listed twice`,
+		},
+		{asnLimitPercentURL, asnURL + "#top", `"` + asnURL + `#top"` + notURL},
+		{blocklistRefresh, "59m", `"59m"` + notAnHour},
+		{blocklistRefresh, off, `"off"` + notAnHour},
+		{blocklistRefresh, "a day", `"a day"` + notAnHour},
+		{blocklistAction, "block", `"block"` + notAction},
+		{blocklistAction, actionLimit, `"limit"` + notAction},
+		{blocklistAction, "limit:101", `"limit:101"` + notAction},
+	} {
+		t.Run(tc.name+"="+tc.value, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
+
+			want := tc.name + ": " + tc.want
+			if err == nil || err.Error() != want {
+				t.Errorf("error %v, want %s", err, want)
+			}
+		})
+	}
+}
+
+func TestASNLimitPercentURLThatIsABlocklistStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	_, err := config.FromEnvironment(environment{
+		blocklistURLs: dropURL + "," + asnURL, asnLimitPercentURL: asnURL,
+	}.lookupEnv)
+
+	want := asnLimitPercentURL + `: "` + asnURL + `" is in SWWAF_BLOCKLIST_URLS too`
+	if err == nil || err.Error() != want {
+		t.Errorf("error %v, want %s", err, want)
+	}
+}
+
 func TestSizesAndOff(t *testing.T) {
 	t.Parallel()
 
@@ -1628,6 +1745,10 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		asnBytesPercent:         "",
 		countryBytesPercent:     "",
 		unknownLimitPercent:     "100",
+		asnLimitPercentURL:      "",
+		blocklistURLs:           "",
+		blocklistRefresh:        "24h",
+		blocklistAction:         actionDeny,
 		banResponse:             "403",
 		limitBanDuration:        "1h",
 		limitBanRepeatWindow:    "24h",