Blocklists and an AS percentage file fetched by URL (closes #29)

SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every
SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands
for its IPv4 address or netblock. reputation.json keeps each list's last
try, failed or not, even one cut off by a stop, which a restart waits on
as a running instance does, and its last good copy, whole, used while a
fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed
client; the log line names the lists, each raises reputation_hit, and a
failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched
the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning.

Judgement call: a failed fetch is retried after the refresh, not sooner.
Not done: ban notes do not name the lists yet.

Model: opus-5-5

internal/proxy/reputation_test.go

@@ -0,0 +1,333 @@
+package proxy_test
+
+import (
+	"maps"
+	"net/http"
+	"net/netip"
+	"slices"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/reputation"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// The reputation settings.
+const (
+	blocklistURLs      = "SWWAF_BLOCKLIST_URLS"
+	blocklistAction    = "SWWAF_BLOCKLIST_ACTION"
+	asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
+)
+
+// The actions of SWWAF_BLOCKLIST_ACTION but limit, which has a
+// percentage.
+const (
+	actionDeny = "deny"
+	actionLog  = "log"
+)
+
+// The lists these tests name, which are never fetched: each test puts in
+// the copies it needs, as reputation.json would at start.
+const (
+	dropURL = "https://lists.example/drop.txt"
+	torURL  = "https://lists.example/tor.txt"
+	asnURL  = "https://lists.example/asn.txt"
+)
+
+func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
+	t.Parallel()
+
+	forward, denied := requestlog.ActionForward, requestlog.ActionDenied
+
+	for _, tc := range []struct {
+		action string
+		// statuses and actions are those of a listed client's three
+		// requests, and percent their limit_percent, as percentText gives it.
+		statuses []int
+		actions  []string
+		percent  string
+	}{
+		{
+			actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
+			[]string{denied, denied, denied}, none,
+		},
+		{
+			// Half of 4 requests a minute: the third breaks the limit.
+			"limit:50", []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
+			[]string{forward, forward, requestlog.ActionRateLimited},
+			"50 from " + blocklistAction,
+		},
+		{
+			actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
+			[]string{forward, forward, forward}, none,
+		},
+	} {
+		t.Run(tc.action, func(t *testing.T) {
+			t.Parallel()
+
+			s, server, _ := startWithLookups(t, map[string]string{
+				rateLimitPerMinute: fourAMinute, blocklistURLs: dropURL,
+				blocklistAction: tc.action,
+			})
+			// fromDE is listed as an address, and fromKP in a netblock.
+			loadLists(t, server, map[string][]string{
+				dropURL: {"; DROP", fromDE, "198.51.100.0/24 ; SBL1"},
+			})
+
+			for _, from := range []string{fromDE, fromKP} {
+				for i := range 3 {
+					line := s.get(from, tc.statuses[i], tc.actions[i])
+					wantReputation(t, line, dropURL)
+					wantPercent(t, "limit_percent", line.LimitPercent,
+						line.LimitPercentSetting, tc.percent)
+
+					// A request refused for the list is not counted.
+					counted := line.fields["counts"] != nil
+					if counted != (tc.actions[i] != denied) {
+						t.Errorf("request from %s counted %t, logged %s", from, counted,
+							tc.actions[i])
+					}
+				}
+			}
+
+			// A client no list lists has the whole limit.
+			for range 3 {
+				line := s.get(unplaced, http.StatusOK, forward)
+				wantReputation(t, line)
+				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+					none)
+			}
+
+			// A refusal for the list makes no ban.
+			if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
+				t.Errorf("bans %+v, want none", held)
+			}
+		})
+	}
+}
+
+func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
+	t.Parallel()
+
+	s, server, queue := startWithLookups(t, map[string]string{
+		blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE,
+	})
+	loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}})
+
+	// fromKP's country refuses it before the list is looked at, and fromDE,
+	// in SWWAF_ALLOW_NETS, is not checked at all: neither is noted, nor
+	// alerted.
+	wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied))
+	wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward))
+	wantAlerts(t, queue)
+}
+
+func TestObserveModeForwardsAClientABlocklistDeniesAndAlertsIt(t *testing.T) {
+	t.Parallel()
+
+	s, server, queue := startWithLookups(t, map[string]string{
+		blocklistURLs: dropURL, mode: observe,
+	})
+	loadLists(t, server, map[string][]string{dropURL: {fromDE}})
+
+	line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionDenied)
+	wantReputation(t, line, dropURL)
+
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit {
+		t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting)
+	}
+}
+
+func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		action, asnPercent string
+		// want is the upload's limit_percent and bytes_percent, as
+		// percentText gives them, and limitHit its limit_hit.
+		want, limitHit string
+	}{
+		{"limit:50", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
+		{"limit:25", asnDEHalf, "25 from " + blocklistAction, minuteBytes},
+		// The AS number's, the first of two alike.
+		{"limit:25", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
+		{actionLog, asnDE + ":100", none, ""},
+	} {
+		t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
+			t.Parallel()
+
+			s, server, _ := startWithLookups(t, map[string]string{
+				bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL,
+				blocklistAction: tc.action, asnLimitPercent: tc.asnPercent,
+			})
+			loadLists(t, server, map[string][]string{dropURL: {fromDE}})
+
+			// The upload's 100 bytes are over 49, a quarter of 199, and 99,
+			// half of it, and within 199.
+			line := s.uploadFrom(fromDE)
+			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+				tc.want)
+			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
+				tc.want)
+
+			if line.LimitHit != tc.limitHit {
+				t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit)
+			}
+		})
+	}
+}
+
+func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T) {
+	t.Parallel()
+
+	const (
+		fromURL     = "25 from " + asnLimitPercentURL
+		fromSetting = "25 from " + asnLimitPercent
+	)
+
+	for _, tc := range []struct {
+		name string
+		env  map[string]string
+		file string
+		// limitPercent and bytesPercent are the upload's, as percentText
+		// gives them.
+		limitPercent, bytesPercent string
+	}{
+		{"the file's alone", nil, asnDEQuarter, fromURL, fromURL},
+		{
+			"the file's, lower than the setting's",
+			map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL,
+		},
+		{
+			"the setting's, lower than the file's",
+			map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf,
+			fromSetting, fromSetting,
+		},
+		{
+			"the setting's, the first of two alike",
+			map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter,
+			fromSetting, fromSetting,
+		},
+		{"none, for an AS number the file does not list", nil, asnKP + ":25", none, none},
+		{
+			"SWWAF_ASN_BYTES_PERCENT's in place of the file's for the byte limits",
+			map[string]string{asnBytesPercent: asnDE + ":100"}, asnDEQuarter, fromURL, none,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			env := map[string]string{asnLimitPercentURL: asnURL}
+			maps.Copy(env, tc.env)
+
+			s, server, _ := startWithLookups(t, env)
+			loadLists(t, server, map[string][]string{asnURL: {"# by AS number", tc.file}})
+
+			line := s.uploadFrom(fromDE)
+			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+				tc.limitPercent)
+			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
+				tc.bytesPercent)
+		})
+	}
+}
+
+func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	const emptyURL = "https://lists.example/empty.txt"
+
+	s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{
+		blocklistURLs:   dropURL + "," + torURL + "," + emptyURL,
+		blocklistAction: actionLog,
+		metricsToken:    token,
+	})
+	loadLists(t, server, map[string][]string{dropURL: {fromDE}, torURL: {fromDE}})
+
+	// The second request's alerts are repeats, which the cooldown holds
+	// back.
+	for range 2 {
+		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
+			dropURL, torURL)
+	}
+
+	hit := func(source string) alerts.Alert {
+		return alerts.Alert{
+			Instance: alertInstance,
+			Time:     clk.Now(),
+			Event:    alerts.EventReputationHit,
+			Client:   netip.MustParseAddr(fromDE),
+			Netblock: netip.MustParsePrefix(fromDE + "/32"),
+			ASN:      asnDE,
+			ASName:   asNameDE,
+			Country:  "DE",
+			Reason:   "listed by a blocklist",
+			Detail:   map[string]any{"source": source},
+		}
+	}
+	wantAlerts(t, queue, hit(dropURL), hit(torURL))
+
+	if queue.Suppressed() != 2 {
+		t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed())
+	}
+
+	// Each list's hits, none of its fetches failed, and when its copy was
+	// fetched, 0 for the one without.
+	metrics := s.scrape(unplaced)
+	fetched := float64(listsFetched().Unix())
+
+	for listURL, want := range map[string]struct{ hits, fetched float64 }{
+		dropURL: {2, fetched}, torURL: {2, fetched}, emptyURL: {0, 0},
+	} {
+		labels := `{instance="` + alertInstance + `",source="` + listURL + `"}`
+
+		if want.hits == 0 {
+			wantNoSeries(t, metrics, "smallwebwaf_reputation_hits_total"+labels)
+		} else {
+			wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, want.hits)
+		}
+
+		wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
+		wantMetric(t, metrics, "smallwebwaf_reputation_last_fetch_timestamp_seconds"+labels,
+			want.fetched)
+	}
+}
+
+// listsFetched is when loadLists has the copies fetched.
+func listsFetched() time.Time {
+	return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
+}
+
+// loadLists puts copies of lists into server's lists, by URL, each with
+// its lines, fetched at listsFetched, as reputation.json would at start.
+func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {
+	t.Helper()
+
+	lists := make([]reputation.List, 0, len(copies))
+	for listURL, lines := range copies {
+		lists = append(lists, reputation.List{
+			URL: listURL, Fetched: listsFetched(), Lines: lines,
+		})
+	}
+
+	err := server.Lists.Load(lists)
+	if err != nil {
+		t.Fatalf("load the lists: %v", err)
+	}
+}
+
+// wantReputation checks the URLs of the blocklists the log line names in
+// its reputation.
+func wantReputation(t *testing.T, line logLine, want ...string) {
+	t.Helper()
+
+	if !slices.Equal(line.Reputation, want) {
+		t.Errorf("log line has reputation %v, want %v", line.Reputation, want)
+	}
+}