DNS blocklists asked in the background, verdicts kept (closes #104)

Zones in SWWAF_DNSBL_ZONES are asked about each client (RFC 5782 names)
in the background, through the host's resolver or SWWAF_DNSBL_RESOLVER;
no request waits. Verdicts last SWWAF_REPUTATION_CACHE_TTL and are kept
in reputation.json, at most 100,000. After the blocklists,
SWWAF_REPUTATION_ACTION (limit:25) denies, limits or logs a listed
client; the log line names the zones, each raises reputation_hit, with
metrics by zone. A failed, timed-out or refused query gives no verdict,
raises source_failure, and pauses the zone a minute.

Judgement call: answers in 127.255.255.0/24 or outside 127.0.0.0/8 are failures.
Judgement call: the minute's pause after a failure; at most 1,000 queries at once.
Rule suppressed: paralleltest on the DNSBL tests (Go's resolver shares state across synctest bubbles), funlen on the test of every logged setting.

Model: opus-5-5

internal/proxy/reputation_test.go

@@ -21,11 +21,14 @@ const (
 	asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
 )
 
-// The actions of SWWAF_BLOCKLIST_ACTION but limit, which has a
-// percentage.
+// The actions of SWWAF_BLOCKLIST_ACTION and SWWAF_REPUTATION_ACTION:
+// limitHalf gives a listed client half of every limit, and limitQuarter a
+// quarter.
 const (
-	actionDeny = "deny"
-	actionLog  = "log"
+	actionDeny   = "deny"
+	actionLog    = "log"
+	limitHalf    = "limit:50"
+	limitQuarter = "limit:25"
 )
 
 // The lists these tests name, which are never fetched: each test puts in
@@ -55,7 +58,7 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
 		},
 		{
 			// Half of 4 requests a minute: the third breaks the limit.
-			"limit:50", []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
+			limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
 			[]string{forward, forward, requestlog.ActionRateLimited},
 			"50 from " + blocklistAction,
 		},
@@ -151,10 +154,10 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
 		// percentText gives them, and limitHit its limit_hit.
 		want, limitHit string
 	}{
-		{"limit:50", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
-		{"limit:25", asnDEHalf, "25 from " + blocklistAction, minuteBytes},
+		{limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
+		{limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes},
 		// The AS number's, the first of two alike.
-		{"limit:25", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
+		{limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
 		{actionLog, asnDE + ":100", none, ""},
 	} {
 		t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
@@ -299,11 +302,298 @@ func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
 	}
 }
 
+// The DNSBL settings.
+const (
+	dnsblZones       = "SWWAF_DNSBL_ZONES"
+	dnsblResolver    = "SWWAF_DNSBL_RESOLVER"
+	reputationAction = "SWWAF_REPUTATION_ACTION"
+)
+
+// The DNSBL zones these tests name, which are never asked about the
+// clients the tests send requests from: each test puts in the verdicts it
+// needs, as reputation.json would at start. A query a test does start is
+// sent to noResolver, where nothing listens, so that none leaves the host.
+const (
+	dnsblZone  = "dnsbl.example"
+	otherZone  = "other.example"
+	noResolver = "127.0.0.1:9"
+)
+
+func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) {
+	t.Parallel()
+
+	forward, denied := requestlog.ActionForward, requestlog.ActionDenied
+
+	for _, tc := range []struct {
+		action string
+		// statuses and actions are those of a listed client's three
+		// requests, and percent their limit_percent, as percentText gives it.
+		statuses []int
+		actions  []string
+		percent  string
+	}{
+		{
+			actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
+			[]string{denied, denied, denied}, none,
+		},
+		{
+			// Half of 4 requests a minute: the third breaks the limit.
+			limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
+			[]string{forward, forward, requestlog.ActionRateLimited},
+			"50 from " + reputationAction,
+		},
+		{
+			actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
+			[]string{forward, forward, forward}, none,
+		},
+	} {
+		t.Run(tc.action, func(t *testing.T) {
+			t.Parallel()
+
+			s, server, _ := startWithLookups(t, map[string]string{
+				rateLimitPerMinute: fourAMinute, dnsblZones: dnsblZone + "," + otherZone,
+				dnsblResolver: noResolver, reputationAction: tc.action,
+			})
+			listedBy := map[string][]string{
+				fromDE: {dnsblZone, otherZone}, fromKP: {otherZone}, unplaced: nil,
+			}
+			loadVerdicts(server, listedBy)
+
+			for _, from := range []string{fromDE, fromKP} {
+				for i := range 3 {
+					line := s.get(from, tc.statuses[i], tc.actions[i])
+					// In the order SWWAF_DNSBL_ZONES names them.
+					wantReputation(t, line, listedBy[from]...)
+					wantPercent(t, "limit_percent", line.LimitPercent,
+						line.LimitPercentSetting, tc.percent)
+
+					// A request refused for the verdict is not counted.
+					counted := line.fields["counts"] != nil
+					if counted != (tc.actions[i] != denied) {
+						t.Errorf("request from %s counted %t, logged %s", from, counted,
+							tc.actions[i])
+					}
+				}
+			}
+
+			// A client no zone lists has the whole limit.
+			for range 3 {
+				line := s.get(unplaced, http.StatusOK, forward)
+				wantReputation(t, line)
+				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+					none)
+			}
+
+			// A refusal for the verdict makes no ban, and every client had its
+			// verdicts, so no zone was asked.
+			if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
+				t.Errorf("bans %+v, want none", held)
+			}
+
+			if queries := server.DNSBL.Queries(dnsblZone); queries != 0 {
+				t.Errorf("%d queries, want none", queries)
+			}
+		})
+	}
+}
+
+func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) {
+	t.Parallel()
+
+	s, server, queue := startWithLookups(t, map[string]string{
+		blocklistURLs: dropURL, dnsblZones: dnsblZone, dnsblResolver: noResolver,
+		reputationAction: actionDeny, allowNets: fromDE,
+	})
+	loadLists(t, server, map[string][]string{dropURL: {fromKP}})
+	loadVerdicts(server, map[string][]string{fromKP: {dnsblZone}, fromDE: {dnsblZone}})
+
+	// The blocklist refuses fromKP before its verdict is looked at, and
+	// fromDE, in SWWAF_ALLOW_NETS, is not checked at all: neither is noted
+	// for the zone, nor alerted, nor asked about.
+	wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied),
+		dropURL)
+	wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward))
+
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != 1 || waiting[0].Detail["source"] != dropURL {
+		t.Errorf("alerts waiting %+v, want the blocklist's reputation_hit alone", waiting)
+	}
+
+	if queries := server.DNSBL.Queries(dnsblZone); queries != 0 {
+		t.Errorf("%d queries, want none", queries)
+	}
+}
+
+func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) {
+	t.Parallel()
+
+	s, server, queue := startWithLookups(t, map[string]string{
+		dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionDeny,
+		mode: observe,
+	})
+	loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
+
+	line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionDenied)
+	wantReputation(t, line, dnsblZone)
+
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit {
+		t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting)
+	}
+}
+
+func TestReputationLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		blocklistAction, reputationAction string
+		// want is the request's limit_percent and bytes_percent, as
+		// percentText gives them.
+		want string
+	}{
+		{limitHalf, limitQuarter, "25 from " + reputationAction},
+		{limitQuarter, limitHalf, "25 from " + blocklistAction},
+		// The blocklist's, the first of two alike.
+		{limitQuarter, limitQuarter, "25 from " + blocklistAction},
+		{actionLog, limitQuarter, "25 from " + reputationAction},
+		{actionLog, actionLog, none},
+	} {
+		t.Run(tc.blocklistAction+" "+tc.reputationAction, func(t *testing.T) {
+			t.Parallel()
+
+			s, server, _ := startWithLookups(t, map[string]string{
+				blocklistURLs: dropURL, blocklistAction: tc.blocklistAction,
+				dnsblZones: dnsblZone, dnsblResolver: noResolver,
+				reputationAction: tc.reputationAction,
+			})
+			loadLists(t, server, map[string][]string{dropURL: {fromDE}})
+			loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
+
+			// Named by the blocklist, then by the zone.
+			line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
+			wantReputation(t, line, dropURL, dnsblZone)
+			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+				tc.want)
+			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
+				tc.want)
+		})
+	}
+}
+
+func TestEachZoneThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{
+		dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver,
+		reputationAction: actionLog, metricsToken: token,
+	})
+	loadVerdicts(server, map[string][]string{
+		fromDE: {dnsblZone, otherZone}, unplaced: nil,
+	})
+
+	// The second request's alerts are repeats, which the cooldown holds
+	// back.
+	for range 2 {
+		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
+			dnsblZone, otherZone)
+	}
+
+	hit := func(zone string) alerts.Alert {
+		return alerts.Alert{
+			Instance: alertInstance,
+			Time:     clk.Now(),
+			Event:    alerts.EventReputationHit,
+			Client:   netip.MustParseAddr(fromDE),
+			Netblock: netip.MustParsePrefix(fromDE + "/32"),
+			ASN:      asnDE,
+			ASName:   asNameDE,
+			Country:  "DE",
+			Reason:   "listed by a DNSBL zone",
+			Detail:   map[string]any{"source": zone},
+		}
+	}
+	wantAlerts(t, queue, hit(dnsblZone), hit(otherZone))
+
+	if queue.Suppressed() != 2 {
+		t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed())
+	}
+
+	// Each zone's hits, and its queries and their failures, none, since
+	// every client had its verdicts.
+	metrics := s.scrape(unplaced)
+
+	for _, zone := range []string{dnsblZone, otherZone} {
+		labels := `{instance="` + alertInstance + `",source="` + zone + `"}`
+
+		wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
+		wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0)
+		wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
+	}
+}
+
+func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) {
+	t.Parallel()
+
+	s, server, _ := startWithLookups(t, map[string]string{
+		dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver,
+	})
+	server.DNSBL.Load([]reputation.Verdict{{
+		Zone: otherZone, Client: netip.MustParseAddr(fromDE), Listed: true,
+		Fetched: verdictsFetched(),
+	}})
+
+	// The verdict of the other zone is used, and dnsbl.example, which has
+	// none, is asked about the client in the background, once: the second
+	// request finds the query under way, or the zone left alone after it
+	// failed, since nothing answers at noResolver.
+	for range 2 {
+		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), otherZone)
+	}
+
+	if queries := server.DNSBL.Queries(dnsblZone); queries != 1 {
+		t.Errorf("%d queries to %s, want 1", queries, dnsblZone)
+	}
+
+	if queries := server.DNSBL.Queries(otherZone); queries != 0 {
+		t.Errorf("%d queries to %s, want none", queries, otherZone)
+	}
+}
+
 // listsFetched is when loadLists has the copies fetched.
 func listsFetched() time.Time {
 	return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
 }
 
+// verdictsFetched is when loadVerdicts has the verdicts fetched: half a
+// day before the time the tests' clock is set to, so that they are in use
+// until half a day later.
+func verdictsFetched() time.Time {
+	return time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+}
+
+// loadVerdicts puts into server's DNSBL, for each client listedBy names,
+// a verdict of each zone SWWAF_DNSBL_ZONES names, fetched at
+// verdictsFetched, as reputation.json would at start: one that lists the
+// client from each zone listedBy gives for it, and one that does not from
+// each other zone.
+func loadVerdicts(server *proxy.Server, listedBy map[string][]string) {
+	verdicts := make([]reputation.Verdict, 0, len(listedBy)*len(server.DNSBL.Zones()))
+
+	for client, zones := range listedBy {
+		for _, zone := range server.DNSBL.Zones() {
+			verdicts = append(verdicts, reputation.Verdict{
+				Zone: zone, Client: netip.MustParseAddr(client),
+				Listed: slices.Contains(zones, zone), Fetched: verdictsFetched(),
+			})
+		}
+	}
+
+	server.DNSBL.Load(verdicts)
+}
+
 // loadLists puts copies of lists into server's lists, by URL, each with
 // its lines, fetched at listsFetched, as reputation.json would at start.
 func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {