DNS blocklists asked in the background, verdicts kept (closes #104)

Zones in SWWAF_DNSBL_ZONES are asked about each client (RFC 5782 names) in the background, through the host's resolver or SWWAF_DNSBL_RESOLVER; no request waits. Verdicts last SWWAF_REPUTATION_CACHE_TTL and are kept in reputation.json, at most 100,000. After the blocklists, SWWAF_REPUTATION_ACTION (limit:25) denies, limits or logs a listed client; the log line names the zones, each raises reputation_hit, with metrics by zone. A failed, timed-out or refused query gives no verdict, raises source_failure, and pauses the zone a minute. Judgement call: answers in 127.255.255.0/24 or outside 127.0.0.0/8 are failures. Judgement call: the minute's pause after a failure; at most 1,000 queries at once. Rule suppressed: paralleltest on the DNSBL tests (Go's resolver shares state across synctest bubbles), funlen on the test of every logged setting. Model: opus-5-5
2026-10-07 18:00:50 +00:00
parent 2b8c98ba1f
commit ddb95f5411
18 changed files with 2125 additions and 236 deletions
@@ -2,7 +2,8 @@
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
 // history, lookups.json GeoJS's answers, reputation.json the last try and
-// last good copy of each list fetched from a URL, and alerts.json the
+// last good copy of each list fetched from a URL and the DNSBL zones'
+// verdicts, and alerts.json the
 // cooldowns, the hour under way, the alerts waiting for each destination
 // and the anomaly counters. Load
 // reads them at start, Watch takes in an admin's edit of one while
@@ -76,13 +77,14 @@ type Params struct {
 	// is (SWWAF_STATE_COUNTER_INTERVAL).
 	WriteDelay      time.Duration
 	CounterInterval time.Duration
-	// Ledger, Limiter, GeoJS, Lists, Alerts and Anomalies hold the state.
-	// Alerts also receive a file_error alert for an edit set aside, and for
-	// a write that fails while smallwebwaf runs.
+	// Ledger, Limiter, GeoJS, Lists, DNSBL, Alerts and Anomalies hold the
+	// state. Alerts also receive a file_error alert for an edit set aside,
+	// and for a write that fails while smallwebwaf runs.
 	Ledger    *bans.Ledger
 	Limiter   *ratelimit.Limiter
 	GeoJS     *lookup.GeoJS
 	Lists     *reputation.Lists
+	DNSBL     *reputation.DNSBL
 	Alerts    *alerts.Queue
 	Anomalies *anomaly.Counters
 	// Now tells the time by which the counters' buckets run out, normally
@@ -145,8 +147,9 @@ type lookupsFile struct {
 // reputationFile is reputation.json, indented for an admin to read and
 // edit, so that each line of a list's copy is on a line of its own.
 type reputationFile struct {
-	Version int               `json:"version"`
-	Lists   []reputation.List `json:"lists"`
+	Version  int                  `json:"version"`
+	Lists    []reputation.List    `json:"lists"`
+	Verdicts []reputation.Verdict `json:"verdicts"`
 }

 // alertsFile is alerts.json, indented for an admin to read and edit.
@@ -432,6 +435,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 			return 0, fmt.Errorf("%s: %w", path, err)
 		}

+		f.params.DNSBL.Load(file.Verdicts)
 		entries = len(file.Lists)
 	case alertsJSON:
 		waiting, err := f.takeInAlerts(path, data)
@@ -567,6 +571,7 @@ func (f *Files) encode(name string) ([]byte, error) {
 	case reputationJSON:
 		return encodeIndented(reputationFile{
 			Version: version, Lists: f.params.Lists.Snapshot(),
+			Verdicts: f.params.DNSBL.Snapshot(),
 		})
 	default: // alerts.json
 		held := f.params.Alerts.Snapshot()
@@ -729,8 +734,12 @@ func (f *lookupsFile) check(data []byte) error {
 // check refuses a list without its URL, which would name no list, or the
 // time it was last tried, which would have it fetched at once, and a copy
 // of it without the time it was fetched, or without its lines, which hold
-// the list.
-func (f *reputationFile) check([]byte) error {
+// the list. It refuses a verdict without its zone or its client, which
+// would be about no one, whether the zone lists the client, or the time
+// it was fetched, which would drop it. A verdict's listed is false for a
+// client the zone does not list, which Verdicts cannot tell from a
+// missing one, so each listed is read again as written.
+func (f *reputationFile) check(data []byte) error {
 	for i, kept := range f.Lists {
 		switch {
 		case kept.URL == "":
@@ -744,6 +753,30 @@ func (f *reputationFile) check([]byte) error {
 		}
 	}

+	var written struct {
+		Verdicts []struct {
+			Listed *bool `json:"listed"`
+		} `json:"verdicts"`
+	}
+
+	err := json.Unmarshal(data, &written)
+	if err != nil {
+		return err
+	}
+
+	for i, verdict := range f.Verdicts {
+		switch {
+		case verdict.Zone == "":
+			return fmt.Errorf("verdicts %w", missing(i, "zone"))
+		case !verdict.Client.IsValid():
+			return fmt.Errorf("verdicts %w", missing(i, "client"))
+		case written.Verdicts[i].Listed == nil:
+			return fmt.Errorf("verdicts %w", missing(i, "listed"))
+		case verdict.Fetched.IsZero():
+			return fmt.Errorf("verdicts %w", missing(i, "fetched"))
+		}
+	}
+
 	return nil
 }

@@ -38,9 +38,11 @@ const (
 	lookupsJSON    = "lookups.json"
 	reputationJSON = "reputation.json"
 	alertsJSON     = "alerts.json"
-	// blocklistURL and torURL are the blocklists the tests' lists name.
+	// blocklistURL and torURL are the blocklists the tests' lists name, and
+	// dnsblZone the DNSBL zone of the tests' verdicts.
 	blocklistURL = "https://lists.example/drop.txt"
 	torURL       = "https://lists.example/tor.txt"
+	dnsblZone    = "dnsbl.example"
 	// The AS number and AS name the tests' clients are looked up in.
 	asn    = "AS64496"
 	asName = "Example Net"
@@ -210,7 +212,8 @@ const filledAlertsJSON = `{
 `

 // filledReputationJSON is reputation.json holding the blocklists' last
-// tries and the copy of one, with its comment line, as fill puts them in.
+// tries and the copy of one, with its comment line, and two verdicts of a
+// DNSBL zone, as fill puts them in.
 const filledReputationJSON = `{
    "version": 1,
    "lists": [
@@ -228,6 +231,20 @@ const filledReputationJSON = `{
            "url": "https://lists.example/tor.txt",
            "tried": "2026-10-06T00:00:00Z"
        }
+    ],
+    "verdicts": [
+        {
+            "zone": "dnsbl.example",
+            "client": "203.0.113.9",
+            "listed": true,
+            "fetched": "2026-10-05T23:00:00Z"
+        },
+        {
+            "zone": "dnsbl.example",
+            "client": "2001:db8::1",
+            "listed": false,
+            "fetched": "2026-10-05T22:00:00Z"
+        }
    ]
 }
 `
@@ -263,6 +280,8 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
 		t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want)
 	}

+	wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
+
 	if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
 		got, want) {
 		t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -422,8 +441,8 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 	held := params.Alerts.Snapshot()
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
 		len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
-		len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
-		held.Hour.Sent != 0 {
+		len(params.DNSBL.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
+		len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -592,6 +611,11 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 		drop    = `"url": "` + blocklistURL + `", `
 		tried   = `"tried": "2026-10-06T00:00:00Z", `
 		fetched = `"fetched": "2026-10-06T00:00:00Z"`
+		// verdictZone, verdictClient and listed start a verdict, which
+		// fetched ends.
+		verdictZone   = `"zone": "` + dnsblZone + `", `
+		verdictClient = `"client": "198.51.100.7", `
+		listed        = `"listed": false, `
 	)

 	for _, tc := range []struct {
@@ -621,6 +645,30 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 				`{"url": "` + torURL + `", ` + tried + fetched + `}]}`,
 			`: entry 2 has no "lines"`,
 		},
+		{
+			"a verdict without its zone",
+			`{"version": 1, "verdicts": [{` + verdictClient + listed + fetched + `}]}`,
+			`: verdicts entry 1 has no "zone"`,
+		},
+		{
+			"a verdict without its client",
+			`{"version": 1, "verdicts": [{` + verdictZone + listed + fetched + `}]}`,
+			`: verdicts entry 1 has no "client"`,
+		},
+		{
+			// A client the zone does not list has a listed of false, which is
+			// not having none.
+			"a verdict without whether the zone lists the client",
+			`{"version": 1, "verdicts": [{` + verdictZone + verdictClient + listed +
+				fetched + `}, {` + verdictZone + verdictClient + fetched + `}]}`,
+			`: verdicts entry 2 has no "listed"`,
+		},
+		{
+			"a verdict without the time it was fetched",
+			`{"version": 1, "verdicts": [{` + verdictZone + verdictClient +
+				`"listed": true}]}`,
+			`: verdicts entry 1 has no "fetched"`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -1133,7 +1181,8 @@ func TestEditOfEachFileTakenIn(t *testing.T) {

 	edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
 		`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
-		`"lines": ["198.51.100.7"]}]}`)
+		`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
+		`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`)
 	wantTakenIn(t, lines, dir, reputationJSON)

 	listedBy := params.Lists.ListedBy(client.Addr())
@@ -1142,6 +1191,10 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 			client.Addr(), listedBy)
 	}

+	wantEqual(t, reputationJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{
+		Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
+	}})
+
 	// A netblock with bits past its length is read as the netblock it is
 	// in.
 	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1558,6 +1611,10 @@ func newParams(dir string) state.Params {
 			BlocklistURLs: []string{blocklistURL, torURL}, Refresh: 24 * time.Hour,
 			Now: midnight, ProcessLog: discard, Alerts: queue,
 		}),
+		DNSBL: reputation.NewDNSBL(reputation.DNSBLParams{
+			Zones: []string{dnsblZone}, CacheTTL: 24 * time.Hour, Timeout: time.Second,
+			Now: midnight, ProcessLog: discard, Alerts: queue,
+		}),
 		Alerts: queue,
 		Anomalies: anomaly.New(anomaly.Params{
 			Net:            anomaly.Thresholds{RequestsPerMinute: 1000},
@@ -1582,9 +1639,10 @@ func office() netip.Prefix {

 // fill puts a permanent ban an admin made, a ban for a broken limit and
 // one for a clear sign of attack, clients with counts and histories,
-// GeoJS answers, the blocklists' last tries and the copy of one, as
-// filledReputationJSON holds them, and alerts and anomaly counters, as
-// filledAlertsJSON holds them, into the parts of params.
+// GeoJS answers, the blocklists' last tries and the copy of one, and two
+// verdicts of a DNSBL zone, as filledReputationJSON holds them, and alerts
+// and anomaly counters, as filledAlertsJSON holds them, into the parts of
+// params.
 func fill(params state.Params) {
 	now := midnight()
 	client := netip.MustParsePrefix("203.0.113.9/32")
@@ -1631,6 +1689,14 @@ func fill(params state.Params) {
 		panic(err) // the copy reads
 	}

+	params.DNSBL.Load([]reputation.Verdict{
+		{
+			Zone: dnsblZone, Client: netip.MustParseAddr("2001:db8::1"),
+			Fetched: now.Add(-2 * time.Hour),
+		},
+		{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
+	})
+
 	// An alert waiting, a repeat of it the cooldown holds back, another
 	// alert waiting, and one past the two an hour, for the hour's summary.
 	ban := alerts.Alert{