AbuseIPDB scores for clients that committed an offence, within a daily budget (closes #105)

With SWWAF_ABUSEIPDB_KEY set, a client whose history counts an offence (a broken limit, a ban rule's match or a block rule's refusal, counted by kind) is checked in the background, at most SWWAF_ABUSEIPDB_DAILY_BUDGET checks a day, the count kept in reputation.json. A client, an IPv4 address or an IPv6 /64, is checked by the address it sent from, and its score serves all its addresses. A score at or over SWWAF_ABUSEIPDB_MIN_SCORE is a hit for SWWAF_REPUTATION_ACTION, logged as abuseipdb and alerted with its score. A failure or the used-up budget gives no score and raises source_failure. The key goes only in the Key header. Judgement call: the budget's day is UTC; AbuseIPDB documents no reset time. Judgement call: each check sent spends budget; a minute's pause after a failure. Model: opus-5-5
2026-10-07 20:27:55 +00:00
parent 0b0f207423
commit e265e8cd04
23 changed files with 2045 additions and 309 deletions
@@ -2,8 +2,8 @@
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
 // history, lookups.json GeoJS's answers, reputation.json the last try and
-// last good copy of each list fetched from a URL and the DNSBL zones'
-// verdicts, and alerts.json the
+// last good copy of each list fetched from a URL, the DNSBL zones'
+// verdicts, and AbuseIPDB's scores and checks spent, and alerts.json the
 // cooldowns, the hour under way, the alerts waiting for each destination
 // and the anomaly counters. Load
 // reads them at start, Watch takes in an admin's edit of one while
@@ -77,14 +77,15 @@ type Params struct {
 	// is (SWWAF_STATE_COUNTER_INTERVAL).
 	WriteDelay      time.Duration
 	CounterInterval time.Duration
-	// Ledger, Limiter, GeoJS, Lists, DNSBL, Alerts and Anomalies hold the
-	// state. Alerts also receive a file_error alert for an edit set aside,
-	// and for a write that fails while smallwebwaf runs.
+	// Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, Alerts and Anomalies
+	// hold the state. Alerts also receive a file_error alert for an edit set
+	// aside, and for a write that fails while smallwebwaf runs.
 	Ledger    *bans.Ledger
 	Limiter   *ratelimit.Limiter
 	GeoJS     *lookup.GeoJS
 	Lists     *reputation.Lists
 	DNSBL     *reputation.DNSBL
+	AbuseIPDB *reputation.AbuseIPDB
 	Alerts    *alerts.Queue
 	Anomalies *anomaly.Counters
 	// Now tells the time by which the counters' buckets run out, normally
@@ -147,9 +148,10 @@ type lookupsFile struct {
 // reputationFile is reputation.json, indented for an admin to read and
 // edit, so that each line of a list's copy is on a line of its own.
 type reputationFile struct {
-	Version  int                  `json:"version"`
-	Lists    []reputation.List    `json:"lists"`
-	Verdicts []reputation.Verdict `json:"verdicts"`
+	Version   int                  `json:"version"`
+	Lists     []reputation.List    `json:"lists"`
+	Verdicts  []reputation.Verdict `json:"verdicts"`
+	AbuseIPDB reputation.Checks    `json:"abuseipdb"`
 }

 // alertsFile is alerts.json, indented for an admin to read and edit.
@@ -436,6 +438,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 		}

 		f.params.DNSBL.Load(file.Verdicts)
+		f.params.AbuseIPDB.Load(file.AbuseIPDB)
 		entries = len(file.Lists)
 	case alertsJSON:
 		waiting, err := f.takeInAlerts(path, data)
@@ -571,7 +574,7 @@ func (f *Files) encode(name string) ([]byte, error) {
 	case reputationJSON:
 		return encodeIndented(reputationFile{
 			Version: version, Lists: f.params.Lists.Snapshot(),
-			Verdicts: f.params.DNSBL.Snapshot(),
+			Verdicts: f.params.DNSBL.Snapshot(), AbuseIPDB: f.params.AbuseIPDB.Snapshot(),
 		})
 	default: // alerts.json
 		held := f.params.Alerts.Snapshot()
@@ -736,9 +739,11 @@ func (f *lookupsFile) check(data []byte) error {
 // of it without the time it was fetched, or without its lines, which hold
 // the list. It refuses a verdict without its zone or its client, which
 // would be about no one, whether the zone lists the client, or the time
-// it was fetched, which would drop it. A verdict's listed is false for a
-// client the zone does not list, which Verdicts cannot tell from a
-// missing one, so each listed is read again as written.
+// it was fetched, which would drop it, and so an AbuseIPDB score without
+// its client, the score, or the time it was fetched. A verdict's listed is
+// false for a client the zone does not list, and a score can be 0, which
+// the structs cannot tell from a missing one, so each is read again as
+// written.
 func (f *reputationFile) check(data []byte) error {
 	for i, kept := range f.Lists {
 		switch {
@@ -777,6 +782,36 @@ func (f *reputationFile) check(data []byte) error {
 		}
 	}

+	return checkScores(f.AbuseIPDB.Scores, data)
+}
+
+// checkScores refuses an AbuseIPDB score, of scores, read from data, as
+// reputationFile's check describes.
+func checkScores(scores []reputation.Score, data []byte) error {
+	var written struct {
+		AbuseIPDB struct {
+			Scores []struct {
+				Score *int64 `json:"score"`
+			} `json:"scores"`
+		} `json:"abuseipdb"`
+	}
+
+	err := json.Unmarshal(data, &written)
+	if err != nil {
+		return err
+	}
+
+	for i, kept := range scores {
+		switch {
+		case !kept.Client.IsValid():
+			return fmt.Errorf("abuseipdb scores %w", missing(i, "client"))
+		case written.AbuseIPDB.Scores[i].Score == nil:
+			return fmt.Errorf("abuseipdb scores %w", missing(i, "score"))
+		case kept.Fetched.IsZero():
+			return fmt.Errorf("abuseipdb scores %w", missing(i, "fetched"))
+		}
+	}
+
 	return nil
 }

@@ -212,8 +212,9 @@ const filledAlertsJSON = `{
 `

 // filledReputationJSON is reputation.json holding the blocklists' last
-// tries and the copy of one, with its comment line, and two verdicts of a
-// DNSBL zone, as fill puts them in.
+// tries and the copy of one, with its comment line, two verdicts of a
+// DNSBL zone, and the AbuseIPDB checks spent today with two scores, as
+// fill puts them in.
 const filledReputationJSON = `{
    "version": 1,
    "lists": [
@@ -245,7 +246,23 @@ const filledReputationJSON = `{
            "listed": false,
            "fetched": "2026-10-05T22:00:00Z"
        }
-    ]
+    ],
+    "abuseipdb": {
+        "day": "2026-10-06T00:00:00Z",
+        "spent": 3,
+        "scores": [
+            {
+                "client": "203.0.113.9/32",
+                "score": 100,
+                "fetched": "2026-10-05T23:00:00Z"
+            },
+            {
+                "client": "2001:db8::/64",
+                "score": 0,
+                "fetched": "2026-10-05T22:00:00Z"
+            }
+        ]
+    }
 }
 `

@@ -282,6 +299,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) {

 	wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())

+	checks, wantChecks := after.AbuseIPDB.Snapshot(), before.AbuseIPDB.Snapshot()
+	if !reflect.DeepEqual(checks, wantChecks) {
+		t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, checks, wantChecks)
+	}
+
 	if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
 		got, want) {
 		t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -439,10 +461,13 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 	load(t, params)

 	held := params.Alerts.Snapshot()
+	checks := params.AbuseIPDB.Snapshot()
+
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
 		len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
-		len(params.DNSBL.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
-		len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
+		len(params.DNSBL.Snapshot()) != 0 || len(checks.Scores) != 0 || checks.Spent != 0 ||
+		len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
+		held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -678,6 +703,47 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	}
 }

+func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	// scores opens the list of AbuseIPDB scores, and ends closes it; client,
+	// score and fetched make a score.
+	const (
+		scores  = `{"version": 1, "abuseipdb": {"scores": [`
+		client  = `"client": "198.51.100.7/32", `
+		score   = `"score": 0, `
+		fetched = `"fetched": "2026-10-06T00:00:00Z"`
+		ends    = `}]}}`
+	)
+
+	for _, tc := range []struct {
+		name, content string
+		// want is what the error says after the file's path.
+		want string
+	}{
+		{
+			"without its client", scores + `{` + score + fetched + ends,
+			`: abuseipdb scores entry 1 has no "client"`,
+		},
+		{
+			// A score of 0 is not having none.
+			"without the score",
+			scores + `{` + client + score + fetched + `}, {` + client + fetched + ends,
+			`: abuseipdb scores entry 2 has no "score"`,
+		},
+		{
+			"without the time it was fetched", scores + `{` + client + `"score": 100` + ends,
+			`: abuseipdb scores entry 1 has no "fetched"`,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			wantRefused(t, reputationJSON, tc.content, tc.want)
+		})
+	}
+}
+
 func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	t.Parallel()

@@ -1182,7 +1248,9 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
 		`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
 		`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
-		`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`)
+		`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+
+		`"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+
+		`{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`)
 	wantTakenIn(t, lines, dir, reputationJSON)

 	listedBy := params.Lists.ListedBy(client.Addr())
@@ -1195,6 +1263,14 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 		Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
 	}})

+	checks := reputation.Checks{
+		Day: midnight(), Spent: 9,
+		Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}},
+	}
+	if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) {
+		t.Errorf("%s taken in as\n%+v\nwant\n%+v", reputationJSON, got, checks)
+	}
+
 	// A netblock with bits past its length is read as the netblock it is
 	// in.
 	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1615,6 +1691,10 @@ func newParams(dir string) state.Params {
 			Zones: []string{dnsblZone}, CacheTTL: 24 * time.Hour, Timeout: time.Second,
 			Now: midnight, ProcessLog: discard, Alerts: queue,
 		}),
+		AbuseIPDB: reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
+			MinScore: 75, DailyBudget: 900, CacheTTL: 24 * time.Hour, Timeout: time.Second,
+			Now: midnight, ProcessLog: discard, Alerts: queue,
+		}),
 		Alerts: queue,
 		Anomalies: anomaly.New(anomaly.Params{
 			Net:            anomaly.Thresholds{RequestsPerMinute: 1000},
@@ -1639,8 +1719,9 @@ func office() netip.Prefix {

 // fill puts a permanent ban an admin made, a ban for a broken limit and
 // one for a clear sign of attack, clients with counts and histories,
-// GeoJS answers, the blocklists' last tries and the copy of one, and two
-// verdicts of a DNSBL zone, as filledReputationJSON holds them, and alerts
+// GeoJS answers, the blocklists' last tries and the copy of one, two
+// verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two
+// scores, as filledReputationJSON holds them, and alerts
 // and anomaly counters, as filledAlertsJSON holds them, into the parts of
 // params.
 func fill(params state.Params) {
@@ -1696,6 +1777,10 @@ func fill(params state.Params) {
 		},
 		{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
 	})
+	params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{
+		{Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)},
+		{Client: client, Score: 100, Fetched: now.Add(-time.Hour)},
+	}})

 	// An alert waiting, a repeat of it the cooldown holds back, another
 	// alert waiting, and one past the two an hour, for the hour's summary.