AbuseIPDB scores for clients that committed an offence, within a daily budget (closes #105)

With SWWAF_ABUSEIPDB_KEY set, a client whose history counts an offence
(a broken limit, a ban rule's match or a block rule's refusal, counted
by kind) is checked in the background, at most
SWWAF_ABUSEIPDB_DAILY_BUDGET checks a day, the count kept in
reputation.json. A client, an IPv4 address or an IPv6 /64, is checked by
the address it sent from, and its score serves all its addresses. A
score at or over SWWAF_ABUSEIPDB_MIN_SCORE is a hit for
SWWAF_REPUTATION_ACTION, logged as abuseipdb and alerted with its score.
A failure or the used-up budget gives no score and raises
source_failure. The key goes only in the Key header.

Judgement call: the budget's day is UTC; AbuseIPDB documents no reset time.
Judgement call: each check sent spends budget; a minute's pause after a failure.

Model: opus-5-5

internal/proxy/reputation_test.go

@@ -1,16 +1,20 @@
 package proxy_test
 
 import (
+	"fmt"
+	"io"
 	"maps"
 	"net/http"
 	"net/netip"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/reputation"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
@@ -599,6 +603,277 @@ func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) {
 	}
 }
 
+// The AbuseIPDB settings, and accountKey, the key the tests set.
+const (
+	abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY"
+	accountKey   = "abuseipdb-key-0123456789abcdef"
+)
+
+// abuseipdb is how the request log, the alerts and the metrics name
+// AbuseIPDB.
+const abuseipdb = reputation.AbuseIPDBSource
+
+// abuseIPDBURL is where newProxy has clients checked with AbuseIPDB: at
+// abuseIPDBStandIn, which TestMain registers with Go's default transport,
+// through which AbuseIPDB is asked.
+const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
+
+// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
+// score 100, at once and without the network.
+type abuseIPDBStandIn struct{}
+
+// RoundTrip answers req with the score 100.
+func (abuseIPDBStandIn) RoundTrip(req *http.Request) (*http.Response, error) {
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Status:     "200 OK",
+		Header:     http.Header{},
+		Body:       io.NopCloser(strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`)),
+		Request:    req,
+	}, nil
+}
+
+func TestOnlyAClientThatHasCommittedAnOffenceIsCheckedWithAbuseIPDB(t *testing.T) {
+	t.Parallel()
+
+	forward := requestlog.ActionForward
+
+	s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{
+		abuseIPDBKey: accountKey, rateLimitPerMinute: "2", reputationAction: actionLog,
+	})
+
+	// Neither fromDE, until it breaks a rate limit, nor fromKP, which never
+	// does, is checked, nor fromDE under the ban that makes.
+	s.get(fromDE, http.StatusOK, forward)
+	s.get(fromDE, http.StatusOK, forward)
+	s.get(fromKP, http.StatusOK, forward)
+	s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
+	s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
+	wantAbuseIPDBChecks(t, server, 0)
+
+	// Once the ban has ended, fromDE's first request has it checked in the
+	// background, and goes on without its score, which its next request
+	// finds.
+	clk.advance(time.Hour)
+	wantReputation(t, s.get(fromDE, http.StatusOK, forward))
+	wantAbuseIPDBChecks(t, server, 1)
+	waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot().Scores) == 1 })
+	wantReputation(t, s.get(fromDE, http.StatusOK, forward), abuseipdb)
+
+	s.get(fromKP, http.StatusOK, forward)
+	wantAbuseIPDBChecks(t, server, 1)
+}
+
+func TestIPv6ClientCostsOneAbuseIPDBCheckWhicheverOfItsAddressesSends(t *testing.T) {
+	t.Parallel()
+
+	forward := requestlog.ActionForward
+
+	// 15 addresses of 2001:db8:1:2::/64, one client, each in a part of it
+	// of its own.
+	var addresses []string
+	for i := 1; i < 16; i++ {
+		addresses = append(addresses, fmt.Sprintf("2001:db8:1:2:%x::9", i<<12))
+	}
+
+	s, clk, server := startWithClock(t, "", map[string]string{
+		abuseIPDBKey: accountKey, reputationAction: actionLog,
+		rateLimitPerMinute: strconv.Itoa(len(addresses)),
+	})
+
+	// The client breaks the rate limit from its first address, which bans
+	// it for an hour.
+	for range addresses {
+		s.get(addresses[0], http.StatusOK, forward)
+	}
+
+	s.get(addresses[0], http.StatusForbidden, requestlog.ActionRateLimited)
+	clk.advance(time.Hour)
+
+	// Once the ban has ended, which set its counters back to zero, a
+	// request from each of its addresses has it checked once.
+	for _, address := range addresses {
+		s.get(address, http.StatusOK, forward)
+	}
+
+	wantAbuseIPDBChecks(t, server, 1)
+}
+
+// probePath is the path the ban rule of testRules, probe, matches.
+const probePath = "/.env"
+
+func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name string
+		// path is what the client asks for, status and action what that
+		// request is answered and logged with, and want the offences its
+		// history then counts.
+		path   string
+		status int
+		action string
+		want   ratelimit.Offences
+	}{
+		{
+			"a block rule", "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked,
+			ratelimit.Offences{RuleBlocked: 1},
+		},
+		{
+			"a ban rule", probePath, http.StatusForbidden, requestlog.ActionBanned,
+			ratelimit.Offences{Attack: 1},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			s, clk, server := startWithClock(t, "", map[string]string{
+				abuseIPDBKey: accountKey, reputationAction: actionLog,
+				rulesDir: writeRules(t, testRules), attackBanDuration: "1h",
+			})
+
+			s.request(client, tc.path, tc.status, tc.action)
+			wantAbuseIPDBChecks(t, server, 0)
+
+			if got := historyOf(t, server, client).Offences; got != tc.want {
+				t.Errorf("history counts the offences %+v, want %+v", got, tc.want)
+			}
+
+			// Its next request, once a ban rule's ban has ended, has it
+			// checked.
+			clk.advance(time.Hour)
+			s.get(client, http.StatusOK, requestlog.ActionForward)
+			wantAbuseIPDBChecks(t, server, 1)
+		})
+	}
+}
+
+func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *testing.T) {
+	t.Parallel()
+
+	forward, denied := requestlog.ActionForward, requestlog.ActionDenied
+
+	for _, tc := range []struct {
+		action string
+		// statuses and actions are those of fromDE's three requests, and
+		// percent their limit_percent, as percentText gives it.
+		statuses []int
+		actions  []string
+		percent  string
+	}{
+		{
+			actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
+			[]string{denied, denied, denied}, none,
+		},
+		{
+			// Half of 4 requests a minute: the third breaks the limit.
+			limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
+			[]string{forward, forward, requestlog.ActionRateLimited},
+			"50 from " + reputationAction,
+		},
+		{
+			actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
+			[]string{forward, forward, forward}, none,
+		},
+	} {
+		t.Run(tc.action, func(t *testing.T) {
+			t.Parallel()
+
+			s, server, _ := startWithLookups(t, map[string]string{
+				rateLimitPerMinute: fourAMinute, abuseIPDBKey: accountKey,
+				reputationAction: tc.action,
+			})
+			// At SWWAF_ABUSEIPDB_MIN_SCORE, 75 by default, and just under it.
+			loadScores(server, map[string]int64{fromDE: 75, fromKP: 74})
+
+			for i := range 3 {
+				line := s.get(fromDE, tc.statuses[i], tc.actions[i])
+				wantReputation(t, line, abuseipdb)
+				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+					tc.percent)
+
+				// A request refused for the score is not counted.
+				counted := line.fields["counts"] != nil
+				if counted != (tc.actions[i] != denied) {
+					t.Errorf("request counted %t, logged %s", counted, tc.actions[i])
+				}
+			}
+
+			// fromKP's score is no hit, and it has the whole limit.
+			for range 3 {
+				line := s.get(fromKP, http.StatusOK, forward)
+				wantReputation(t, line)
+				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
+					none)
+			}
+
+			// A refusal for the score makes no ban.
+			if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
+				t.Errorf("bans %+v, want none", held)
+			}
+		})
+	}
+}
+
+func TestAbuseIPDBHitRaisesAnAlertWithTheScoreOncePerCooldownAndIsCounted(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s, server, queue := startWithLookups(t, map[string]string{
+		abuseIPDBKey: accountKey, reputationAction: actionLog, metricsToken: token,
+	})
+	loadScores(server, map[string]int64{fromDE: 90})
+
+	// The second request's alert is a repeat, which the cooldown holds back.
+	for range 2 {
+		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
+			abuseipdb)
+	}
+
+	// The alert is made as a DNSBL zone's is, with the score besides.
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
+		waiting[0].Reason != "scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE" ||
+		waiting[0].Detail["source"] != abuseipdb || waiting[0].Detail["score"] != int64(90) ||
+		queue.Suppressed() != 1 {
+		t.Errorf("alerts waiting %+v, %d held back, want AbuseIPDB's reputation_hit "+
+			"with the score 90, and 1", waiting, queue.Suppressed())
+	}
+
+	// The hits, and the checks, none, since no client committed an
+	// offence, so that the whole budget is left.
+	metrics := s.scrape(unplaced)
+	labels := `{instance="` + alertInstance + `",source="` + abuseipdb + `"}`
+
+	wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
+	wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0)
+	wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
+	wantMetric(t, metrics, "smallwebwaf_reputation_daily_budget_remaining"+labels, 900)
+}
+
+func TestWithoutAnAbuseIPDBKeyNoClientIsCheckedNorAScoreUsed(t *testing.T) {
+	t.Parallel()
+
+	forward := requestlog.ActionForward
+
+	s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{
+		rateLimitPerMinute: "1", metricsToken: token,
+	})
+	loadScores(server, map[string]int64{fromDE: 100})
+
+	// fromDE's score is not used, and once it has committed an offence it
+	// is not checked either.
+	wantReputation(t, s.get(fromDE, http.StatusOK, forward))
+	s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
+	clk.advance(time.Hour)
+	wantReputation(t, s.get(fromDE, http.StatusOK, forward))
+	wantAbuseIPDBChecks(t, server, 0)
+
+	wantNoSeries(t, s.scrape(unplaced), `smallwebwaf_reputation_daily_budget_remaining{`+
+		`instance="`+alertInstance+`",source="`+abuseipdb+`"}`)
+}
+
 // listsFetched is when loadLists has the copies fetched.
 func listsFetched() time.Time {
 	return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
@@ -631,6 +906,31 @@ func loadVerdicts(server *proxy.Server, listedBy map[string][]string) {
 	server.DNSBL.Load(verdicts)
 }
 
+// loadScores puts into server's AbuseIPDB the score scores gives each
+// client, an IPv4 address, fetched at verdictsFetched, as reputation.json
+// would at start.
+func loadScores(server *proxy.Server, scores map[string]int64) {
+	kept := make([]reputation.Score, 0, len(scores))
+	for client, score := range scores {
+		kept = append(kept, reputation.Score{
+			Client: netip.MustParsePrefix(client + "/32"), Score: score,
+			Fetched: verdictsFetched(),
+		})
+	}
+
+	server.AbuseIPDB.Load(reputation.Checks{Scores: kept})
+}
+
+// wantAbuseIPDBChecks checks how many clients server has checked with
+// AbuseIPDB.
+func wantAbuseIPDBChecks(t *testing.T, server *proxy.Server, want int) {
+	t.Helper()
+
+	if got := server.AbuseIPDB.Checked(); got != want {
+		t.Errorf("%d clients checked with AbuseIPDB, want %d", got, want)
+	}
+}
+
 // loadLists puts copies of lists into server's lists, by URL, each with
 // its lines, fetched at listsFetched, as reputation.json would at start.
 func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {