smallwebwaf/internal/proxy/reputation_test.go

package proxy_test

import (
	"maps"
	"net/http"
	"net/netip"
	"slices"
	"testing"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/proxy"
	"sneak.berlin/go/smallwebwaf/internal/reputation"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
)

// The reputation settings.
const (
	blocklistURLs      = "SWWAF_BLOCKLIST_URLS"
	blocklistAction    = "SWWAF_BLOCKLIST_ACTION"
	asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
)

// The actions of SWWAF_BLOCKLIST_ACTION and SWWAF_REPUTATION_ACTION:
// limitHalf gives a listed client half of every limit, and limitQuarter a
// quarter.
const (
	actionDeny   = "deny"
	actionLog    = "log"
	limitHalf    = "limit:50"
	limitQuarter = "limit:25"
)

// The lists these tests name, which are never fetched: each test puts in
// the copies it needs, as reputation.json would at start.
const (
	dropURL = "https://lists.example/drop.txt"
	torURL  = "https://lists.example/tor.txt"
	asnURL  = "https://lists.example/asn.txt"
)

func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
	t.Parallel()

	forward, denied := requestlog.ActionForward, requestlog.ActionDenied

	for _, tc := range []struct {
		action string
		// statuses and actions are those of a listed client's three
		// requests, and percent their limit_percent, as percentText gives it.
		statuses []int
		actions  []string
		percent  string
	}{
		{
			actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
			[]string{denied, denied, denied}, none,
		},
		{
			// Half of 4 requests a minute: the third breaks the limit.
			limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
			[]string{forward, forward, requestlog.ActionRateLimited},
			"50 from " + blocklistAction,
		},
		{
			actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
			[]string{forward, forward, forward}, none,
		},
	} {
		t.Run(tc.action, func(t *testing.T) {
			t.Parallel()

			s, server, _ := startWithLookups(t, map[string]string{
				rateLimitPerMinute: fourAMinute, blocklistURLs: dropURL,
				blocklistAction: tc.action,
			})
			// fromDE is listed as an address, and fromKP in a netblock.
			loadLists(t, server, map[string][]string{
				dropURL: {"; DROP", fromDE, "198.51.100.0/24 ; SBL1"},
			})

			for _, from := range []string{fromDE, fromKP} {
				for i := range 3 {
					line := s.get(from, tc.statuses[i], tc.actions[i])
					wantReputation(t, line, dropURL)
					wantPercent(t, "limit_percent", line.LimitPercent,
						line.LimitPercentSetting, tc.percent)

					// A request refused for the list is not counted.
					counted := line.fields["counts"] != nil
					if counted != (tc.actions[i] != denied) {
						t.Errorf("request from %s counted %t, logged %s", from, counted,
							tc.actions[i])
					}
				}
			}

			// A client no list lists has the whole limit.
			for range 3 {
				line := s.get(unplaced, http.StatusOK, forward)
				wantReputation(t, line)
				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
					none)
			}

			// A refusal for the list makes no ban.
			if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
				t.Errorf("bans %+v, want none", held)
			}
		})
	}
}

func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
	t.Parallel()

	s, server, queue := startWithLookups(t, map[string]string{
		blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE,
	})
	loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}})

	// fromKP's country refuses it before the list is looked at, and fromDE,
	// in SWWAF_ALLOW_NETS, is not checked at all: neither is noted, nor
	// alerted.
	wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied))
	wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward))
	wantAlerts(t, queue)
}

func TestObserveModeForwardsAClientABlocklistDeniesAndAlertsIt(t *testing.T) {
	t.Parallel()

	s, server, queue := startWithLookups(t, map[string]string{
		blocklistURLs: dropURL, mode: observe,
	})
	loadLists(t, server, map[string][]string{dropURL: {fromDE}})

	line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	wantWouldAction(t, line, requestlog.ActionDenied)
	wantReputation(t, line, dropURL)

	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
	if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit {
		t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting)
	}
}

func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		action, asnPercent string
		// want is the upload's limit_percent and bytes_percent, as
		// percentText gives them, and limitHit its limit_hit.
		want, limitHit string
	}{
		{limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
		{limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes},
		// The AS number's, the first of two alike.
		{limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
		{actionLog, asnDE + ":100", none, ""},
	} {
		t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
			t.Parallel()

			s, server, _ := startWithLookups(t, map[string]string{
				bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL,
				blocklistAction: tc.action, asnLimitPercent: tc.asnPercent,
			})
			loadLists(t, server, map[string][]string{dropURL: {fromDE}})

			// The upload's 100 bytes are over 49, a quarter of 199, and 99,
			// half of it, and within 199.
			line := s.uploadFrom(fromDE)
			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
				tc.want)
			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
				tc.want)

			if line.LimitHit != tc.limitHit {
				t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit)
			}
		})
	}
}

func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T) {
	t.Parallel()

	const (
		fromURL     = "25 from " + asnLimitPercentURL
		fromSetting = "25 from " + asnLimitPercent
	)

	for _, tc := range []struct {
		name string
		env  map[string]string
		file string
		// limitPercent and bytesPercent are the upload's, as percentText
		// gives them.
		limitPercent, bytesPercent string
	}{
		{"the file's alone", nil, asnDEQuarter, fromURL, fromURL},
		{
			"the file's, lower than the setting's",
			map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL,
		},
		{
			"the setting's, lower than the file's",
			map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf,
			fromSetting, fromSetting,
		},
		{
			"the setting's, the first of two alike",
			map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter,
			fromSetting, fromSetting,
		},
		{"none, for an AS number the file does not list", nil, asnKP + ":25", none, none},
		{
			"SWWAF_ASN_BYTES_PERCENT's in place of the file's for the byte limits",
			map[string]string{asnBytesPercent: asnDE + ":100"}, asnDEQuarter, fromURL, none,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			env := map[string]string{asnLimitPercentURL: asnURL}
			maps.Copy(env, tc.env)

			s, server, _ := startWithLookups(t, env)
			loadLists(t, server, map[string][]string{asnURL: {"# by AS number", tc.file}})

			line := s.uploadFrom(fromDE)
			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
				tc.limitPercent)
			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
				tc.bytesPercent)
		})
	}
}

func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
	t *testing.T,
) {
	t.Parallel()

	const emptyURL = "https://lists.example/empty.txt"

	s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{
		blocklistURLs:   dropURL + "," + torURL + "," + emptyURL,
		blocklistAction: actionLog,
		metricsToken:    token,
	})
	loadLists(t, server, map[string][]string{dropURL: {fromDE}, torURL: {fromDE}})

	// The second request's alerts are repeats, which the cooldown holds
	// back.
	for range 2 {
		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
			dropURL, torURL)
	}

	hit := func(source string) alerts.Alert {
		return alerts.Alert{
			Instance: alertInstance,
			Time:     clk.Now(),
			Event:    alerts.EventReputationHit,
			Client:   netip.MustParseAddr(fromDE),
			Netblock: netip.MustParsePrefix(fromDE + "/32"),
			ASN:      asnDE,
			ASName:   asNameDE,
			Country:  "DE",
			Reason:   "listed by a blocklist",
			Detail:   map[string]any{"source": source},
		}
	}
	wantAlerts(t, queue, hit(dropURL), hit(torURL))

	if queue.Suppressed() != 2 {
		t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed())
	}

	// Each list's hits, none of its fetches failed, and when its copy was
	// fetched, 0 for the one without.
	metrics := s.scrape(unplaced)
	fetched := float64(listsFetched().Unix())

	for listURL, want := range map[string]struct{ hits, fetched float64 }{
		dropURL: {2, fetched}, torURL: {2, fetched}, emptyURL: {0, 0},
	} {
		labels := `{instance="` + alertInstance + `",source="` + listURL + `"}`

		if want.hits == 0 {
			wantNoSeries(t, metrics, "smallwebwaf_reputation_hits_total"+labels)
		} else {
			wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, want.hits)
		}

		wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
		wantMetric(t, metrics, "smallwebwaf_reputation_last_fetch_timestamp_seconds"+labels,
			want.fetched)
	}
}

// The DNSBL settings.
const (
	dnsblZones       = "SWWAF_DNSBL_ZONES"
	dnsblResolver    = "SWWAF_DNSBL_RESOLVER"
	reputationAction = "SWWAF_REPUTATION_ACTION"
)

// The DNSBL zones these tests name, which are never asked about the
// clients the tests send requests from: each test puts in the verdicts it
// needs, as reputation.json would at start. A query a test does start is
// sent to noResolver, where nothing listens, so that none leaves the host.
const (
	dnsblZone  = "dnsbl.example"
	otherZone  = "other.example"
	noResolver = "127.0.0.1:9"
)

func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) {
	t.Parallel()

	forward, denied := requestlog.ActionForward, requestlog.ActionDenied

	for _, tc := range []struct {
		action string
		// statuses and actions are those of a listed client's three
		// requests, and percent their limit_percent, as percentText gives it.
		statuses []int
		actions  []string
		percent  string
	}{
		{
			actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
			[]string{denied, denied, denied}, none,
		},
		{
			// Half of 4 requests a minute: the third breaks the limit.
			limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
			[]string{forward, forward, requestlog.ActionRateLimited},
			"50 from " + reputationAction,
		},
		{
			actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
			[]string{forward, forward, forward}, none,
		},
	} {
		t.Run(tc.action, func(t *testing.T) {
			t.Parallel()

			s, server, _ := startWithLookups(t, map[string]string{
				rateLimitPerMinute: fourAMinute, dnsblZones: dnsblZone + "," + otherZone,
				dnsblResolver: noResolver, reputationAction: tc.action,
			})
			listedBy := map[string][]string{
				fromDE: {dnsblZone, otherZone}, fromKP: {otherZone}, unplaced: nil,
			}
			loadVerdicts(server, listedBy)

			for _, from := range []string{fromDE, fromKP} {
				for i := range 3 {
					line := s.get(from, tc.statuses[i], tc.actions[i])
					// In the order SWWAF_DNSBL_ZONES names them.
					wantReputation(t, line, listedBy[from]...)
					wantPercent(t, "limit_percent", line.LimitPercent,
						line.LimitPercentSetting, tc.percent)

					// A request refused for the verdict is not counted.
					counted := line.fields["counts"] != nil
					if counted != (tc.actions[i] != denied) {
						t.Errorf("request from %s counted %t, logged %s", from, counted,
							tc.actions[i])
					}
				}
			}

			// A client no zone lists has the whole limit.
			for range 3 {
				line := s.get(unplaced, http.StatusOK, forward)
				wantReputation(t, line)
				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
					none)
			}

			// A refusal for the verdict makes no ban, and every client had its
			// verdicts, so no zone was asked.
			if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
				t.Errorf("bans %+v, want none", held)
			}

			if queries := server.DNSBL.Queries(dnsblZone); queries != 0 {
				t.Errorf("%d queries, want none", queries)
			}
		})
	}
}

func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) {
	t.Parallel()

	s, server, queue := startWithLookups(t, map[string]string{
		blocklistURLs: dropURL, dnsblZones: dnsblZone, dnsblResolver: noResolver,
		reputationAction: actionDeny, allowNets: fromDE,
	})
	loadLists(t, server, map[string][]string{dropURL: {fromKP}})
	loadVerdicts(server, map[string][]string{fromKP: {dnsblZone}, fromDE: {dnsblZone}})

	// The blocklist refuses fromKP before its verdict is looked at, and
	// fromDE, in SWWAF_ALLOW_NETS, is not checked at all: neither is noted
	// for the zone, nor alerted, nor asked about.
	wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied),
		dropURL)
	wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward))

	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
	if len(waiting) != 1 || waiting[0].Detail["source"] != dropURL {
		t.Errorf("alerts waiting %+v, want the blocklist's reputation_hit alone", waiting)
	}

	if queries := server.DNSBL.Queries(dnsblZone); queries != 0 {
		t.Errorf("%d queries, want none", queries)
	}
}

func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) {
	t.Parallel()

	s, server, queue := startWithLookups(t, map[string]string{
		dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionDeny,
		mode: observe,
	})
	loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})

	line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	wantWouldAction(t, line, requestlog.ActionDenied)
	wantReputation(t, line, dnsblZone)

	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
	if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit {
		t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting)
	}
}

func TestReputationLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		blocklistAction, reputationAction string
		// want is the request's limit_percent and bytes_percent, as
		// percentText gives them.
		want string
	}{
		{limitHalf, limitQuarter, "25 from " + reputationAction},
		{limitQuarter, limitHalf, "25 from " + blocklistAction},
		// The blocklist's, the first of two alike.
		{limitQuarter, limitQuarter, "25 from " + blocklistAction},
		{actionLog, limitQuarter, "25 from " + reputationAction},
		{actionLog, actionLog, none},
	} {
		t.Run(tc.blocklistAction+" "+tc.reputationAction, func(t *testing.T) {
			t.Parallel()

			s, server, _ := startWithLookups(t, map[string]string{
				blocklistURLs: dropURL, blocklistAction: tc.blocklistAction,
				dnsblZones: dnsblZone, dnsblResolver: noResolver,
				reputationAction: tc.reputationAction,
			})
			loadLists(t, server, map[string][]string{dropURL: {fromDE}})
			loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})

			// Named by the blocklist, then by the zone.
			line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
			wantReputation(t, line, dropURL, dnsblZone)
			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
				tc.want)
			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
				tc.want)
		})
	}
}

func TestEachZoneThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
	t *testing.T,
) {
	t.Parallel()

	s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{
		dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver,
		reputationAction: actionLog, metricsToken: token,
	})
	loadVerdicts(server, map[string][]string{
		fromDE: {dnsblZone, otherZone}, unplaced: nil,
	})

	// The second request's alerts are repeats, which the cooldown holds
	// back.
	for range 2 {
		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
			dnsblZone, otherZone)
	}

	hit := func(zone string) alerts.Alert {
		return alerts.Alert{
			Instance: alertInstance,
			Time:     clk.Now(),
			Event:    alerts.EventReputationHit,
			Client:   netip.MustParseAddr(fromDE),
			Netblock: netip.MustParsePrefix(fromDE + "/32"),
			ASN:      asnDE,
			ASName:   asNameDE,
			Country:  "DE",
			Reason:   "listed by a DNSBL zone",
			Detail:   map[string]any{"source": zone},
		}
	}
	wantAlerts(t, queue, hit(dnsblZone), hit(otherZone))

	if queue.Suppressed() != 2 {
		t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed())
	}

	// Each zone's hits, and its queries and their failures, none, since
	// every client had its verdicts.
	metrics := s.scrape(unplaced)

	for _, zone := range []string{dnsblZone, otherZone} {
		labels := `{instance="` + alertInstance + `",source="` + zone + `"}`

		wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
		wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0)
		wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
	}
}

func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) {
	t.Parallel()

	s, server, _ := startWithLookups(t, map[string]string{
		dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver,
	})
	server.DNSBL.Load([]reputation.Verdict{{
		Zone: otherZone, Client: netip.MustParseAddr(fromDE), Listed: true,
		Fetched: verdictsFetched(),
	}})

	// The verdict of the other zone is used, and dnsbl.example, which has
	// none, is asked about the client in the background, once: the second
	// request finds the query under way, or the zone left alone after it
	// failed, since nothing answers at noResolver.
	for range 2 {
		wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), otherZone)
	}

	if queries := server.DNSBL.Queries(dnsblZone); queries != 1 {
		t.Errorf("%d queries to %s, want 1", queries, dnsblZone)
	}

	if queries := server.DNSBL.Queries(otherZone); queries != 0 {
		t.Errorf("%d queries to %s, want none", queries, otherZone)
	}
}

// listsFetched is when loadLists has the copies fetched.
func listsFetched() time.Time {
	return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
}

// verdictsFetched is when loadVerdicts has the verdicts fetched: half a
// day before the time the tests' clock is set to, so that they are in use
// until half a day later.
func verdictsFetched() time.Time {
	return time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
}

// loadVerdicts puts into server's DNSBL, for each client listedBy names,
// a verdict of each zone SWWAF_DNSBL_ZONES names, fetched at
// verdictsFetched, as reputation.json would at start: one that lists the
// client from each zone listedBy gives for it, and one that does not from
// each other zone.
func loadVerdicts(server *proxy.Server, listedBy map[string][]string) {
	verdicts := make([]reputation.Verdict, 0, len(listedBy)*len(server.DNSBL.Zones()))

	for client, zones := range listedBy {
		for _, zone := range server.DNSBL.Zones() {
			verdicts = append(verdicts, reputation.Verdict{
				Zone: zone, Client: netip.MustParseAddr(client),
				Listed: slices.Contains(zones, zone), Fetched: verdictsFetched(),
			})
		}
	}

	server.DNSBL.Load(verdicts)
}

// loadLists puts copies of lists into server's lists, by URL, each with
// its lines, fetched at listsFetched, as reputation.json would at start.
func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {
	t.Helper()

	lists := make([]reputation.List, 0, len(copies))
	for listURL, lines := range copies {
		lists = append(lists, reputation.List{
			URL: listURL, Fetched: listsFetched(), Lines: lines,
		})
	}

	err := server.Lists.Load(lists)
	if err != nil {
		t.Fatalf("load the lists: %v", err)
	}
}

// wantReputation checks the URLs of the blocklists the log line names in
// its reputation.
func wantReputation(t *testing.T, line logLine, want ...string) {
	t.Helper()

	if !slices.Equal(line.Reputation, want) {
		t.Errorf("log line has reputation %v, want %v", line.Reputation, want)
	}
}