smallwebwaf/internal/proxy/biased_test.go

package proxy_test

import (
	"fmt"
	"io"
	"maps"
	"net"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"path/filepath"
	"strings"
	"testing"
	"testing/synctest"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
	"sneak.berlin/go/smallwebwaf/internal/proxy"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
)

// The biased thresholds.
const (
	asnLimitPercent     = "SWWAF_ASN_LIMIT_PERCENT"
	countryLimitPercent = "SWWAF_COUNTRY_LIMIT_PERCENT"
	asnBytesPercent     = "SWWAF_ASN_BYTES_PERCENT"
	countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT"
	unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT"
)

const (
	// asnDEHalf and countryDEHalf give fromDE's AS number and its country
	// half of every limit, and asnDEQuarter gives its AS number a quarter.
	asnDEHalf     = asnDE + ":50"
	asnDEQuarter  = asnDE + ":25"
	countryDEHalf = "de:50"
	// noCountry is in an AS of its own, AS64500, and in no country.
	noCountry = "192.0.2.80"
	// fourAMinute is the rate limit these tests set: half of it is 2
	// requests a minute, a quarter of it 1.
	fourAMinute = "4"
	// twoUploads is the byte limit these tests set: 199 bytes, which an
	// upload, a request with a body and its answer, 100 bytes, is within,
	// and half of which, 99 bytes, it is over.
	twoUploads = "199"
	// none is how percentText gives a percentage left out.
	none = "none"
)

func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		setting, value, from string
	}{
		{asnLimitPercent, asnDEHalf, fromDE},
		{countryLimitPercent, countryDEHalf, fromDE},
		{unknownLimitPercent, "50", unplaced},
	} {
		t.Run(tc.setting, func(t *testing.T) {
			t.Parallel()

			s, _, _ := startWithLookups(t, map[string]string{
				rateLimitPerMinute: fourAMinute, tc.setting: tc.value,
			})

			// Half of 4 requests a minute: the third breaks the limit.
			for _, sent := range []struct {
				status int
				action string
			}{
				{http.StatusOK, requestlog.ActionForward},
				{http.StatusOK, requestlog.ActionForward},
				{http.StatusForbidden, requestlog.ActionRateLimited},
			} {
				line := s.get(tc.from, sent.status, sent.action)
				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
					"50 from "+tc.setting)
			}

			// fromKP, which no setting lists, has the whole limit.
			for range 3 {
				line := s.get(fromKP, http.StatusOK, requestlog.ActionForward)
				wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
					none)
			}
		})
	}
}

func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) {
	t.Parallel()

	// The AS numbers and countries are given in either case.
	for _, tc := range []struct {
		setting, value, from string
	}{
		{asnLimitPercent, asnDEHalf, fromDE},
		{countryLimitPercent, "DE:50", fromDE},
		{unknownLimitPercent, "50", unplaced},
		{asnBytesPercent, "as64496:50", fromDE},
		{countryBytesPercent, countryDEHalf, fromDE},
	} {
		t.Run(tc.setting, func(t *testing.T) {
			t.Parallel()

			s, _, _ := startWithLookups(t, map[string]string{
				bytesLimitPerMinute: twoUploads, tc.setting: tc.value,
			})

			// The upload's 100 bytes are over half of 199, 99.
			line := s.uploadFrom(tc.from)
			if line.LimitHit != minuteBytes {
				t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes)
			}

			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
				"50 from "+tc.setting)

			// fromKP, which no setting lists, has the whole limit.
			line = s.uploadFrom(fromKP)
			if line.LimitHit != "" {
				t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit)
			}

			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none)
		})
	}
}

func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		name string
		env  map[string]string
		// limitPercent and bytesPercent are the log line's, as percentText
		// gives them, and limitHit is its limit_hit.
		limitPercent, bytesPercent, limitHit string
	}{
		{
			"lowering the byte limits alone",
			map[string]string{asnBytesPercent: asnDEHalf},
			none, "50 from " + asnBytesPercent, minuteBytes,
		},
		{
			"lowering the byte limits alone, by country",
			map[string]string{countryBytesPercent: countryDEHalf},
			none, "50 from " + countryBytesPercent, minuteBytes,
		},
		{
			"raising the byte limits back",
			map[string]string{asnLimitPercent: asnDEHalf, asnBytesPercent: asnDE + ":100"},
			"50 from " + asnLimitPercent, none, "",
		},
		{
			"raising the byte limits back, by country",
			map[string]string{countryLimitPercent: countryDEHalf, countryBytesPercent: "de:100"},
			"50 from " + countryLimitPercent, none, "",
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			env := map[string]string{bytesLimitPerMinute: twoUploads}
			maps.Copy(env, tc.env)
			s, _, _ := startWithLookups(t, env)

			// The upload's 100 bytes are over 99, half of 199, and within 199.
			line := s.uploadFrom(fromDE)
			if line.LimitHit != tc.limitHit {
				t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit)
			}

			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
				tc.limitPercent)
			wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
				tc.bytesPercent)
		})
	}
}

func TestZeroPercentIsAZeroAllowance(t *testing.T) {
	t.Parallel()

	s, _, _ := startWithLookups(t, map[string]string{asnLimitPercent: asnDE + ":0"})

	// The first request breaks the limit, and bans the client; the log line
	// gives the 0.
	line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
	if line.fields["limit_percent"] != float64(0) ||
		line.fields["limit_percent_setting"] != asnLimitPercent {
		t.Errorf("log line has limit_percent %v from %v, want 0 from %s",
			line.fields["limit_percent"], line.fields["limit_percent_setting"],
			asnLimitPercent)
	}

	s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
}

func TestLowestPercentageApplies(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		name string
		env  map[string]string
		from string
		// want is the log line's limit_percent, as percentText gives it.
		want string
	}{
		{
			"the country's",
			map[string]string{asnLimitPercent: asnDEHalf, countryLimitPercent: "de:25"},
			fromDE, "25 from " + countryLimitPercent,
		},
		{
			"the AS number's",
			map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: countryDEHalf},
			fromDE, "25 from " + asnLimitPercent,
		},
		{
			"the AS number's, the first of two alike",
			map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: "de:25"},
			fromDE, "25 from " + asnLimitPercent,
		},
		{
			"that for a client without a country",
			map[string]string{asnLimitPercent: "AS64500:50", unknownLimitPercent: "25"},
			noCountry, "25 from " + unknownLimitPercent,
		},
		{
			// SWWAF_UNKNOWN_LIMIT_PERCENT is left at its default, 100.
			"the AS number's, for a client without a country",
			map[string]string{asnLimitPercent: "AS64500:25"},
			noCountry, "25 from " + asnLimitPercent,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			env := map[string]string{rateLimitPerMinute: fourAMinute}
			maps.Copy(env, tc.env)
			s, _, _ := startWithLookups(t, env)

			// A quarter of 4 requests a minute: the second breaks the limit.
			s.get(tc.from, http.StatusOK, requestlog.ActionForward)

			line := s.get(tc.from, http.StatusForbidden, requestlog.ActionRateLimited)
			wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
				tc.want)
		})
	}
}

func TestUnknownLimitPercentGivesEveryClientWithoutACountryItsPercentage(t *testing.T) {
	t.Parallel()

	s, _, _ := startWithLookups(t, map[string]string{
		rateLimitPerMinute: fourAMinute, unknownLimitPercent: "50",
	})

	// One the lookup database does not hold, and one on a private address,
	// which is never looked up: the third request of each breaks half of 4.
	for _, from := range []string{unplaced, "10.0.0.8"} {
		s.get(from, http.StatusOK, requestlog.ActionForward)
		s.get(from, http.StatusOK, requestlog.ActionForward)
		s.get(from, http.StatusForbidden, requestlog.ActionRateLimited)
	}

	// One in a country has the whole limit.
	for range 3 {
		s.get(fromDE, http.StatusOK, requestlog.ActionForward)
	}
}

func TestClientWithoutAnAnswerInTimeHasTheUnknownLimitPercent(t *testing.T) {
	t.Parallel()

	// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
	// says, with a GeoJS that never answers.
	synctest.Test(t, func(t *testing.T) {
		server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL,
			time.Now, map[string]string{unknownLimitPercent: "0"})

		// Once the time the request waits for its answer is up, the client
		// counts as without a country, and its zero allowance refuses the
		// request before it reaches the app.
		serveFromDE(t, server, http.MethodGet, http.NoBody)

		line := out.requestLine(t)
		wantLine(t, line, http.StatusForbidden, requestlog.ActionRateLimited)
		wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
			"0 from "+unknownLimitPercent)
	})
}

func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
	t.Parallel()

	const timeout = 3 * time.Second

	for _, tc := range []struct {
		setting, value string
		waits          bool
	}{
		{asnLimitPercent, asnDEHalf, true},
		{countryLimitPercent, countryDEHalf, true},
		{asnBytesPercent, asnDEHalf, true},
		{countryBytesPercent, countryDEHalf, true},
		{unknownLimitPercent, "99", true},
		{asnLimitPercentURL, asnURL, true},
		// At 100, its default, it lowers no limit.
		{unknownLimitPercent, "100", false},
	} {
		t.Run(tc.setting+"="+tc.value, func(t *testing.T) {
			t.Parallel()

			// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
			// says, with a GeoJS that never answers.
			synctest.Test(t, func(t *testing.T) {
				// The request's body is over SWWAF_REQUEST_MAX_BYTES, so that it
				// is refused after the checks, and never reaches the app.
				server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL,
					time.Now, map[string]string{
						lookupTimeout: timeout.String(), requestMaxBytes: "1",
						tc.setting: tc.value,
					})
				began := time.Now()

				serveFromDE(t, server, http.MethodPost, strings.NewReader("ab"))

				want := time.Duration(0)
				if tc.waits {
					want = timeout
				}

				if waited := time.Since(began); waited != want {
					t.Errorf("the request waited %s for its answer, want %s", waited, want)
				}

				wantLine(t, out.requestLine(t), http.StatusRequestEntityTooLarge,
					requestlog.ActionTooLarge)

				// The bubble's clock stops once this function returns, so the
				// request to GeoJS, which a request that did not wait leaves
				// under way, has to be abandoned before then.
				time.Sleep(timeout)
			})
		})
	}
}

func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		name string
		env  map[string]string
		// before is how many uploads come before the one that breaks a
		// limit, which is answered with status and logged with action.
		before int
		status int
		action string
		// reason and want are the ban's reason, and its notes' limit
		// percentage, as percentText gives it.
		reason, want string
	}{
		{
			// A quarter of 12 requests a minute is 3: the fourth breaks it.
			"a rate limit",
			map[string]string{rateLimitPerMinute: "12", asnLimitPercent: asnDEQuarter},
			3, http.StatusForbidden, requestlog.ActionRateLimited,
			"requests per minute over the limit of 3", "25 from " + asnLimitPercent,
		},
		{
			// The byte limits' percentage, not the rate limits'.
			"a byte limit",
			map[string]string{
				bytesLimitPerMinute: twoUploads, asnLimitPercent: asnDEQuarter,
				asnBytesPercent: asnDEHalf,
			},
			0, http.StatusOK, requestlog.ActionForward,
			"bytes per minute over the limit of 99", "50 from " + asnBytesPercent,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			s, server, queue := startWithLookups(t, tc.env)

			for range tc.before {
				s.uploadFrom(fromDE)
			}

			s.requestWithBody(http.MethodPost, fromDE, "/", uploadHeader, uploadBody,
				tc.status, tc.action)

			held := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32"))
			if len(held) != 1 {
				t.Fatalf("bans %+v, want one", held)
			}

			notes := held[0].Notes
			if held[0].Reason != tc.reason {
				t.Errorf("the ban's reason is %q, want %q", held[0].Reason, tc.reason)
			}

			wantPercent(t, "the notes' limit_percent", notes.LimitPercent,
				notes.LimitPercentSetting, tc.want)

			waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
			if len(waiting) != 1 {
				t.Fatalf("%d alerts wait, want the ban's alone: %+v", len(waiting), waiting)
			}

			alerted, _ := waiting[0].Detail["notes"].(bans.Notes)
			wantPercent(t, "the alert's notes' limit_percent", alerted.LimitPercent,
				alerted.LimitPercentSetting, tc.want)
		})
	}
}

// startWithLookups is startWithLookupsAndClock for a test that needs no
// clock.
func startWithLookups(
	t *testing.T, env map[string]string,
) (*sender, *proxy.Server, *alerts.Queue) {
	t.Helper()

	s, _, server, queue := startWithLookupsAndClock(t, env)

	return s, server, queue
}

// startWithLookupsAndClock is startAppWithAlerts in front of
// readAndAnswer, with the settings in env on top of clients looked up in a
// lookup database, which places fromDE and fromKP in the AS numbers and
// countries the stand-in for GeoJS gives them, noCountry in AS64500 and no
// country, and no other address.
func startWithLookupsAndClock(
	t *testing.T, env map[string]string,
) (*sender, *clock, *proxy.Server, *alerts.Queue) {
	t.Helper()

	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
	lookuptest.Write(t, path, map[string]lookuptest.Network{
		fromDE + "/32":    {ASN: asnDE, ASName: asNameDE, Country: "DE"},
		fromKP + "/32":    {ASN: asnKP, ASName: asNameKP, Country: "KP"},
		noCountry + "/32": {ASN: "AS64500", ASName: "Nowhere Net"},
	})

	settings := map[string]string{lookupSource: fileSource, lookupDBPath: path}
	maps.Copy(settings, env)

	return startAppWithAlerts(t, readAndAnswer, settings)
}

// uploadFrom is upload from the client at from.
func (s *sender) uploadFrom(from string) logLine {
	s.t.Helper()

	line, _ := s.requestWithBody(http.MethodPost, from, "/", uploadHeader, uploadBody,
		http.StatusOK, requestlog.ActionForward)

	return line
}

// serveFromDE hands a request from fromDE with method and body straight to
// server's handler, without the network, and returns once it is answered.
func serveFromDE(t *testing.T, server *proxy.Server, method string, body io.Reader) {
	t.Helper()

	req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
	req.RemoteAddr = net.JoinHostPort(fromDE, "1234")

	server.Handler.ServeHTTP(httptest.NewRecorder(), req)
}

// wantPercent checks a limit percentage that a log line or a ban's notes
// give, what, and the setting that gave it, against want, as percentText
// gives them.
func wantPercent(t *testing.T, what string, percent *int64, setting, want string) {
	t.Helper()

	if got := percentText(percent, setting); got != want {
		t.Errorf("%s is %s, want %s", what, got, want)
	}
}

// percentText gives a limit percentage and the setting that gave it as
// text, such as "50 from SWWAF_ASN_LIMIT_PERCENT", or none when both are
// left out.
func percentText(percent *int64, setting string) string {
	switch {
	case percent == nil && setting == "":
		return none
	case percent == nil:
		return "none from " + setting
	default:
		return fmt.Sprintf("%d from %s", *percent, setting)
	}
}