package proxy_test import ( "fmt" "io" "maps" "net" "net/http" "net/http/httptest" "net/netip" "path/filepath" "strings" "testing" "testing/synctest" "time" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) // The biased thresholds. const ( asnLimitPercent = "SWWAF_ASN_LIMIT_PERCENT" countryLimitPercent = "SWWAF_COUNTRY_LIMIT_PERCENT" asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT" countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT" unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT" ) const ( // asnDEHalf and countryDEHalf give fromDE's AS number and its country // half of every limit, and asnDEQuarter gives its AS number a quarter. asnDEHalf = asnDE + ":50" asnDEQuarter = asnDE + ":25" countryDEHalf = "de:50" // noCountry is in an AS of its own, AS64500, and in no country. noCountry = "192.0.2.80" // fourAMinute is the rate limit these tests set: half of it is 2 // requests a minute, a quarter of it 1. fourAMinute = "4" // twoUploads is the byte limit these tests set: 199 bytes, which an // upload, a request with a body and its answer, 100 bytes, is within, // and half of which, 99 bytes, it is over. twoUploads = "199" // none is how percentText gives a percentage left out. none = "none" ) func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) { t.Parallel() for _, tc := range []struct { setting, value, from string }{ {asnLimitPercent, asnDEHalf, fromDE}, {countryLimitPercent, countryDEHalf, fromDE}, {unknownLimitPercent, "50", unplaced}, } { t.Run(tc.setting, func(t *testing.T) { t.Parallel() s, _, _ := startWithLookups(t, map[string]string{ rateLimitPerMinute: fourAMinute, tc.setting: tc.value, }) // Half of 4 requests a minute: the third breaks the limit. for _, sent := range []struct { status int action string }{ {http.StatusOK, requestlog.ActionForward}, {http.StatusOK, requestlog.ActionForward}, {http.StatusForbidden, requestlog.ActionRateLimited}, } { line := s.get(tc.from, sent.status, sent.action) wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, "50 from "+tc.setting) } // fromKP, which no setting lists, has the whole limit. for range 3 { line := s.get(fromKP, http.StatusOK, requestlog.ActionForward) wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, none) } }) } } func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) { t.Parallel() // The AS numbers and countries are given in either case. for _, tc := range []struct { setting, value, from string }{ {asnLimitPercent, asnDEHalf, fromDE}, {countryLimitPercent, "DE:50", fromDE}, {unknownLimitPercent, "50", unplaced}, {asnBytesPercent, "as64496:50", fromDE}, {countryBytesPercent, countryDEHalf, fromDE}, } { t.Run(tc.setting, func(t *testing.T) { t.Parallel() s, _, _ := startWithLookups(t, map[string]string{ bytesLimitPerMinute: twoUploads, tc.setting: tc.value, }) // The upload's 100 bytes are over half of 199, 99. line := s.uploadFrom(tc.from) if line.LimitHit != minuteBytes { t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes) } wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, "50 from "+tc.setting) // fromKP, which no setting lists, has the whole limit. line = s.uploadFrom(fromKP) if line.LimitHit != "" { t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit) } wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none) }) } } func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T) { t.Parallel() for _, tc := range []struct { name string env map[string]string // limitPercent and bytesPercent are the log line's, as percentText // gives them, and limitHit is its limit_hit. limitPercent, bytesPercent, limitHit string }{ { "lowering the byte limits alone", map[string]string{asnBytesPercent: asnDEHalf}, none, "50 from " + asnBytesPercent, minuteBytes, }, { "lowering the byte limits alone, by country", map[string]string{countryBytesPercent: countryDEHalf}, none, "50 from " + countryBytesPercent, minuteBytes, }, { "raising the byte limits back", map[string]string{asnLimitPercent: asnDEHalf, asnBytesPercent: asnDE + ":100"}, "50 from " + asnLimitPercent, none, "", }, { "raising the byte limits back, by country", map[string]string{countryLimitPercent: countryDEHalf, countryBytesPercent: "de:100"}, "50 from " + countryLimitPercent, none, "", }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() env := map[string]string{bytesLimitPerMinute: twoUploads} maps.Copy(env, tc.env) s, _, _ := startWithLookups(t, env) // The upload's 100 bytes are over 99, half of 199, and within 199. line := s.uploadFrom(fromDE) if line.LimitHit != tc.limitHit { t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit) } wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, tc.limitPercent) wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, tc.bytesPercent) }) } } func TestZeroPercentIsAZeroAllowance(t *testing.T) { t.Parallel() s, _, _ := startWithLookups(t, map[string]string{asnLimitPercent: asnDE + ":0"}) // The first request breaks the limit, and bans the client; the log line // gives the 0. line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) if line.fields["limit_percent"] != float64(0) || line.fields["limit_percent_setting"] != asnLimitPercent { t.Errorf("log line has limit_percent %v from %v, want 0 from %s", line.fields["limit_percent"], line.fields["limit_percent_setting"], asnLimitPercent) } s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) } func TestLowestPercentageApplies(t *testing.T) { t.Parallel() for _, tc := range []struct { name string env map[string]string from string // want is the log line's limit_percent, as percentText gives it. want string }{ { "the country's", map[string]string{asnLimitPercent: asnDEHalf, countryLimitPercent: "de:25"}, fromDE, "25 from " + countryLimitPercent, }, { "the AS number's", map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: countryDEHalf}, fromDE, "25 from " + asnLimitPercent, }, { "the AS number's, the first of two alike", map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: "de:25"}, fromDE, "25 from " + asnLimitPercent, }, { "that for a client without a country", map[string]string{asnLimitPercent: "AS64500:50", unknownLimitPercent: "25"}, noCountry, "25 from " + unknownLimitPercent, }, { // SWWAF_UNKNOWN_LIMIT_PERCENT is left at its default, 100. "the AS number's, for a client without a country", map[string]string{asnLimitPercent: "AS64500:25"}, noCountry, "25 from " + asnLimitPercent, }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() env := map[string]string{rateLimitPerMinute: fourAMinute} maps.Copy(env, tc.env) s, _, _ := startWithLookups(t, env) // A quarter of 4 requests a minute: the second breaks the limit. s.get(tc.from, http.StatusOK, requestlog.ActionForward) line := s.get(tc.from, http.StatusForbidden, requestlog.ActionRateLimited) wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, tc.want) }) } } func TestUnknownLimitPercentGivesEveryClientWithoutACountryItsPercentage(t *testing.T) { t.Parallel() s, _, _ := startWithLookups(t, map[string]string{ rateLimitPerMinute: fourAMinute, unknownLimitPercent: "50", }) // One the lookup database does not hold, and one on a private address, // which is never looked up: the third request of each breaks half of 4. for _, from := range []string{unplaced, "10.0.0.8"} { s.get(from, http.StatusOK, requestlog.ActionForward) s.get(from, http.StatusOK, requestlog.ActionForward) s.get(from, http.StatusForbidden, requestlog.ActionRateLimited) } // One in a country has the whole limit. for range 3 { s.get(fromDE, http.StatusOK, requestlog.ActionForward) } } func TestClientWithoutAnAnswerInTimeHasTheUnknownLimitPercent(t *testing.T) { t.Parallel() // In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays // says, with a GeoJS that never answers. synctest.Test(t, func(t *testing.T) { server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL, time.Now, map[string]string{unknownLimitPercent: "0"}) // Once the second the request waits for its answer is up, the client // counts as without a country, and its zero allowance refuses the // request before it reaches the app. serveFromDE(t, server, http.MethodGet, http.NoBody) line := out.requestLine(t) wantLine(t, line, http.StatusForbidden, requestlog.ActionRateLimited) wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, "0 from "+unknownLimitPercent) }) } func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) { t.Parallel() const timeout = 3 * time.Second for _, tc := range []struct { setting, value string waits bool }{ {asnLimitPercent, asnDEHalf, true}, {countryLimitPercent, countryDEHalf, true}, {asnBytesPercent, asnDEHalf, true}, {countryBytesPercent, countryDEHalf, true}, {unknownLimitPercent, "99", true}, {asnLimitPercentURL, asnURL, true}, // At 100, its default, it lowers no limit. {unknownLimitPercent, "100", false}, } { t.Run(tc.setting+"="+tc.value, func(t *testing.T) { t.Parallel() // In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays // says, with a GeoJS that never answers. synctest.Test(t, func(t *testing.T) { // The request's body is over SWWAF_REQUEST_MAX_BYTES, so that it // is refused after the checks, and never reaches the app. server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL, time.Now, map[string]string{ lookupTimeout: timeout.String(), requestMaxBytes: "1", tc.setting: tc.value, }) began := time.Now() serveFromDE(t, server, http.MethodPost, strings.NewReader("ab")) want := time.Duration(0) if tc.waits { want = timeout } if waited := time.Since(began); waited != want { t.Errorf("the request waited %s for its answer, want %s", waited, want) } wantLine(t, out.requestLine(t), http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge) // The bubble's clock stops once this function returns, so the // request to GeoJS, which a request that did not wait leaves // under way, has to be abandoned before then. time.Sleep(timeout) }) }) } } func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T) { t.Parallel() for _, tc := range []struct { name string env map[string]string // before is how many uploads come before the one that breaks a // limit, which is answered with status and logged with action. before int status int action string // reason and want are the ban's reason, and its notes' limit // percentage, as percentText gives it. reason, want string }{ { // A quarter of 12 requests a minute is 3: the fourth breaks it. "a rate limit", map[string]string{rateLimitPerMinute: "12", asnLimitPercent: asnDEQuarter}, 3, http.StatusForbidden, requestlog.ActionRateLimited, "requests per minute over the limit of 3", "25 from " + asnLimitPercent, }, { // The byte limits' percentage, not the rate limits'. "a byte limit", map[string]string{ bytesLimitPerMinute: twoUploads, asnLimitPercent: asnDEQuarter, asnBytesPercent: asnDEHalf, }, 0, http.StatusOK, requestlog.ActionForward, "bytes per minute over the limit of 99", "50 from " + asnBytesPercent, }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() s, server, queue := startWithLookups(t, tc.env) for range tc.before { s.uploadFrom(fromDE) } s.requestWithBody(http.MethodPost, fromDE, "/", uploadHeader, uploadBody, tc.status, tc.action) held := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32")) if len(held) != 1 { t.Fatalf("bans %+v, want one", held) } notes := held[0].Notes if held[0].Reason != tc.reason { t.Errorf("the ban's reason is %q, want %q", held[0].Reason, tc.reason) } wantPercent(t, "the notes' limit_percent", notes.LimitPercent, notes.LimitPercentSetting, tc.want) waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 { t.Fatalf("%d alerts wait, want the ban's alone: %+v", len(waiting), waiting) } alerted, _ := waiting[0].Detail["notes"].(bans.Notes) wantPercent(t, "the alert's notes' limit_percent", alerted.LimitPercent, alerted.LimitPercentSetting, tc.want) }) } } // startWithLookups is startWithLookupsAndClock for a test that needs no // clock. func startWithLookups( t *testing.T, env map[string]string, ) (*sender, *proxy.Server, *alerts.Queue) { t.Helper() s, _, server, queue := startWithLookupsAndClock(t, env) return s, server, queue } // startWithLookupsAndClock is startAppWithAlerts in front of // readAndAnswer, with the settings in env on top of clients looked up in a // lookup database, which places fromDE and fromKP in the AS numbers and // countries the stand-in for GeoJS gives them, noCountry in AS64500 and no // country, and no other address. func startWithLookupsAndClock( t *testing.T, env map[string]string, ) (*sender, *clock, *proxy.Server, *alerts.Queue) { t.Helper() path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") lookuptest.Write(t, path, map[string]lookuptest.Network{ fromDE + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"}, fromKP + "/32": {ASN: asnKP, ASName: asNameKP, Country: "KP"}, noCountry + "/32": {ASN: "AS64500", ASName: "Nowhere Net"}, }) settings := map[string]string{lookupSource: fileSource, lookupDBPath: path} maps.Copy(settings, env) return startAppWithAlerts(t, readAndAnswer, settings) } // uploadFrom is upload from the client at from. func (s *sender) uploadFrom(from string) logLine { s.t.Helper() line, _ := s.requestWithBody(http.MethodPost, from, "/", uploadHeader, uploadBody, http.StatusOK, requestlog.ActionForward) return line } // serveFromDE hands a request from fromDE with method and body straight to // server's handler, without the network, and returns once it is answered. func serveFromDE(t *testing.T, server *proxy.Server, method string, body io.Reader) { t.Helper() req := httptest.NewRequestWithContext(t.Context(), method, "/", body) req.RemoteAddr = net.JoinHostPort(fromDE, "1234") server.Handler.ServeHTTP(httptest.NewRecorder(), req) } // wantPercent checks a limit percentage that a log line or a ban's notes // give, what, and the setting that gave it, against want, as percentText // gives them. func wantPercent(t *testing.T, what string, percent *int64, setting, want string) { t.Helper() if got := percentText(percent, setting); got != want { t.Errorf("%s is %s, want %s", what, got, want) } } // percentText gives a limit percentage and the setting that gave it as // text, such as "50 from SWWAF_ASN_LIMIT_PERCENT", or none when both are // left out. func percentText(percent *int64, setting string) string { switch { case percent == nil && setting == "": return none case percent == nil: return "none from " + setting default: return fmt.Sprintf("%d from %s", *percent, setting) } }