From 611ae5c0082894ed87e54cdf9c96882175667b88 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 10:06:38 +0000 Subject: [PATCH] Leave SWWAF_RATE_LIMIT_EXEMPT_PATHS out of the request rate limits (closes #77) A request whose path, as the client sent it and before the query string, starts with one of the comma-separated prefixes in SWWAF_RATE_LIMIT_EXEMPT_PATHS is neither counted nor refused by the request rate limits; the static lists, bans and the country lists still apply. The setting is empty by default, and a prefix that does not start with / stops the start. README.md documents it. Judgement call: plain prefix on the path as sent, as the issue rules, so /assets/../login matches /assets/; traefik removes such dot segments by default, but a request that reaches smallwebwaf uncleaned is matched as sent. Model: opus-5-5 --- README.md | 37 +++++++++++++++--------- internal/config/config.go | 34 +++++++++++++++++++++- internal/config/config_test.go | 26 +++++++++++++++++ internal/proxy/proxy_test.go | 1 + internal/proxy/ratelimits_test.go | 32 ++++++++++++++++++++ internal/proxy/request.go | 16 ++++++++-- internal/smallwebwaf/smallwebwaf_test.go | 1 + 7 files changed, 131 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0c020a0..c230682 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,14 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are five parts of -milestone 3: the static lists, the bans that broken rate limits lead to and the -JSON state files, which come next in the build order, and the metrics endpoint -and the header size and the idle time as settings, which come last in it. -`smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -bans a client that sends too many requests, refuses a client that comes from a +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are six parts of +milestone 3: the static lists, the bans that broken rate limits lead to, the +JSON state files and the paths the rate limits do not count, which come next in +the build order, and the metrics endpoint and the header size and the idle time +as settings, which come last in it. `smallwebwaf` passes each request to the app +and the app's answer back, unchanged, within its timeouts and size limits, works +out each client's address, bans a client that sends too many requests, not +counting those for the paths you choose, refuses a client that comes from a country you refuse or from a network you refuse, lets the networks you choose through, keeps its bans, each client's counters and history, and GeoJS's answers in JSON files across restarts, writes a JSON log line for every request, and @@ -77,12 +78,14 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. - Counts each client's requests over a minute, an hour and a day. A request that takes the client over one of the rate limits below is refused with `SWWAF_BAN_RESPONSE`, `403` by default, before anything reaches the app, and - bans the client. A client is one IPv4 address, or one IPv6 /64, since one - abuser usually holds a whole /64. Each window is counted in two fixed buckets, - the earlier one weighted by how much of it the window still covers. At most - 20,000 clients are kept, the least recently seen dropped first, with their - history, and a restart gives no client a fresh allowance (see "State files" - below). + bans the client. A request whose path starts with one of + `SWWAF_RATE_LIMIT_EXEMPT_PATHS` is neither counted nor refused by the rate + limits; the static lists, bans and the country lists still apply to it. A + client is one IPv4 address, or one IPv6 /64, since one abuser usually holds a + whole /64. Each window is counted in two fixed buckets, the earlier one + weighted by how much of it the window still covers. At most 20,000 clients are + kept, the least recently seen dropped first, with their history, and a restart + gives no client a fresh allowance (see "State files" below). - Bans a client that breaks a rate limit, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit broken again within a day of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and @@ -174,6 +177,14 @@ it, and the effective settings are logged at start. requests a client may make in a minute, an hour and a day. The defaults are several times what one busy person produces, since a browser loading a heavy page makes a few hundred requests and several people often share one address. +- `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes whose requests + the rate limits neither count nor refuse, such as `/assets/` for static + assets; each starts with `/`. A prefix is compared with the start of the path + as the client sent it, before any query string, as the request log's `path` + shows it, character for character: `/assets/` matches `/assets/app.js` and + `/assets/img/logo.png`, but not `/assets`, `/Assets/app.js` or + `/static/assets/app.js`. There are no wildcards: `*` is a character like any + other. - `SWWAF_DENIED_COUNTRIES` (default empty): countries whose clients are refused, for example `cn,ru,kp`. - `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only diff --git a/internal/config/config.go b/internal/config/config.go index 65281c1..35b533a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -70,6 +70,10 @@ type Config struct { RateLimitPerMinute int64 RateLimitPerHour int64 RateLimitPerDay int64 + // RateLimitExemptPaths are the path prefixes whose requests the rate + // limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_PATHS). + // Each starts with /. + RateLimitExemptPaths []string // DeniedCountries are the countries whose clients are refused // (SWWAF_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not // empty, are the only countries whose clients are let through @@ -161,7 +165,9 @@ var ( "is not the length of an IPv4 netblock, from 0 to 32, such as 24") errNotAbsolutePath = errors.New( "is not an absolute path, such as /var/lib/smallwebwaf") - errShortToken = errors.New("is shorter than 32 characters") + errShortToken = errors.New("is shorter than 32 characters") + errNotPathPrefix = errors.New( + "is not a path prefix starting with /, such as /assets/") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -188,6 +194,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"), RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), + RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), @@ -315,6 +322,14 @@ func (e *environment) count(name, defaultValue string) int64 { return count } +// pathPrefixes reads a setting that is a list of path prefixes. +func (e *environment) pathPrefixes(name, defaultValue string) []string { + prefixes, err := parsePathPrefixes(e.value(name, defaultValue)) + e.check(name, err) + + return prefixes +} + // countries reads a setting that is a list of countries. func (e *environment) countries(name, defaultValue string) []string { countries, err := parseCountries(e.value(name, defaultValue)) @@ -593,6 +608,23 @@ func parseNetblock(value string) (netip.Prefix, error) { return netip.PrefixFrom(addr, addr.BitLen()), nil } +// parsePathPrefixes reads a comma-separated list of path prefixes, each +// starting with /. +func parsePathPrefixes(value string) ([]string, error) { + prefixes, err := parseList(value) + if err != nil { + return nil, err + } + + for _, prefix := range prefixes { + if !strings.HasPrefix(prefix, "/") { + return nil, fmt.Errorf("%q %w", prefix, errNotPathPrefix) + } + } + + return prefixes, nil +} + // countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK, // the code in common use for Kosovo. golang.org/x/text/language cannot // check them: it also takes withdrawn codes such as su, and reserved ones diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 955ff70..6c0fe42 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -33,6 +33,7 @@ const ( rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWWAF_BAN_RESPONSE" @@ -118,6 +119,10 @@ func TestDefaults(t *testing.T) { wantNetblocks(t, cfg.DenyNets) wantCountries(t, deniedCountries, cfg.DeniedCountries) wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries) + + if len(cfg.RateLimitExemptPaths) != 0 { + t.Errorf("%s gave %v, want none", rateLimitExemptPaths, cfg.RateLimitExemptPaths) + } } func TestValuesAsSet(t *testing.T) { @@ -141,6 +146,7 @@ func TestValuesAsSet(t *testing.T) { rateLimitPerMinute: "60", rateLimitPerHour: "600", rateLimitPerDay: "6000", + rateLimitExemptPaths: "/assets/, /favicon.ico", deniedCountries: "cn, RU,kp,Xk", allowedCountries: "de", banResponse: "429", @@ -192,6 +198,24 @@ func TestValuesAsSet(t *testing.T) { wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24") wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK") wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") + + if !slices.Equal(cfg.RateLimitExemptPaths, []string{"/assets/", "/favicon.ico"}) { + t.Errorf("%s gave %v, want /assets/ and /favicon.ico", + rateLimitExemptPaths, cfg.RateLimitExemptPaths) + } +} + +func TestPathPrefixNotStartingWithSlashStopsTheStart(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment( + environment{rateLimitExemptPaths: "/favicon.ico,assets/"}.lookupEnv) + + want := rateLimitExemptPaths + `: "assets/" is not a path prefix ` + + `starting with /, such as /assets/` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } } func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { @@ -334,6 +358,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {rateLimitPerHour, "1.5"}, {rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"}, + {rateLimitExemptPaths, "/assets/,,/static/"}, {deniedCountries, "nk"}, {deniedCountries, "kp,,ir"}, {deniedCountries, "prk"}, @@ -441,6 +466,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerMinute: "1000", rateLimitPerHour: "10000", rateLimitPerDay: "50000", + rateLimitExemptPaths: "", deniedCountries: "", allowedCountries: "", banResponse: "403", diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 31334d0..dcb7e03 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -58,6 +58,7 @@ const ( denyNets = "SWWAF_DENY_NETS" rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWWAF_BAN_RESPONSE" diff --git a/internal/proxy/ratelimits_test.go b/internal/proxy/ratelimits_test.go index 8d57142..aa0122f 100644 --- a/internal/proxy/ratelimits_test.go +++ b/internal/proxy/ratelimits_test.go @@ -69,3 +69,35 @@ func TestRateLimitRefusesBeforeTheApp(t *testing.T) { t.Errorf("the app was called %d times, want 4", calls.Load()) } } + +func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) { + t.Parallel() + + const denied = "192.0.2.50" // in SWWAF_DENY_NETS + + s, _, _ := startWithClock(t, "", map[string]string{ + rateLimitPerMinute: "1", + rateLimitExemptPaths: "/assets/,/favicon.ico", + denyNets: denied, + }) + + // With a limit of one request a minute, the requests for paths under a + // prefix are not counted, so client's first request for / is within + // the limit; and once client has reached it, they are not refused. + s.request(client, "/assets/app.js", http.StatusOK, requestlog.ActionForward) + s.request(client, "/favicon.ico?v=2", http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusOK, requestlog.ActionForward) + + line := s.request(client, "/assets/app.js", http.StatusOK, requestlog.ActionForward) + if line.LimitHit != "" { + t.Errorf("log line has limit_hit %q, want none", line.LimitHit) + } + + // A path outside every prefix is counted: /assets is not under + // /assets/, and breaks the limit. + s.request(client, "/assets", http.StatusForbidden, requestlog.ActionRateLimited) + + // A ban and SWWAF_DENY_NETS still refuse a path under a prefix. + s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned) + s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied) +} diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 31fdb5c..3500b43 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -8,6 +8,8 @@ import ( "net/http/httputil" "net/netip" "os" + "slices" + "strings" "sync" "sync/atomic" "time" @@ -115,13 +117,16 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { // client either refuses is not looked up, and then the country lists; a // request any of them refuses is not counted for the rate limits. Then // come the rate limits, unless the client is in -// SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted, +// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path, as the client sent +// it and the log line shows it, starts with one of +// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted, // one refused for its size too. Every refusal but the size limit's is // answered with SWWAF_BAN_RESPONSE. ctx is the request's own context. func (rq *request) check(ctx context.Context) *refusal { cfg := rq.h.config allowed := isInside(rq.client, cfg.AllowNets) - exempt := isInside(rq.client, cfg.RateLimitExemptNets) + exempt := isInside(rq.client, cfg.RateLimitExemptNets) || + startsWithAny(rq.in.URL.EscapedPath(), cfg.RateLimitExemptPaths) now := rq.h.now() if !allowed && isInside(rq.client, cfg.DenyNets) { @@ -152,6 +157,13 @@ func (rq *request) check(ctx context.Context) *refusal { return nil } +// startsWithAny reports whether path starts with one of prefixes. +func startsWithAny(path string, prefixes []string) bool { + return slices.ContainsFunc(prefixes, func(prefix string) bool { + return strings.HasPrefix(path, prefix) + }) +} + // forward passes the request to the app and the app's answer back. ctx // is the request's own context. func (rq *request) forward(ctx context.Context) { diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index f0ddb88..ca6a9e3 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -400,6 +400,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", "SWWAF_RATE_LIMIT_PER_HOUR": "10000", rateLimitPerDay: "50000", + "SWWAF_RATE_LIMIT_EXEMPT_PATHS": "", "SWWAF_DENIED_COUNTRIES": "", "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", "SWWAF_BAN_RESPONSE": "403",