diff --git a/README.md b/README.md index 712ad5a..8677eb7 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,15 @@ reference documents under [`docs/`](docs/) are listed under "Documents" below. ## What it protects against - Scrapers that fetch every page of a site, all day. Rate limiting gives each - client 6000 requests a minute, 100,000 an hour and 1,000,000 a day. Byte - limits give it 4 GiB a minute, 20 GiB an hour and 200 GiB a day. A client over - a limit is refused and banned. The first ban lasts an hour. Each repeat within - a day makes the next ban three times as long. A ban that would last more than - seven days is permanent. + client 30,000 requests an hour and 300,000 a day, with no limit per minute. + That leaves room for programmatic use of an API, such as an automation fleet + on another host. The hour limit catches a crawler that keeps up 10 requests a + second. The day limit catches one that keeps up 4 a second around the clock. + Counting cannot catch a slower client; only the protections below can. Byte + limits give each client 4 GiB a minute, 20 GiB an hour and 200 GiB a day. A + client over a limit is refused and banned. The first ban lasts an hour. Each + repeat within a day makes the next ban three times as long. A ban that would + last more than seven days is permanent. - Scanners that probe for secrets, such as `/.env` or `/.git/config`. The default rule file bans such a client for seven days at its first probe. Its next request during the ban makes the ban permanent. The default rule file diff --git a/SPEC.md b/SPEC.md index 0b3d7b6..d0a6b19 100644 --- a/SPEC.md +++ b/SPEC.md @@ -369,22 +369,31 @@ Conventions: owner's own networks. `SWWAF_RATE_LIMIT_EXEMPT_NETS` bypasses the request and byte limits only; the error burst, attack detection and bans still apply. - Many people can share one address, as in an office, on a mobile carrier or - through Tor. They are one client, and share its limits and its bans. The - default limits are sized for about 40 busy people on one address. A known - shared address busier than that belongs in `SWWAF_ALLOW_NETS` or + through Tor. They are one client, and share its limits and its bans. A known + shared address busier than the limits allow belongs in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`. ### Request rate limits, per client -Breaking one bans the client, so the defaults sit about twice above what 40 -people busy with heavy web apps make together on one address. Each such person -makes about 75 requests in a busy minute, and 1,250 an hour for ten hours a day. +Breaking one bans the client. The defaults suit programmatic use of an API, such +as an automation fleet on another host, and still stop abusive bots. They count +per hour and per day, so a short burst of requests breaks no limit on its own. -A client slower than those 40 people cannot be told from them by counting. Only -the other checks can catch an abuser that slow. They include the trap paths, the -rule files, the Core Rule Set and the blocklists. An abuser that trips none of -them is not caught. A machine that talks to the app all day can still pass the -day limit. Its address belongs in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. +- Per minute: `off`. +- Per hour: 30,000, about 8 requests a second for a whole hour. A crawler that + keeps up 10 a second makes 36,000 an hour, and passes it. +- Per day: 300,000, about 3.5 requests a second around the clock. A crawler that + keeps up 4 a second makes 345,600 a day, and passes it. + +A fleet of 20 agents, each making a request every 10 seconds, makes 7,200 +requests an hour. Working around the clock, it makes 172,800 a day. Both stay +under the limits. + +Counting cannot catch a client slower than these limits. Only the other checks +can catch an abuser that slow. They include the trap paths, the rule files, the +Core Rule Set and the blocklists. An abuser that trips none of them is not +caught. A busier machine that talks to the app all day can still pass the day +limit. Its address belongs in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. `SWWAF_RATE_LIMIT_EXEMPT_PATHS` takes path prefixes out of the count, such as static assets or health checks; their bytes are not counted either. A prefix is @@ -397,12 +406,12 @@ matched the same way. ### Byte limits, per client -Each of the same 40 people moves about 256 MiB an hour, for ten hours a day. The -hour and day limits sit about twice above what they move together. A response's -bytes are counted when it ends (see step 15). So the minute limit holds two of -the largest responses, with their requests, ending in the same minute. No single -download then breaks a limit. An operator who raises `SWWAF_RESPONSE_MAX_BYTES` -raises the byte limits with it. +The hour and day limits sit about twice above what 40 people busy with heavy web +apps move together on one address. Each moves about 256 MiB an hour, for ten +hours a day. A response's bytes are counted when it ends (see step 15). So the +minute limit holds two of the largest responses, with their requests, ending in +the same minute. No single download then breaks a limit. An operator who raises +`SWWAF_RESPONSE_MAX_BYTES` raises the byte limits with it. ### Size and time limits diff --git a/docs/alerts.md b/docs/alerts.md index 916912d..e22a590 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -42,7 +42,7 @@ on one line: "asn": "", "as_name": "", "country": "", - "reason": "requests per minute over the limit of 6000", + "reason": "requests per hour over the limit of 30000", "detail": { "ban_expires": "2026-10-06T13:00:00.123Z", "cause": "limit", @@ -51,9 +51,9 @@ on one line: "as_name": "", "country": "", "kind": "requests", - "limit": 6000, - "window": "minute", - "count": 6001, + "limit": 30000, + "window": "hour", + "count": 30001, "request": { "time": "2026-10-06T12:00:00.123456789Z", "method": "GET", @@ -62,7 +62,7 @@ on one line: "status": 403, "user_agent": "scraper/1.0" }, - "requests": 8210, + "requests": 30001, "refused": 0, "earlier_bans": { "limit": 0, @@ -89,7 +89,7 @@ on one line: - `asn`, `as_name` and `country`: the client's, as known when the alert was raised. They are empty when GeoJS had not answered yet. - `reason`: a short sentence, such as - `requests per minute over the limit of 6000` or `listed by a blocklist`. + `requests per hour over the limit of 30000` or `listed by a blocklist`. - `detail`: what is particular to the event: - for a ban, its `cause`, `ban_expires` and `notes` (the notes are listed under `bans.json` in [`state-files.md`](state-files.md)); diff --git a/docs/gitea.md b/docs/gitea.md index df40899..bdbc923 100644 --- a/docs/gitea.md +++ b/docs/gitea.md @@ -34,8 +34,8 @@ checked again when gitea or the Core Rule Set changes version. day. One older than version 0.4 (April 2026) asks for work every 2 seconds, 43,200 requests a day. While it runs a job, it reports the job's log and state every second instead, two requests a second. So one runner busy all day sends - 172,800 requests, under the request limits. Six such runners on one address, - each busy all day, pass the day limit of 1,000,000 within two days; five stay + 172,800 requests, under the request limits. Two such runners on one address, + each busy all day, make 345,600 and pass the day limit of 300,000; one stays under it. Their address is then banned, their running jobs fail, and each repeat within a day makes the ban longer. Newer runners ask for work less often. Put the runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. diff --git a/docs/settings.md b/docs/settings.md index ad41aee..51de7dc 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -99,9 +99,18 @@ that take `off`, the `_FILE` form, and what each group does and why. ## Request rate limits, per client -- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `6000`), `SWWAF_RATE_LIMIT_PER_HOUR` - (default `100000`) and `SWWAF_RATE_LIMIT_PER_DAY` (default `1000000`): the - most requests a client may make in each window. A count or `off`. +- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `off`): the most requests a client may + make in a minute. A count or `off`. It is off by default so that a short burst + of API requests breaks no limit on its own. +- `SWWAF_RATE_LIMIT_PER_HOUR` (default `30000`, about 8 a second): the most + requests a client may make in an hour. A count or `off`. The default leaves + room for an automation fleet at work, and catches a crawler that keeps up 10 a + second. +- `SWWAF_RATE_LIMIT_PER_DAY` (default `300000`, about 3.5 a second): the most + requests a client may make in a day. A count or `off`. The default catches a + crawler that keeps up 4 a second around the clock. +- Counting cannot catch a client slower than these limits (see "Request rate + limits, per client" in [`SPEC.md`](../SPEC.md)). - `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. diff --git a/internal/config/config.go b/internal/config/config.go index 0cc5def..61fc677 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -506,9 +506,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""), RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""), DenyNets: env.netblocks("SWWAF_DENY_NETS", ""), - RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "6000"), - RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "100000"), - RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "1000000"), + RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", off), + RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "30000"), + RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "300000"), RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "4G"), BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ff61910..3a2ac28 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -262,9 +262,9 @@ func TestDefaults(t *testing.T) { UpstreamResponseTimeout: 30 * time.Minute, RequestMaxBytes: 100 << 20, ResponseMaxBytes: 1 << 30, - RateLimitPerMinute: 6000, - RateLimitPerHour: 100000, - RateLimitPerDay: 1000000, + RateLimitPerMinute: 0, + RateLimitPerHour: 30000, + RateLimitPerDay: 300000, BanResponse: 403, LimitBanDuration: time.Hour, LimitBanRepeatWindow: 24 * time.Hour, @@ -2375,9 +2375,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { allowNets: "", rateLimitExemptNets: "", denyNets: "", - rateLimitPerMinute: "6000", - rateLimitPerHour: "100000", - rateLimitPerDay: "1000000", + rateLimitPerMinute: off, + rateLimitPerHour: "30000", + rateLimitPerDay: "300000", rateLimitExemptPaths: "", bytesLimitPerMinute: "4G", bytesLimitPerHour: "20G", diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 0aee935..ad9b686 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -54,6 +54,8 @@ const ( greeting = "hello from the app" // placed is the client the tests' lookup databases place. placed = "203.0.113.9" + // off is the value that switches a setting off. + off = "off" ) // output collects what smallwebwaf writes on stdout. @@ -118,7 +120,7 @@ func run(ctx context.Context, env map[string]string, out *output) int { LookupEnv: func(name string) (string, bool) { value, ok := env[name] if !ok && name == "SWWAF_LOOKUP_SOURCE" { - return "off", true + return off, true } return value, ok @@ -502,7 +504,7 @@ func TestRuleFileAddedWhileRunningTakesEffect(t *testing.T) { rulesDir: dir, // The requests sent until the rule takes effect must not break a // rate limit, whose ban would refuse them too. - "SWWAF_RATE_LIMIT_PER_MINUTE": "off", + "SWWAF_RATE_LIMIT_PER_MINUTE": off, } out := runUntilStopped(t, env, func(url string) { @@ -1244,9 +1246,9 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { "SWWAF_ALLOW_NETS": "", rateLimitExemptNets: "", "SWWAF_DENY_NETS": "", - "SWWAF_RATE_LIMIT_PER_MINUTE": "6000", - "SWWAF_RATE_LIMIT_PER_HOUR": "100000", - rateLimitPerDay: "1000000", + "SWWAF_RATE_LIMIT_PER_MINUTE": off, + "SWWAF_RATE_LIMIT_PER_HOUR": "30000", + rateLimitPerDay: "300000", "SWWAF_RATE_LIMIT_EXEMPT_PATHS": "", "SWWAF_DENIED_COUNTRIES": "", "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",