From 21cac70b5002292cbea96d8c67bcebbfd4b8eefe Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 6 Nov 2026 20:14:23 +0100 Subject: [PATCH] Lower the default request and byte limits (closes #193) New defaults, as sneak approved: 120 requests a minute, 1,500 an hour and 6,000 a day; bytes off a minute, 50G an hour and 100G a day. The README, `SPEC.md`, `docs/settings.md` and `docs/gitea.md` give the new figures and reasoning, and point Actions runners and shared addresses at `SWWAF_RATE_LIMIT_EXEMPT_NETS`. The README examples now hold against the new defaults. An Actions runner that is not exempted makes more requests an hour than the new limit and will be banned. Model: opus-5-5 --- README.md | 38 +++++++--------- SPEC.md | 55 +++++++++++------------- deploy/gitea/compose.yaml | 2 +- docs/gitea.md | 35 ++++++--------- docs/settings.md | 33 +++++++------- internal/config/config.go | 12 +++--- internal/config/config_test.go | 22 +++++----- internal/smallwebwaf/smallwebwaf_test.go | 6 +-- 8 files changed, 95 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 64d0967..bf6cdb6 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ It protects against: - Scrapers that fetch every page of a site, all day. Each client gets a limit on requests and on bytes per minute, per hour and per day. The defaults leave - room for programmatic use of an API, such as an automation fleet, and catch a - client firing requests as fast as it can within seconds. A client over a limit - is refused and banned: for a while at first, for three times as long at each - repeat, and permanently when it keeps at it. + room for a person reading a site for hours. They catch a scraper that keeps up + one request every 2 seconds. A client over a limit is refused and banned: for + a while at first, for three times as long at each repeat, and permanently when + it keeps at it. - Scanners that probe for secrets, such as `/.env` or `/.git/config`. The default rule file bans such a client at its first probe, and its next request during the ban makes the ban permanent. Your own rules, or a list of trap @@ -236,13 +236,13 @@ Every setting the code reads, with its default: | `SWWAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. | | `SWWAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients the request and byte limits neither count nor refuse. | | `SWWAF_DENY_NETS` | empty | Netblocks whose clients are refused. | -| `SWWAF_RATE_LIMIT_PER_MINUTE` | `500` | The most requests a client may make in a minute, a count or `off`; one more bans it. | -| `SWWAF_RATE_LIMIT_PER_HOUR` | `20000` | The most requests a client may make in an hour, a count or `off`; one more bans it. | -| `SWWAF_RATE_LIMIT_PER_DAY` | `300000` | The most requests a client may make in a day, a count or `off`; one more bans it. | +| `SWWAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; one more bans it. | +| `SWWAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; one more bans it. | +| `SWWAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; one more bans it. | | `SWWAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes whose requests the request and byte limits neither count nor refuse, such as `/assets/`. | -| `SWWAF_BYTES_LIMIT_PER_MINUTE` | `4G` | The most bytes a client's requests may carry in a minute, a size or `off`; the request that takes a client past any byte limit bans it. | -| `SWWAF_BYTES_LIMIT_PER_HOUR` | `20G` | The most bytes a client's requests may carry in an hour, a size or `off`. | -| `SWWAF_BYTES_LIMIT_PER_DAY` | `200G` | The most bytes a client's requests may carry in a day, a size or `off`. | +| `SWWAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes a client's requests may carry in a minute, a size or `off`; the request that takes a client past any byte limit bans it. | +| `SWWAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes a client's requests may carry in an hour, a size or `off`. | +| `SWWAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes a client's requests may carry in a day, a size or `off`. | | `SWWAF_BYTES_COUNT` | `both` | Which body bytes the byte limits count: `response`, `request` or `both`. | | `SWWAF_ERROR_BURST_THRESHOLD` | `30` | The most refusals by a rule file, a trap path or the Core Rule Set a client may get in a minute, a count or `off`; one more bans it. | | `SWWAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. | @@ -407,10 +407,10 @@ services: SWWAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7 # A netblock that sends nothing wanted. SWWAF_DENY_NETS: 198.51.100.128/25 - # Tighter limits for everyone else: a site with no API. - SWWAF_RATE_LIMIT_PER_MINUTE: "120" - SWWAF_RATE_LIMIT_PER_HOUR: "3000" - SWWAF_RATE_LIMIT_PER_DAY: "30000" + # Tighter limits for everyone else: a small static site. + SWWAF_RATE_LIMIT_PER_MINUTE: "60" + SWWAF_RATE_LIMIT_PER_HOUR: "600" + SWWAF_RATE_LIMIT_PER_DAY: "3000" # Hosting networks and one country get a share of those limits. SWWAF_ASN_LIMIT_PERCENT: AS14061:50,AS16276:25 SWWAF_COUNTRY_LIMIT_PERCENT: cn:25 @@ -431,10 +431,9 @@ volumes: gitea behind traefik, set up for a code forge: - the upload limits raised, in size and in time; -- the response size limit and the byte limits raised, for clones over 1 GiB; +- the response size limit raised, for clones over 1 GiB; - the Actions runners exempt from counting; -- the default request limits kept, since they already leave room for API - clients. +- the default request and byte limits kept. `traefik` is the network traefik is on, made outside this file. @@ -488,11 +487,6 @@ services: SWWAF_INSTANCE_NAME: fsn1/servewaf # The Actions runners ask for work all day; never count them. SWWAF_RATE_LIMIT_EXEMPT_NETS: 10.0.4.0/24 - # Raised with rproxy's response limit, to four times the default - # each, so that one clone of that size breaks no byte limit. - SWWAF_BYTES_LIMIT_PER_MINUTE: 16G - SWWAF_BYTES_LIMIT_PER_HOUR: 80G - SWWAF_BYTES_LIMIT_PER_DAY: 800G # gitea never serves these; a request for one bans the client. SWWAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php,/wp-admin/ volumes: diff --git a/SPEC.md b/SPEC.md index 37db7db..875ba19 100644 --- a/SPEC.md +++ b/SPEC.md @@ -397,32 +397,31 @@ Conventions: ### Request rate limits, per client -Breaking one bans the client. The defaults suit programmatic use of an API, such -as an automation fleet on another host, and still stop abusive bots. A fleet of -20 agents, each making a request every 10 seconds at its busiest, makes 120 -requests a minute and 7,200 an hour. Working around the clock, it makes 172,800 -a day. It stays under all three limits. +Breaking one bans the client. The defaults let people read a site and stop +scrapers that send many requests slowly over hours or days. The hour limit is +the main one. -- Per minute: 500, about 8 requests a second. It leaves room for the fleet's 120 - a minute and a few quick first visits with an empty cache. A gitea page opened - with an empty cache loads about 40 files. A client firing requests as fast as - it can passes the limit within seconds. -- Per hour: 20,000, about 5.5 requests a second for a whole hour. A client that - keeps up 6 a second makes 21,600 in an hour, and passes it, at 360 a minute, - under the minute limit. The fleet's 7,200 an hour is about a third of it. -- Per day: 300,000, about 3.5 requests a second around the clock. A scraper that - keeps up 4 a second makes 345,600 a day, and passes it. At 240 a minute and - 14,400 an hour, it stays under the other two limits. +- Per minute: 120, 2 requests a second. It leaves room for a person opening two + pages at once with an empty cache. A gitea page opened with an empty cache + loads 20 to 60 files, and a handful once cached. +- Per hour: 1,500. A person reading code in gitea for a whole hour, a page every + 20 seconds, makes about 900 requests. A scraper making one request every 2 + seconds makes 1,800 an hour, more than the limit. +- Per day: 6,000, four hours at the hour limit. A scraper making one request + every 10 seconds makes 8,640 a day, more than the limit. -Counting cannot catch a client that stays under all three limits, such as one -that keeps up 3 requests a second. Only the other checks can catch an abuser +Counting cannot catch a client that stays under all three limits, about 4 +requests a minute around the clock. Only the other checks can catch an abuser that slow. They include the trap paths, the rule files, the Core Rule Set and -the blocklists. An abuser that trips none of them is not caught. A machine that -keeps up more than 500 requests a minute passes the minute limit. One that keeps -up more than about 5.5 a second for an hour passes the hour limit. One that -keeps up more than about 3.5 a second all day passes the day limit, as two busy -Actions runners on one address do. Such a machine's address belongs in -`SWWAF_RATE_LIMIT_EXEMPT_NETS`. +the blocklists. An abuser that trips none of them is not caught. + +Some legitimate clients pass the limits: + +- An Actions runner asks gitea for work all day and can pass the limits. +- Everyone behind one shared address, such as an office, shares one client's + limits. + +Their addresses belong in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. `SWWAF_RATE_LIMIT_EXEMPT_PATHS` takes path prefixes out of the count, such as static assets or health checks; their bytes are not counted either. A prefix is @@ -435,12 +434,10 @@ matched the same way. ### Byte limits, per client -The hour and day limits sit about twice above what 40 people busy with heavy web -apps move together on one address. Each moves about 256 MiB an hour, for ten -hours a day. A response's bytes are counted when it ends (see step 15). So the -minute limit holds three of the largest responses, with their requests, ending -in the same minute. No single download then breaks a limit. An operator who -raises `SWRP_MAX_RESPONSE_SIZE` must raise the byte limits too. +By default there is no minute limit, so a client can download in a burst at full +speed. The hour limit defaults to 50 GiB and the day limit to 100 GiB. They +allow several downloads of a few GiB each. A response's bytes are counted when +it ends (see step 15). ### Lookup of AS number and country diff --git a/deploy/gitea/compose.yaml b/deploy/gitea/compose.yaml index 3a9a7b5..d5652a7 100644 --- a/deploy/gitea/compose.yaml +++ b/deploy/gitea/compose.yaml @@ -102,7 +102,7 @@ services: SMALLWEBWAF_COMMAND: servewaf # The containers on the shared network skip the checks. Add the # address of an automation fleet on another host here too; without - # that, the default request limits are sized for such a fleet. + # that, it is counted like any client. SWWAF_ALLOW_NETS: 10.99.0.128/25,fd10:99:0:0:8000::/65 # In block mode the Core Rule Set refuses everyday Gitea searches # and file names, and 31 refusals in a minute ban the address. diff --git a/docs/gitea.md b/docs/gitea.md index eb10da6..fde4cca 100644 --- a/docs/gitea.md +++ b/docs/gitea.md @@ -72,11 +72,10 @@ A clone or a download is a response: - `SWRP_MAX_RESPONSE_TIME_MS` is `900000`, 15 minutes, half the default. A clone or download that takes longer is cut off. - `SWRP_MAX_RESPONSE_SIZE` stays at its default, 1 GiB. A repository whose clone - is larger needs it raised, and the byte limits with it ("Byte limits, per - client" in `SPEC.md`). + is larger needs it raised. -The byte limits stay at their defaults. Two uploads of 2 GiB from one address in -one minute pass the minute's limit of 4 GiB, and ban the address. +The byte limits stay at their defaults: no minute limit, 50 GiB an hour and 100 +GiB a day. ## The checks @@ -86,9 +85,9 @@ gitea searches and file names. The section "What the Core Rule Set refuses in `block` mode" below lists them. Each such refusal counts toward the error burst, so 31 in one minute ban the address. -Every other check stays at its default. A client may make 500 requests a minute, -20,000 an hour and 300,000 a day. Archive downloads and blame and history pages -are what scrapers hammer, and these limits do most of the work there. +Every other check stays at its default. A client may make 120 requests a minute, +1,500 an hour and 6,000 a day. Archive downloads and blame and history pages are +what scrapers hammer, and these limits do most of the work there. ## The shared network @@ -108,7 +107,7 @@ Containers on the shared network skip `servewaf`'s checks: - A container on the shared network that asks for the site at the host's public address arrives from the gateway. It is checked like any visitor. - A fleet on another host can have its address added to `SWWAF_ALLOW_NETS`. - Without that, the default request limits are sized for such a fleet. + Without that, it is counted like any client. - traefik replaces the `X-Forwarded-For` a visitor sends, by default. Keep it so: otherwise a visitor could name an address on the shared network, and skip the checks. @@ -145,20 +144,14 @@ in the query string, as `?token=`, is then in the log. Send it in the A runner on the shared network skips the checks. One on another host is counted like any client. A runner older than version 0.4 (April 2026) asks for work -every 2 seconds, 43,200 requests a day. While it runs a job, it reports the -job's log and state every second instead, two requests a second. So one runner -busy all day sends 120 requests a minute and 172,800 a day, under the request -limits. +every 2 seconds, 1,800 requests an hour. While it runs a job, it reports the +job's log and state every second instead, two requests a second. -Several runners on one address can pass them: - -- Two busy all day make 345,600 requests, and pass the day limit of 300,000. -- Three busy at once make 21,600 an hour, and pass the hour limit of 20,000. -- Five busy at once make 600 a minute, and pass the minute limit of 500. - -Their address is then banned, their running jobs fail, and each repeat within a -day makes the ban longer. Newer runners ask for work less often. Put the -runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. +So one idle runner makes 1,800 requests an hour, more than the hour limit of +1,500. A busy one, at two requests a second, passes the minute limit of 120 +within two minutes. Its address is then banned, its running jobs fail, and each +repeat within a day makes the ban longer. Newer runners ask for work less often. +Put the runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. ## What the Core Rule Set refuses in `block` mode diff --git a/docs/settings.md b/docs/settings.md index 3997b25..ca41840 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -99,19 +99,22 @@ chooses between the two (see "The image's command" below). ## Request rate limits, per client -- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `500`, about 8 a second): the most - requests a client may make in a minute. A count or `off`. The default catches, - within seconds, a client firing requests as fast as it can. It leaves room for - a fleet of about 20 agents at its busiest, about 120 requests a minute. It - also leaves room for a few quick first visits to gitea with an empty cache, - about 40 files each. -- `SWWAF_RATE_LIMIT_PER_HOUR` (default `20000`, about 5.5 a second): the most - requests a client may make in an hour. A count or `off`. The default leaves - room for an automation fleet at work. It catches a client that keeps up more - than about 5.5 a second for an hour. -- `SWWAF_RATE_LIMIT_PER_DAY` (default `300000`, about 3.5 a second): the most - requests a client may make in a day. A count or `off`. The default catches a - scraper that keeps up 4 a second around the clock. +- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `120`, 2 a second): the most requests a + client may make in a minute. A count or `off`. The default leaves room for a + person opening two gitea pages at once with an empty cache, up to 60 files + each. +- `SWWAF_RATE_LIMIT_PER_HOUR` (default `1500`): the most requests a client may + make in an hour. A count or `off`. It is the main limit against scrapers. A + person reading code in gitea for an hour, a page every 20 seconds, makes + about 900. A scraper making one request every 2 seconds makes 1,800 an hour, + more than the limit. +- `SWWAF_RATE_LIMIT_PER_DAY` (default `6000`): the most requests a client may + make in a day. A count or `off`. The default is four hours at the hour limit. + A scraper making one request every 10 seconds makes 8,640 a day, more than the + limit. +- An Actions runner asks gitea for work all day and can pass the limits. Put the + runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. Do the same for a + network where many people share one address, such as an office. - Counting cannot catch a client that stays under all three limits (see "Request rate limits, per client" in [`SPEC.md`](../SPEC.md)). - `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes exempt from @@ -119,8 +122,8 @@ chooses between the two (see "The image's command" below). ## Byte limits, per client -- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `4G`), `SWWAF_BYTES_LIMIT_PER_HOUR` - (default `20G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `200G`): the most +- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `off`), `SWWAF_BYTES_LIMIT_PER_HOUR` + (default `50G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `100G`): the most bytes counted for a client in each window. A size or `off`. - `SWWAF_BYTES_COUNT` (default `both`): which body bytes the byte limits count: `response`, `request` or `both`. diff --git a/internal/config/config.go b/internal/config/config.go index dbe8dd9..22fb33f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -468,13 +468,13 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""), RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""), DenyNets: env.netblocks("SWWAF_DENY_NETS", ""), - RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "500"), - RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "20000"), - RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "300000"), + RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "120"), + RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "1500"), + RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "6000"), RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), - BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "4G"), - BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"), - BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "200G"), + BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "off"), + BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "50G"), + BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "100G"), BytesCount: env.bytesCount("SWWAF_BYTES_COUNT", "both"), LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"), LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index fa256cb..6ea28fc 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -244,9 +244,9 @@ func TestDefaults(t *testing.T) { wantSettings(t, cfg, config.Config{ ListenAddr: ":8080", Observe: false, - RateLimitPerMinute: 500, - RateLimitPerHour: 20000, - RateLimitPerDay: 300000, + RateLimitPerMinute: 120, + RateLimitPerHour: 1500, + RateLimitPerDay: 6000, BanResponse: 403, LimitBanDuration: time.Hour, LimitBanRepeatWindow: 24 * time.Hour, @@ -266,8 +266,8 @@ func TestDefaults(t *testing.T) { LookupSource: defaultLookupSource, LookupTimeout: 800 * time.Millisecond, }) wantByteLimitSettings(t, cfg, config.Config{ - BytesLimitPerMinute: 4 << 30, BytesLimitPerHour: 20 << 30, - BytesLimitPerDay: 200 << 30, BytesCount: "both", + BytesLimitPerMinute: 0, BytesLimitPerHour: 50 << 30, + BytesLimitPerDay: 100 << 30, BytesCount: "both", }) wantNetblocks(t, cfg.AllowNets) @@ -308,7 +308,7 @@ func TestNoSingleRequestBreaksAByteLimitAtTheDefaults(t *testing.T) { for _, limit := range []int64{ cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.BytesLimitPerDay, } { - if largest > limit { + if limit != 0 && largest > limit { // 0 is off t.Errorf("a request of %d bytes breaks the byte limit of %d", largest, limit) } } @@ -2271,12 +2271,12 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitExemptNets: "", denyNets: "", rateLimitPerMinute: "45", - rateLimitPerHour: "20000", - rateLimitPerDay: "300000", + rateLimitPerHour: "1500", + rateLimitPerDay: "6000", rateLimitExemptPaths: "", - bytesLimitPerMinute: "4G", - bytesLimitPerHour: "20G", - bytesLimitPerDay: "200G", + bytesLimitPerMinute: "off", + bytesLimitPerHour: "50G", + bytesLimitPerDay: "100G", bytesCount: "both", lookupSource: defaultLookupSource, lookupDBPath: "", diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 4fab2ff..5cf085d 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -1225,9 +1225,9 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) { "SWWAF_ALLOW_NETS": "", rateLimitExemptNets: "", "SWWAF_DENY_NETS": "", - "SWWAF_RATE_LIMIT_PER_MINUTE": "500", - "SWWAF_RATE_LIMIT_PER_HOUR": "20000", - rateLimitPerDay: "300000", + "SWWAF_RATE_LIMIT_PER_MINUTE": "120", + "SWWAF_RATE_LIMIT_PER_HOUR": "1500", + rateLimitPerDay: "6000", "SWWAF_RATE_LIMIT_EXEMPT_PATHS": "", "SWWAF_DENIED_COUNTRIES": "", "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",