next into main

Merged by sneak's order of 6 November ("take the current state of smallwebwaf and get it on to main. i am going to deploy it soon in its alpha state").

Model: opus-5-5

README.md

@@ -26,10 +26,10 @@ It protects against:
 
 - Scrapers that fetch every page of a site, all day. Each client gets a limit on
   requests and on bytes per minute, per hour and per day. The defaults leave
-  room for a person reading a site for hours. They catch a scraper that keeps up
+  room for programmatic use of an API, such as an automation fleet, and catch a
-  one request every 2 seconds. A client over a limit is refused and banned: for
+  client firing requests as fast as it can within seconds. A client over a limit
-  a while at first, for three times as long at each repeat, and permanently when
+  is refused and banned: for a while at first, for three times as long at each
-  it keeps at it.
+  repeat, and permanently when it keeps at it.
 - Scanners that probe for secrets, such as `/.env` or `/.git/config`. The
   default rule file bans such a client at its first probe, and its next request
   during the ban makes the ban permanent. Your own rules, or a list of trap
@@ -237,13 +237,13 @@ Every setting the code reads, with its default:
 | `SWWAF_ALLOW_NETS`                         | empty                                                                          | Netblocks whose clients skip every check, such as your monitoring.                                                                                                  |
 | `SWWAF_RATE_LIMIT_EXEMPT_NETS`             | empty                                                                          | Netblocks whose clients the request and byte limits neither count nor refuse.                                                                                       |
 | `SWWAF_DENY_NETS`                          | empty                                                                          | Netblocks whose clients are refused.                                                                                                                                |
-| `SWWAF_RATE_LIMIT_PER_MINUTE`              | `120`                                                                          | The most requests a client may make in a minute, a count or `off`; one more bans it.                                                                                |
+| `SWWAF_RATE_LIMIT_PER_MINUTE`              | `500`                                                                          | The most requests a client may make in a minute, a count or `off`; one more bans it.                                                                                |
-| `SWWAF_RATE_LIMIT_PER_HOUR`                | `1500`                                                                         | The most requests a client may make in an hour, a count or `off`; one more bans it.                                                                                 |
+| `SWWAF_RATE_LIMIT_PER_HOUR`                | `20000`                                                                        | The most requests a client may make in an hour, a count or `off`; one more bans it.                                                                                 |
-| `SWWAF_RATE_LIMIT_PER_DAY`                 | `6000`                                                                         | The most requests a client may make in a day, a count or `off`; one more bans it.                                                                                   |
+| `SWWAF_RATE_LIMIT_PER_DAY`                 | `300000`                                                                       | The most requests a client may make in a day, a count or `off`; one more bans it.                                                                                   |
 | `SWWAF_RATE_LIMIT_EXEMPT_PATHS`            | empty                                                                          | Path prefixes whose requests the request and byte limits neither count nor refuse, such as `/assets/`.                                                              |
-| `SWWAF_BYTES_LIMIT_PER_MINUTE`             | `off`                                                                          | The most bytes a client's requests may carry in a minute, a size or `off`; the request that takes a client past any byte limit bans it.                             |
+| `SWWAF_BYTES_LIMIT_PER_MINUTE`             | `4G`                                                                           | The most bytes a client's requests may carry in a minute, a size or `off`; the request that takes a client past any byte limit bans it.                             |
-| `SWWAF_BYTES_LIMIT_PER_HOUR`               | `50G`                                                                          | The most bytes a client's requests may carry in an hour, a size or `off`.                                                                                           |
+| `SWWAF_BYTES_LIMIT_PER_HOUR`               | `20G`                                                                          | The most bytes a client's requests may carry in an hour, a size or `off`.                                                                                           |
-| `SWWAF_BYTES_LIMIT_PER_DAY`                | `100G`                                                                         | The most bytes a client's requests may carry in a day, a size or `off`.                                                                                             |
+| `SWWAF_BYTES_LIMIT_PER_DAY`                | `200G`                                                                         | The most bytes a client's requests may carry in a day, a size or `off`.                                                                                             |
 | `SWWAF_BYTES_COUNT`                        | `both`                                                                         | Which body bytes the byte limits count: `response`, `request` or `both`.                                                                                            |
 | `SWWAF_ERROR_BURST_THRESHOLD`              | `30`                                                                           | The most refusals by a rule file, a trap path or the Core Rule Set a client may get in a minute, a count or `off`; one more bans it.                                |
 | `SWWAF_BAN_RESPONSE`                       | `403`                                                                          | How a refused client is answered: `403`, `429` or `close`, which drops the connection.                                                                              |
@@ -408,10 +408,10 @@ services:
             SWWAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
             # A netblock that sends nothing wanted.
             SWWAF_DENY_NETS: 198.51.100.128/25
-            # Tighter limits for everyone else: a small static site.
+            # Tighter limits for everyone else: a site with no API.
-            SWWAF_RATE_LIMIT_PER_MINUTE: "60"
+            SWWAF_RATE_LIMIT_PER_MINUTE: "120"
-            SWWAF_RATE_LIMIT_PER_HOUR: "600"
+            SWWAF_RATE_LIMIT_PER_HOUR: "3000"
-            SWWAF_RATE_LIMIT_PER_DAY: "3000"
+            SWWAF_RATE_LIMIT_PER_DAY: "30000"
             # Hosting networks and one country get a share of those limits.
             SWWAF_ASN_LIMIT_PERCENT: AS14061:50,AS16276:25
             SWWAF_COUNTRY_LIMIT_PERCENT: cn:25
@@ -432,9 +432,10 @@ volumes:
 gitea behind traefik, set up for a code forge:
 
 - the upload limits raised, in size and in time;
-- the response size limit raised, for clones over 1 GiB;
+- the response size limit and the byte limits raised, for clones over 1 GiB;
 - the Actions runners exempt from counting;
-- the default request and byte limits kept.
+- the default request limits kept, since they already leave room for API
+  clients.
 
 `traefik` is the network traefik is on, made outside this file.
 
@@ -488,6 +489,11 @@ services:
             SWWAF_INSTANCE_NAME: fsn1/servewaf
             # The Actions runners ask for work all day; never count them.
             SWWAF_RATE_LIMIT_EXEMPT_NETS: 10.0.4.0/24
+            # Raised with rproxy's response limit, to four times the default
+            # each, so that one clone of that size breaks no byte limit.
+            SWWAF_BYTES_LIMIT_PER_MINUTE: 16G
+            SWWAF_BYTES_LIMIT_PER_HOUR: 80G
+            SWWAF_BYTES_LIMIT_PER_DAY: 800G
             # gitea never serves these; a request for one bans the client.
             SWWAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php,/wp-admin/
         volumes:

SPEC.md

@@ -397,31 +397,32 @@ Conventions:
 
 ### Request rate limits, per client
 
-Breaking one bans the client. The defaults let people read a site and stop
+Breaking one bans the client. The defaults suit programmatic use of an API, such
-scrapers that send many requests slowly over hours or days. The hour limit is
+as an automation fleet on another host, and still stop abusive bots. A fleet of
-the main one.
+20 agents, each making a request every 10 seconds at its busiest, makes 120
+requests a minute and 7,200 an hour. Working around the clock, it makes 172,800
+a day. It stays under all three limits.
 
-- Per minute: 120, 2 requests a second. It leaves room for a person opening
+- Per minute: 500, about 8 requests a second. It leaves room for the fleet's 120
-  several pages at once with an empty cache. A gitea page opened with an empty
+  a minute and a few quick first visits with an empty cache. A gitea page opened
-  cache loads 20 to 60 files, and a handful once cached.
+  with an empty cache loads about 40 files. A client firing requests as fast as
-- Per hour: 1,500. A person reading code in gitea for a whole hour, a page every
+  it can passes the limit within seconds.
-  20 seconds, makes about 900 requests. A scraper making one request every 2
+- Per hour: 20,000, about 5.5 requests a second for a whole hour. A client that
-  seconds makes 1,800 an hour, more than the limit.
+  keeps up 6 a second makes 21,600 in an hour, and passes it, at 360 a minute,
-- Per day: 6,000, four hours at the hour limit. A scraper making one request
+  under the minute limit. The fleet's 7,200 an hour is about a third of it.
-  every 10 seconds makes 8,640 a day, more than the limit.
+- Per day: 300,000, about 3.5 requests a second around the clock. A scraper that
+  keeps up 4 a second makes 345,600 a day, and passes it. At 240 a minute and
+  14,400 an hour, it stays under the other two limits.
 
-Counting cannot catch a client that stays under all three limits, about 4
+Counting cannot catch a client that stays under all three limits, such as one
-requests a minute around the clock. Only the other checks can catch an abuser
+that keeps up 3 requests a second. Only the other checks can catch an abuser
 that slow. They include the trap paths, the rule files, the Core Rule Set and
-the blocklists. An abuser that trips none of them is not caught.
+the blocklists. An abuser that trips none of them is not caught. A machine that
-
+keeps up more than 500 requests a minute passes the minute limit. One that keeps
-Some legitimate clients pass the limits:
+up more than about 5.5 a second for an hour passes the hour limit. One that
-
+keeps up more than about 3.5 a second all day passes the day limit, as two busy
-- An Actions runner asks gitea for work all day and can pass the limits.
+Actions runners on one address do. Such a machine's address belongs in
-- Everyone behind one shared address, such as an office, shares one client's
+`SWWAF_RATE_LIMIT_EXEMPT_NETS`.
-  limits.
-
-Their addresses belong in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
 
 `SWWAF_RATE_LIMIT_EXEMPT_PATHS` takes path prefixes out of the count, such as
 static assets or health checks; their bytes are not counted either. A prefix is
@@ -434,9 +435,12 @@ matched the same way.
 
 ### Byte limits, per client
 
-There is no minute limit, so a client can download in a burst at full speed. The
+The hour and day limits sit about twice above what 40 people busy with heavy web
-hour limit is 50 GiB and the day limit 100 GiB. They allow several downloads of
+apps move together on one address. Each moves about 256 MiB an hour, for ten
-a few GiB each. A response's bytes are counted when it ends (see step 15).
+hours a day. A response's bytes are counted when it ends (see step 15). So the
+minute limit holds three of the largest responses, with their requests, ending
+in the same minute. No single download then breaks a limit. An operator who
+raises `SWRP_MAX_RESPONSE_SIZE` must raise the byte limits too.
 
 ### Lookup of AS number and country
 

docs/gitea.md

@@ -14,7 +14,8 @@ version.
   by it.
 - A clone is a response. It fits the defaults of 30 minutes and 1 GiB unless the
   repository is large or the link slow. A repository whose clone is larger than
-  1 GiB needs `SWRP_MAX_RESPONSE_SIZE` raised.
+  1 GiB needs `SWRP_MAX_RESPONSE_SIZE` raised, and the byte limits with it
+  ("Byte limits, per client" in `SPEC.md`).
 - A push is a request, and so is every other upload:
     - a git push;
     - an LFS object;
@@ -31,13 +32,15 @@ version.
   request limits do most of the work there.
 - An Actions runner that reaches gitea through `smallwebwaf` sends requests all
   day. One older than version 0.4 (April 2026) asks for work every 2 seconds,
-  1,800 requests an hour. While it runs a job, it reports the job's log and
+  43,200 requests a day. While it runs a job, it reports the job's log and state
-  state every second instead, two requests a second. So one idle runner makes
+  every second instead, two requests a second. So one runner busy all day sends
-  1,800 requests an hour, more than the hour limit of 1,500. A busy one, at two
+  120 requests a minute and 172,800 a day, under the request limits. Two such
-  requests a second, passes the minute limit of 120 within two minutes. Its
+  runners on one address, each busy all day, make 345,600 and pass the day limit
-  address is then banned, its running jobs fail, and each repeat within a day
+  of 300,000. Three busy at once make 21,600 an hour and pass the hour limit of
-  makes the ban longer. Newer runners ask for work less often. Put the runners'
+  20,000. Five busy at once make 600 a minute and pass the minute limit of 500.
-  addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
+  Their address is then banned, their running jobs fail, and each repeat within
+  a day makes the ban longer. Newer runners ask for work less often. Put the
+  runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
 
 ## What the Core Rule Set lets through
 

docs/settings.md

@@ -99,22 +99,19 @@ chooses between the two (see "The image's command" below).
 
 ## Request rate limits, per client
 
-- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `120`, 2 a second): the most requests a
+- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `500`, about 8 a second): the most
-  client may make in a minute. A count or `off`. The default leaves room for a
+  requests a client may make in a minute. A count or `off`. The default catches,
-  person opening several gitea pages at once with an empty cache, 20 to 60 files
+  within seconds, a client firing requests as fast as it can. It leaves room for
-  each.
+  a fleet of about 20 agents at its busiest, about 120 requests a minute. It
-- `SWWAF_RATE_LIMIT_PER_HOUR` (default `1500`): the most requests a client may
+  also leaves room for a few quick first visits to gitea with an empty cache,
-  make in an hour. A count or `off`. It is the main limit against scrapers. A
+  about 40 files each.
-  person reading code in gitea for an hour, a page every 20 seconds, makes
+- `SWWAF_RATE_LIMIT_PER_HOUR` (default `20000`, about 5.5 a second): the most
-  about 900. A scraper making one request every 2 seconds makes 1,800 an hour,
+  requests a client may make in an hour. A count or `off`. The default leaves
-  more than the limit.
+  room for an automation fleet at work. It catches a client that keeps up more
-- `SWWAF_RATE_LIMIT_PER_DAY` (default `6000`): the most requests a client may
+  than about 5.5 a second for an hour.
-  make in a day. A count or `off`. The default is four hours at the hour limit.
+- `SWWAF_RATE_LIMIT_PER_DAY` (default `300000`, about 3.5 a second): the most
-  A scraper making one request every 10 seconds makes 8,640 a day, more than the
+  requests a client may make in a day. A count or `off`. The default catches a
-  limit.
+  scraper that keeps up 4 a second around the clock.
-- An Actions runner asks gitea for work all day and can pass the limits. Put
-  their addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. Do the same for a network
-  where many people share one address, such as an office.
 - Counting cannot catch a client that stays under all three limits (see "Request
   rate limits, per client" in [`SPEC.md`](../SPEC.md)).
 - `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes exempt from
@@ -122,8 +119,8 @@ chooses between the two (see "The image's command" below).
 
 ## Byte limits, per client
 
-- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `off`), `SWWAF_BYTES_LIMIT_PER_HOUR`
+- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `4G`), `SWWAF_BYTES_LIMIT_PER_HOUR`
-  (default `50G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `100G`): the most
+  (default `20G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `200G`): the most
   bytes counted for a client in each window. A size or `off`.
 - `SWWAF_BYTES_COUNT` (default `both`): which body bytes the byte limits count:
   `response`, `request` or `both`.

internal/config/config.go

@@ -468,13 +468,13 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		AllowNets:            env.netblocks("SWWAF_ALLOW_NETS", ""),
 		RateLimitExemptNets:  env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""),
 		DenyNets:             env.netblocks("SWWAF_DENY_NETS", ""),
-		RateLimitPerMinute:   env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "120"),
+		RateLimitPerMinute:   env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "500"),
-		RateLimitPerHour:     env.count("SWWAF_RATE_LIMIT_PER_HOUR", "1500"),
+		RateLimitPerHour:     env.count("SWWAF_RATE_LIMIT_PER_HOUR", "20000"),
-		RateLimitPerDay:      env.count("SWWAF_RATE_LIMIT_PER_DAY", "6000"),
+		RateLimitPerDay:      env.count("SWWAF_RATE_LIMIT_PER_DAY", "300000"),
 		RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
-		BytesLimitPerMinute:  env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "off"),
+		BytesLimitPerMinute:  env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "4G"),
-		BytesLimitPerHour:    env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "50G"),
+		BytesLimitPerHour:    env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"),
-		BytesLimitPerDay:     env.size("SWWAF_BYTES_LIMIT_PER_DAY", "100G"),
+		BytesLimitPerDay:     env.size("SWWAF_BYTES_LIMIT_PER_DAY", "200G"),
 		BytesCount:           env.bytesCount("SWWAF_BYTES_COUNT", "both"),
 		LookupSource:         env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
 		LookupDBPath:         env.value("SWWAF_LOOKUP_DB_PATH", ""),

internal/config/config_test.go

@@ -244,9 +244,9 @@ func TestDefaults(t *testing.T) {
 	wantSettings(t, cfg, config.Config{
 		ListenAddr:           ":8080",
 		Observe:              false,
-		RateLimitPerMinute:   120,
+		RateLimitPerMinute:   500,
-		RateLimitPerHour:     1500,
+		RateLimitPerHour:     20000,
-		RateLimitPerDay:      6000,
+		RateLimitPerDay:      300000,
 		BanResponse:          403,
 		LimitBanDuration:     time.Hour,
 		LimitBanRepeatWindow: 24 * time.Hour,
@@ -266,8 +266,8 @@ func TestDefaults(t *testing.T) {
 		LookupSource: defaultLookupSource, LookupTimeout: 800 * time.Millisecond,
 	})
 	wantByteLimitSettings(t, cfg, config.Config{
-		BytesLimitPerMinute: 0, BytesLimitPerHour: 50 << 30,
+		BytesLimitPerMinute: 4 << 30, BytesLimitPerHour: 20 << 30,
-		BytesLimitPerDay: 100 << 30, BytesCount: "both",
+		BytesLimitPerDay: 200 << 30, BytesCount: "both",
 	})
 
 	wantNetblocks(t, cfg.AllowNets)
@@ -308,7 +308,7 @@ func TestNoSingleRequestBreaksAByteLimitAtTheDefaults(t *testing.T) {
 	for _, limit := range []int64{
 		cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.BytesLimitPerDay,
 	} {
-		if limit != 0 && largest > limit { // 0 is off
+		if largest > limit {
 			t.Errorf("a request of %d bytes breaks the byte limit of %d", largest, limit)
 		}
 	}
@@ -2271,12 +2271,12 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		rateLimitExemptNets:  "",
 		denyNets:             "",
 		rateLimitPerMinute:   "45",
-		rateLimitPerHour:     "1500",
+		rateLimitPerHour:     "20000",
-		rateLimitPerDay:      "6000",
+		rateLimitPerDay:      "300000",
 		rateLimitExemptPaths: "",
-		bytesLimitPerMinute:  "off",
+		bytesLimitPerMinute:  "4G",
-		bytesLimitPerHour:    "50G",
+		bytesLimitPerHour:    "20G",
-		bytesLimitPerDay:     "100G",
+		bytesLimitPerDay:     "200G",
 		bytesCount:           "both",
 		lookupSource:         defaultLookupSource,
 		lookupDBPath:         "",

internal/smallwebwaf/smallwebwaf_test.go

@@ -1225,9 +1225,9 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) {
 		"SWWAF_ALLOW_NETS":                    "",
 		rateLimitExemptNets:                   "",
 		"SWWAF_DENY_NETS":                     "",
-		"SWWAF_RATE_LIMIT_PER_MINUTE":         "120",
+		"SWWAF_RATE_LIMIT_PER_MINUTE":         "500",
-		"SWWAF_RATE_LIMIT_PER_HOUR":           "1500",
+		"SWWAF_RATE_LIMIT_PER_HOUR":           "20000",
-		rateLimitPerDay:                       "6000",
+		rateLimitPerDay:                       "300000",
 		"SWWAF_RATE_LIMIT_EXEMPT_PATHS":       "",
 		"SWWAF_DENIED_COUNTRIES":              "",
 		"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",