The image creates SWW_STATE_DIR when it is missing (closes #337)

Before waf starts, share/sww.run makes SWW_STATE_DIR if it is missing,
gives it to sww:sww and sets its mode to 0700. A setting that is not an
absolute path, or a SWW_STATE_DIR_FILE that cannot be read, still
changes nothing. script/example-app now checks that a waf container
whose SWW_STATE_DIR does not exist becomes healthy with the directory
made for sww, mode 0700.

Model: opus-5-5

AGENTS.md

@@ -37,3 +37,6 @@ These are the rules for agents working on this repository.
         - `192.168.0.0/16`.
     - `SPEC.md` states that default.
     - Forwarded headers count only when the peer is a trusted proxy.
+- Every time is in UTC, whatever `TZ` or the host's time zone says.
+    - Every day is a date written `YYYY-MM-DD`, never a time.
+    - `SPEC.md` states this near its start.

Dockerfile

@@ -222,8 +222,8 @@ RUN groupadd --system --gid 65532 sww \
         sww
 
 # The state files' directory, SWW_STATE_DIR by default, where a volume
-# is mounted to keep them across deploys. The run script gives it to the
-# sww user at each start of waf.
+# is mounted to keep them across deploys. At each start of waf, the run
+# script makes SWW_STATE_DIR if it is missing and gives it to the sww user.
 RUN mkdir /var/lib/sww
 
 # The default rule file, in SWW_RULES_DIR by default, where a Dockerfile

README.md

@@ -278,7 +278,6 @@ Every setting the code reads, with its default:
 | `SWW_WAF_EXEMPT_PATHS`                   | empty                                                                                    | Path prefixes the Core Rule Set does not inspect.                                                                                                                                                                                                               |
 | `SWW_WAF_BODY_LIMIT`                     | `off`                                                                                    | How much of a request body the Core Rule Set reads, a size up to `64K`; `off` reads none.                                                                                                                                                                       |
 | `SWW_CORE_RULE_SET_EXCLUSIONS_FILE`      | unset                                                                                    | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only.                                                                                                                                                  |
-| `SWW_APPS`                               | unset                                                                                    | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Unset, every app's fixes apply to every hostname.                                             |
 | `SWW_BLOCKLIST_URLS`                     | empty                                                                                    | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`.                                                                                                                                     |
 | `SWW_BLOCKLIST_REFRESH`                  | `24h`                                                                                    | How long after a list was last tried it is fetched again, `1h` or more.                                                                                                                                                                                         |
 | `SWW_BLOCKLIST_ACTION`                   | `deny`                                                                                   | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`.                                                                                                                                                                          |
@@ -294,7 +293,8 @@ Every setting the code reads, with its default:
 | `SWW_REPUTATION_TIMEOUT`                 | `2s`                                                                                     | How long a DNS blocklist query or an AbuseIPDB check may take.                                                                                                                                                                                                  |
 | `SWW_STATE_DIR`                          | `/var/lib/sww`                                                                           | The directory of the state files, an absolute path.                                                                                                                                                                                                             |
 | `SWW_STATE_WRITE_DELAY`                  | `10s`                                                                                    | How long `bans.json` waits to be written after a ban is made, lifted or made permanent.                                                                                                                                                                         |
-| `SWW_STATE_COUNTER_INTERVAL`             | `15m`                                                                                    | How often every state file is written.                                                                                                                                                                                                                          |
+| `SWW_STATE_COUNTER_INTERVAL`             | `15m`                                                                                    | How often every state and cache file is written.                                                                                                                                                                                                                |
+| `SWW_CACHE_DIR`                          | `/var/cache/sww`                                                                         | The directory of the cache files, an absolute path.                                                                                                                                                                                                             |
 | `SWW_LOG_LEVEL`                          | `info`                                                                                   | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written.                                                                                                                                |
 | `SWW_LOG_REQUEST_HEADERS`                | `accept,accept-language,accept-encoding,content-type,origin,range`                       | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded.                                                                                                                                                      |
 | `SWW_LOG_REMOTE_URL`                     | unset                                                                                    | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port.                                                                                                                             |
@@ -510,7 +510,8 @@ request reaches the app.
 
 `waf` takes the checks:
 
-3. The tag lists.
+3. The header limits, then the tag lists. A client in `SWW_ALLOW_NETS` skips the
+   header limits.
 4. The static lists, `SWW_ALLOW_NETS` and `SWW_DENY_NETS`.
 5. The bans.
 6. The lookup of the client's AS number and country.
@@ -522,7 +523,7 @@ request reaches the app.
 11. The rate limits.
 12. The trap paths.
 13. The rule files.
-14. The header limits, and then the Core Rule Set.
+14. The Core Rule Set.
 15. The tag refusals again, so that `exploitsignature` can refuse too.
 16. A path under `/_sww/` is answered `404`, so that the app never sees that
     prefix.
@@ -538,7 +539,7 @@ request reaches the app.
 A refused visitor is answered with `SWW_BAN_RESPONSE`. A `block` rule or a Core
 Rule Set match answers `403` and bans nobody, since the Core Rule Set has false
 positives; such refusals count toward `SWW_ERROR_BURST_THRESHOLD` instead. A
-request over the header limits of step 14 is answered `431`, and bans nobody
+request over the header limits of step 3 is answered `431`, and bans nobody
 either. `rproxy` itself answers `405` to a method it does not pass on, and the
 status for a size or time limit, such as `413` for a body too large.
 
@@ -580,28 +581,27 @@ client's limits by tag. See "Request tags" in [`SPEC.md`](SPEC.md).
   headers of every request, and no body unless `SWW_WAF_BODY_LIMIT` is set. In
   front of a code forge, bodies carry what people write and publish, which the
   rules take for attacks.
-- `smallwebwaf` changes the Core Rule Set in five ways so that it lets ordinary
-  requests to a code forge through, such as git over HTTP and sign-in through
-  Git Credential Manager. No setting undoes the changes; "Attack detection" in
-  [`SPEC.md`](SPEC.md) lists them.
-- Each supported app also has its own fixes, which let its ordinary requests
-  through. Its recorded ordinary requests pass the Core Rule Set at the default
-  settings. The supported apps, by their names in `SWW_APPS`, are:
-    - `gitea`, whose fixes let its searches, file names and branch names
-      through;
-    - `wordpress`, whose fixes let through its searches, its names, its file
-      names and a post saved from the block editor;
-    - `drupal`, whose fixes let through the page to return to, its searches and
-      its names;
-    - `mediawiki`, whose fixes let through its searches, its page titles, its
-      names and its wikitext;
-    - `mastodon`, whose fixes let through its searches, its names, its email
-      addresses, its posts and deliveries from other servers;
-    - `vaultwarden`, whose fixes let through its names, its password hints, its
-      email addresses and its encrypted file names;
-    - `nextcloud`, whose fixes let through its sync client, its WebDAV methods,
-      its searches, its file names, its account names, its comments and its
-      documents.
+- `smallwebwaf` changes the Core Rule Set in fourteen ways so that ordinary
+  requests get through to the apps listed below. No setting undoes the changes;
+  "Attack detection" in [`SPEC.md`](SPEC.md) lists them.
+- These apps work out of the box at the default settings, and their recorded
+  ordinary requests pass the Core Rule Set:
+    - gitea: git over HTTP and sign-in through Git Credential Manager get
+      through. On gitea's own paths, its searches, file names and branch names
+      do too.
+    - WordPress: saving a post from the block editor gets through. On
+      WordPress's own paths, its searches, names and file names do too.
+    - Drupal: on Drupal's own paths, the page to return to, its searches and its
+      names get through.
+    - MediaWiki: on MediaWiki's own paths, its searches, page titles, names and
+      wikitext get through.
+    - Mastodon: on Mastodon's own paths, its searches, names, email addresses,
+      posts and deliveries from other servers get through.
+    - Vaultwarden: on Vaultwarden's own paths, its names, password hints, email
+      addresses and encrypted file names get through.
+    - Nextcloud: its sync client gets through. On Nextcloud's own paths, its
+      WebDAV methods, searches, file names, account names, comments and
+      documents do too.
 - A random password, as a password manager makes one, gets through these forms:
     - signing in, signing up, and changing or resetting a password, on
       WordPress, Drupal, MediaWiki, Mastodon, Nextcloud and gitea;
@@ -609,24 +609,35 @@ client's limits by tag. See "Request tags" in [`SPEC.md`](SPEC.md).
     - deleting the account, on gitea;
     - the admin page, on Vaultwarden, whose apps send a hash of the master
       password, not the password itself.
-- `SWW_APPS` names the apps each hostname serves. A hostname gets only its own
-  apps' fixes. A hostname `SWW_APPS` does not list gets none. Unset, every app's
-  fixes apply to every hostname.
 - `SWW_CORE_RULE_SET_EXCLUSIONS_FILE` switches single rules off for an app's own
   fields or paths.
 - Rule files are plain text, one rule to a line, edited by hand and read again
   while `waf` runs. The image ships one file of probes no real visitor sends
   (see [`docs/rules.md`](docs/rules.md)).
 
-### State files
+### State and cache files
 
-`waf` keeps everything in memory, and the five JSON files in `SWW_STATE_DIR`
-hold a copy: `bans.json`, `clients.json`, `lookups.json`, `reputation.json` and
-`alerts.json`. Each is written whole, to a temporary file renamed over the real
-one, so a crash leaves a complete file. `waf` notices a file edited on disk and
-takes the edit in; an edit that does not parse is set aside as `<name>.bad`, and
-never stops the running process (see
-[`docs/state-files.md`](docs/state-files.md)).
+`waf` keeps everything in memory, and JSON files hold a copy:
+
+- The state files, in `SWW_STATE_DIR`, hold what cannot be fetched again:
+    - `bans.json`;
+    - `clients.json`;
+    - `abuseipdb-checks-spent.json`;
+    - `alerts.json`.
+- The cache files, in `SWW_CACHE_DIR`, hold answers from outside services:
+    - `lookups.json`, from GeoJS;
+    - `lists.json`, the lists fetched by URL;
+    - `dnsbl.json`, from the DNS blocklists;
+    - `abuseipdb-scores.json`, from AbuseIPDB;
+    - `routewatch.json`, from routewatch.
+
+Back up the state directory. The cache directory can be left out of backups: a
+lost cache file starts empty and fills again.
+
+Each file is written whole, to a temporary file renamed over the real one, so a
+crash leaves a complete file. `waf` notices a file edited on disk and takes the
+edit in. An edit that does not parse is set aside as `<name>.bad`, and never
+stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
 
 ### The image and its processes
 
@@ -637,7 +648,7 @@ never stops the running process (see
 - runit, a service supervisor, runs `sww` as the `sww` user (uid and gid 65532)
   and starts it again when it exits. The `run` script of `waf` gives the state
   directory to the `sww` user at each start, so a volume needs no change of
-  owner.
+  owner. It makes the cache directory for that user too.
 - `rproxy` and `waf` use no token between them. `waf`'s own endpoints, under
   `/_sww/`, are for your admin tools and your metrics collector, at its own
   address on the private network; nothing routes the internet there.
@@ -662,8 +673,8 @@ never stops the running process (see
   `internal/routewatch`, `internal/anomaly` and `internal/alerts`: one check or
   source each.
 - `internal/state`, `internal/requestlog`, `internal/remotelog` and
-  `internal/metrics`: the state files, the log lines, the syslog sender and the
-  metrics.
+  `internal/metrics`: the state and cache files, the log lines, the syslog
+  sender and the metrics.
 
 ## Rationale
 
@@ -743,6 +754,7 @@ on the host.
       `http://127.0.0.1:8081`;
     - `waf` listens on port 8090;
     - its state files go in `bin/state`;
+    - its cache files go in `bin/cache`;
     - its rule files are those of `share/rules.d`.
 - `script/example-app`: builds the image and the example app
   (`deploy/example-app`) on it, runs the app with `rproxy` in its container and
@@ -779,8 +791,8 @@ on the host.
 - [`docs/alerts.md`](docs/alerts.md): the alert events and what each destination
   gets.
 - [`docs/metrics.md`](docs/metrics.md): the metrics.
-- [`docs/state-files.md`](docs/state-files.md): the state files and what each
-  holds.
+- [`docs/state-files.md`](docs/state-files.md): the state and cache files and
+  what each holds.
 - [`docs/image.md`](docs/image.md): what the image holds and how it is pinned.
 
 ## License

docs/alerts.md

@@ -23,7 +23,8 @@ back or dropped.
     - the day's AbuseIPDB checks used up.
 - `file_error`: one of:
     - a rule file or state file edit with an error;
-    - a state file that cannot be written;
+    - a cache file that does not parse, at start or after an edit;
+    - a state file or cache file that cannot be written;
     - a replacement lookup database that cannot be read.
 - `summary`: the hourly summary of alerts held back.
 

docs/image.md

@@ -19,7 +19,8 @@ way.
 - The service directory `/etc/service/sww`, with the `run` script
   `share/sww.run`.
 - `/var/lib/sww` for `waf`'s state files, and `/etc/sww/rules.d` with the
-  default rule file `share/rules.d/00-default.rules`.
+  default rule file `share/rules.d/00-default.rules`. The `run` script makes
+  `SWW_STATE_DIR` when it is missing.
 - `EXPOSE 8080`, and the `HEALTHCHECK` that runs `sww healthcheck`.
 
 ## Pins

docs/metrics.md

@@ -58,9 +58,9 @@ By `source`: each list's URL, each zone, and `abuseipdb`:
 - `sww_reputation_last_fetch_timestamp_seconds`, for a list;
 - `sww_reputation_daily_budget_remaining`, for AbuseIPDB.
 
-## State files
+## State and cache files
 
-By `file`:
+By `file`, the name of a state or cache file:
 
 - `sww_state_file_writes_total`, `sww_state_file_write_failures_total`,
   `sww_state_file_last_write_timestamp_seconds` and `sww_state_file_size_bytes`;

docs/settings.md

@@ -50,14 +50,16 @@ the two (see "The image's command" below).
 - `SWW_METRICS_TOKEN` (default unset): the token for `/_sww/metrics`. At least
   32 characters.
 
-## State files
+## State and cache files
 
 - `SWW_STATE_DIR` (default `/var/lib/sww`): the directory of the state files, an
-  absolute path that exists and can be written.
+  absolute path that can be written. The image's `run` script makes it.
 - `SWW_STATE_WRITE_DELAY` (default `10s`): how long `bans.json` waits to be
   written after a ban is made, lifted or made permanent. A duration.
-- `SWW_STATE_COUNTER_INTERVAL` (default `15m`): how often every state file is
-  written. A duration.
+- `SWW_STATE_COUNTER_INTERVAL` (default `15m`): how often every state and cache
+  file is written. A duration.
+- `SWW_CACHE_DIR` (default `/var/cache/sww`): the directory of the cache files,
+  an absolute path that can be written. The image's `run` script makes it.
 
 ## Logging
 
@@ -254,31 +256,6 @@ the request first.
   a range. So does a directive that removes or changes rules as the file loads,
   such as `SecRuleRemoveById`, since none of the Core Rule Set's rules is loaded
   yet. It has no `_FILE` form.
-- `SWW_APPS` (default unset): the apps each hostname serves. A request gets the
-  fixes of its hostname's apps only (see "Attack detection" in
-  [`SPEC.md`](../SPEC.md)).
-    - Entries are separated by semicolons. Each is a hostname, a colon, then one
-      or more apps separated by commas, for example
-      `git.example.com:gitea;blog.example.com:wordpress`.
-    - The supported apps are:
-        - `drupal`;
-        - `gitea`;
-        - `mastodon`;
-        - `mediawiki`;
-        - `nextcloud`;
-        - `vaultwarden`;
-        - `wordpress`.
-    - The request's `Host` is matched by exact name, without its port and
-      without regard to case. There are no wildcards.
-    - A hostname not listed gets no app's fixes. Set empty, `SWW_APPS` lists no
-      hostname.
-    - Unset, every app's fixes apply to every hostname.
-    - Any of these mistakes stops the start:
-        - an unknown app;
-        - a hostname that is not a DNS name, such as `*.example.com`;
-        - a hostname listed twice;
-        - an app listed twice for one hostname;
-        - an entry without a colon.
 - `SWW_ERROR_BURST_THRESHOLD` (default `30`): the most refusals a client may get
   in a minute from a rule file, a trap path or the Core Rule Set, a count or
   `off`. The next one bans the client (see "Attack detection" in `SPEC.md`).

docs/state-files.md

@@ -1,8 +1,23 @@
-# State files
+# State and cache files
 
-The five JSON files in `SWW_STATE_DIR`, and what each holds. "Persistent state"
-in [`SPEC.md`](../SPEC.md) says how they are written, read and edited. Each has
-a top-level `version` of 1.
+The JSON files `waf` keeps, and what each holds. "Persistent state" in
+[`SPEC.md`](../SPEC.md) says how they are written, read and edited. Each has a
+top-level `version` of 1.
+
+- The state files are in `SWW_STATE_DIR`, `/var/lib/sww` by default. They hold
+  what cannot be fetched again, so back them up:
+    - `bans.json`;
+    - `clients.json`;
+    - `abuseipdb-checks-spent.json`;
+    - `alerts.json`.
+- The cache files are in `SWW_CACHE_DIR`, `/var/cache/sww` by default. They hold
+  answers from outside services, which are asked again when a file is lost, so
+  the directory can be left out of backups:
+    - `lookups.json`;
+    - `lists.json`;
+    - `dnsbl.json`;
+    - `abuseipdb-scores.json`;
+    - `routewatch.json`.
 
 ## `bans.json`
 
@@ -67,21 +82,13 @@ since it was first seen:
   `blocked_by_core_rule_set`. A file may also hold `token_refused`, which is
   kept, and which nothing adds to.
 
-## `lookups.json`
+## `abuseipdb-checks-spent.json`
 
-GeoJS answers, one per line:
+The AbuseIPDB checks spent today, so that a restart does not spend the daily
+budget again:
 
-- `client`;
-- `asn`, `as_name` and `country`;
-- `answered`: when GeoJS answered;
-- `used`: when the answer was last used.
-
-## `reputation.json`
-
-- the last good copy of each list fetched from a URL, with when it was last
-  tried;
-- each DNS blocklist verdict still in use;
-- AbuseIPDB's scores, and the checks spent today.
+- `day`: the day, in UTC;
+- `spent`: the checks spent on it.
 
 ## `alerts.json`
 
@@ -91,6 +98,51 @@ GeoJS answers, one per line:
 - the alerts sent and held back in the current hour;
 - the alerts waiting for each destination.
 
+## `lookups.json`
+
+A cache file. GeoJS answers, one per line:
+
+- `client`;
+- `asn`, `as_name` and `country`;
+- `answered`: when GeoJS answered;
+- `used`: when the answer was last used.
+
+## `lists.json`
+
+A cache file. The last good copy of each list fetched from a URL, with when it
+was last tried:
+
+- `url`;
+- `tried`: when it was last fetched or tried;
+- `fetched` and `lines`: when the last good copy was fetched, and its lines,
+  comment lines included.
+
+## `dnsbl.json`
+
+A cache file. Each DNS blocklist verdict still in use, one per line:
+
+- `zone` and `client`;
+- `listed`: whether the zone lists the client;
+- `fetched`: when the zone answered.
+
+## `abuseipdb-scores.json`
+
+A cache file. AbuseIPDB's scores still in use, one per line:
+
+- `client`;
+- `score`: from 0 to 100;
+- `fetched`: when AbuseIPDB answered.
+
+## `routewatch.json`
+
+A cache file. routewatch's answers from the last hour, one per line:
+
+- `client`;
+- `netblock`: the netblock the client's address is announced in, empty for an
+  address without a live route;
+- `asn`: the number of the AS that originates the netblock, 0 for none;
+- `answered`: when routewatch answered.
+
 ## Sizes
 
 - `clients.json`: about 1 KiB per client, about 20 MiB at the default
@@ -102,4 +154,7 @@ GeoJS answers, one per line:
   more.
 - `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
   full.
-- `reputation.json` and `alerts.json`: usually a few MiB or less.
+- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
+- `abuseipdb-checks-spent.json`: a few bytes.
+- `lists.json`, `dnsbl.json`, `abuseipdb-scores.json` and `alerts.json`: usually
+  a few MiB or less.

go.mod

@@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf
 go 1.26.0
 
 require (
+	cloud.google.com/go v0.123.0
 	github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
 	github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
 	github.com/fsnotify/fsnotify v1.10.1

go.sum

@@ -1,3 +1,5 @@
+cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
+cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

internal/alerts/alerts.go

@@ -56,9 +56,11 @@ const (
 	// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
 	// failing or refused, or the day's AbuseIPDB checks used up.
 	EventSourceFailure = "source_failure"
-	// EventFileError is a rule file or state file edited while sww
-	// runs that does not parse, a replacement of the lookup database that
-	// cannot be read, or a state file that cannot be written.
+	// EventFileError is one of:
+	//   - a rule file or state file edited while sww runs that does not parse;
+	//   - a cache file that does not parse, at start or after an edit;
+	//   - a state file or cache file that cannot be written;
+	//   - a replacement of the lookup database that cannot be read.
 	EventFileError = "file_error"
 	// EventSummary is the summary sent as an hour ends: of the alerts held
 	// back in it past SWW_ALERT_MAX_PER_HOUR, and of the repeats held

internal/config/config.go

@@ -29,7 +29,6 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/anomaly"
-	"sneak.berlin/go/smallwebwaf/internal/coreruleset"
 	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 )
 
@@ -192,12 +191,15 @@ type Config struct {
 	BanScopeV4Prefix int
 	// StateDir is the directory of the state files, an absolute path
 	// (SWW_STATE_DIR). bans.json is written StateWriteDelay after a ban
-	// is made (SWW_STATE_WRITE_DELAY), and every state file every
-	// StateCounterInterval (SWW_STATE_COUNTER_INTERVAL). Neither can be
-	// off.
+	// is made (SWW_STATE_WRITE_DELAY), and every state and cache file
+	// every StateCounterInterval (SWW_STATE_COUNTER_INTERVAL). Neither
+	// can be off.
 	StateDir             string
 	StateWriteDelay      time.Duration
 	StateCounterInterval time.Duration
+	// CacheDir is the directory of the cache files, which keep answers
+	// from outside services, an absolute path (SWW_CACHE_DIR).
+	CacheDir string
 	// LogRequestHeaders are the request headers whose values the request
 	// log gives, in lower case (SWW_LOG_REQUEST_HEADERS).
 	LogRequestHeaders []string
@@ -236,11 +238,6 @@ type Config struct {
 	WAFExemptPaths            []string
 	WAFBodyLimit              int64
 	CoreRuleSetExclusionsFile string
-	// Apps are the names of the apps each hostname serves, by hostname in
-	// lowercase (SWW_APPS): a request gets only its hostname's apps' fixes
-	// to the Core Rule Set. Apps is nil while SWW_APPS is unset, and every
-	// request gets every app's fixes.
-	Apps map[string][]string
 	// TrapPaths are the paths a request for which is a clear sign of
 	// attack (SWW_TRAP_PATHS), each starting with / and without a ?.
 	TrapPaths []string
@@ -410,11 +407,6 @@ var (
 	errNotTrapPath = errors.New(
 		"is not a path starting with / and without a ?, such as /wp-login.php",
 	)
-	errNotAppsEntry = errors.New(
-		"is not a hostname, : and a list of apps, such as git.example.com:gitea",
-	)
-	errNotHostname      = errors.New("is not a DNS name such as git.example.com")
-	errNotApp           = errors.New("is not an app sww has fixes for; the apps are")
 	errNotWAFMode       = errors.New("is not off, detect or block")
 	errNotParanoiaLevel = errors.New("is not a paranoia level, from 1 to 4")
 	errNotRuleID        = errors.New(
@@ -571,6 +563,7 @@ func FromEnvironment(
 		StateDir:             env.absolutePath("SWW_STATE_DIR", "/var/lib/sww"),
 		StateWriteDelay:      env.durationNotOff("SWW_STATE_WRITE_DELAY", "10s"),
 		StateCounterInterval: env.durationNotOff("SWW_STATE_COUNTER_INTERVAL", "15m"),
+		CacheDir:             env.absolutePath("SWW_CACHE_DIR", "/var/cache/sww"),
 		LogRequestHeaders: env.headerNames("SWW_LOG_REQUEST_HEADERS",
 			"accept,accept-language,accept-encoding,content-type,origin,range"),
 		LogLevel:            env.logLevel("SWW_LOG_LEVEL", "info"),
@@ -588,7 +581,6 @@ func FromEnvironment(
 		WAFBodyLimit:   env.wafBodyLimit("SWW_WAF_BODY_LIMIT", off),
 		CoreRuleSetExclusionsFile: env.filePath(
 			"SWW_CORE_RULE_SET_EXCLUSIONS_FILE"),
-		Apps:                 env.apps("SWW_APPS"),
 		TrapPaths:            env.trapPaths("SWW_TRAP_PATHS"),
 		ErrorBurstThreshold:  env.count("SWW_ERROR_BURST_THRESHOLD", "30"),
 		LogRemoteURL:         env.logRemoteURL("SWW_LOG_REMOTE_URL"),
@@ -844,25 +836,6 @@ func (e *environment) pathPrefixes(name, defaultValue string) []string {
 	return prefixes
 }
 
-// apps reads the setting that is the apps each hostname serves. It is nil
-// while the setting is unset, and logged as null, unlike a setting set
-// empty.
-func (e *environment) apps(name string) map[string][]string {
-	value, set := e.lookup(name)
-	if !set {
-		e.settings = append(e.settings, slog.Any(name, nil))
-
-		return nil
-	}
-
-	e.settings = append(e.settings, slog.String(name, value))
-
-	apps, err := parseApps(value)
-	e.check(name, err)
-
-	return apps
-}
-
 // trapPaths reads the setting that is the list of trap paths. It is empty
 // by default.
 func (e *environment) trapPaths(name string) []string {
@@ -1880,65 +1853,6 @@ func parseTrapPaths(value string) ([]string, error) {
 	return paths, nil
 }
 
-// parseApps reads the apps each hostname serves: entries separated by
-// semicolons, each a hostname, a colon, and a comma-separated list of the
-// names coreruleset.AppNames returns, such as
-// git.example.com:gitea;cloud.example.com:nextcloud,wordpress. Each
-// hostname is a DNS name, without a wildcard. It returns them by hostname
-// in lowercase, since a hostname is matched without regard to case. An
-// empty value lists no hostname.
-func parseApps(value string) (map[string][]string, error) {
-	apps := map[string][]string{}
-
-	if strings.TrimSpace(value) == "" {
-		return apps, nil
-	}
-
-	for entry := range strings.SplitSeq(value, ";") {
-		hostname, list, found := strings.Cut(entry, ":")
-		if !found {
-			return nil, fmt.Errorf("%q %w", entry, errNotAppsEntry)
-		}
-
-		hostname = strings.ToLower(strings.TrimSpace(hostname))
-		if hostname == "" {
-			return nil, fmt.Errorf("%q %w", entry, errNotAppsEntry)
-		}
-
-		if !isZone(hostname) {
-			return nil, fmt.Errorf("%q %w", hostname, errNotHostname)
-		}
-
-		if _, listed := apps[hostname]; listed {
-			return nil, fmt.Errorf("%q %w", hostname, errListedTwice)
-		}
-
-		names, err := parseList(list)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(names) == 0 {
-			return nil, fmt.Errorf("%q %w", entry, errNotAppsEntry)
-		}
-
-		for i, name := range names {
-			if !slices.Contains(coreruleset.AppNames(), name) {
-				return nil, fmt.Errorf("%q %w %s", name, errNotApp,
-					strings.Join(coreruleset.AppNames(), ", "))
-			}
-
-			if slices.Contains(names[:i], name) {
-				return nil, fmt.Errorf("%q %w", name, errListedTwice)
-			}
-		}
-
-		apps[hostname] = names
-	}
-
-	return apps, nil
-}
-
 // parseRuleIDs reads a comma-separated list of the ids of Core Rule Set
 // rules, each a whole number above zero and outside firstSetupRuleID to
 // lastSetupRuleID.
@@ -2545,8 +2459,7 @@ func MaskZoneKey(zone string) string {
 	return masked + "." + rest
 }
 
-// isZone reports whether each label of zone is as parseZones takes it. It
-// also checks the hostnames of SWW_APPS, which are DNS names too.
+// isZone reports whether each label of zone is as parseZones takes it.
 func isZone(zone string) bool {
 	for label := range strings.SplitSeq(zone, ".") {
 		badChar := strings.ContainsFunc(label, func(char rune) bool {

internal/config/config_test.go

@@ -74,6 +74,7 @@ const (
 	stateDir             = "SWW_STATE_DIR"
 	stateWriteDelay      = "SWW_STATE_WRITE_DELAY"
 	stateCounterInterval = "SWW_STATE_COUNTER_INTERVAL"
+	cacheDir             = "SWW_CACHE_DIR"
 	adminToken           = "SWW_ADMIN_TOKEN"   //nolint:gosec // the setting's name
 	metricsToken         = "SWW_METRICS_TOKEN" //nolint:gosec // the setting's name
 	metricsTopN          = "SWW_METRICS_TOP_N"
@@ -89,7 +90,6 @@ const (
 	wafExemptPaths       = "SWW_WAF_EXEMPT_PATHS"
 	wafBodyLimit         = "SWW_WAF_BODY_LIMIT"
 	exclusionsFile       = "SWW_CORE_RULE_SET_EXCLUSIONS_FILE"
-	apps                 = "SWW_APPS"
 	trapPaths            = "SWW_TRAP_PATHS"
 	errorBurstThreshold  = "SWW_ERROR_BURST_THRESHOLD"
 	logRemoteURL         = "SWW_LOG_REMOTE_URL"
@@ -263,6 +263,7 @@ func TestDefaults(t *testing.T) {
 		StateDir:             "/var/lib/sww",
 		StateWriteDelay:      10 * time.Second,
 		StateCounterInterval: 15 * time.Minute,
+		CacheDir:             "/var/cache/sww",
 		MetricsToken:         "",
 		MetricsTopN:          50,
 		RulesDir:             "/etc/sww/rules.d",
@@ -347,6 +348,7 @@ func TestValuesAsSet(t *testing.T) {
 		stateDir:             "/srv/waf-state",
 		stateWriteDelay:      "500ms",
 		stateCounterInterval: "1h",
+		cacheDir:             "/srv/waf-cache",
 		metricsToken:         token,
 		metricsTopN:          "10",
 		rulesDir:             "/srv/waf-rules",
@@ -369,6 +371,7 @@ func TestValuesAsSet(t *testing.T) {
 		StateDir:             "/srv/waf-state",
 		StateWriteDelay:      500 * time.Millisecond,
 		StateCounterInterval: time.Hour,
+		CacheDir:             "/srv/waf-cache",
 		MetricsToken:         token,
 		MetricsTopN:          10,
 		RulesDir:             "/srv/waf-rules",
@@ -583,101 +586,6 @@ func TestCoreRuleSetSettings(t *testing.T) {
 	}
 }
 
-// gitea is gitea's name in SWW_APPS.
-const gitea = "gitea"
-
-func TestApps(t *testing.T) {
-	t.Parallel()
-
-	for _, tc := range []struct {
-		env  environment
-		want map[string][]string
-	}{
-		{environment{}, nil},
-		{environment{apps: ""}, map[string][]string{}},
-		{
-			environment{apps: "Git.Example.com:gitea; cloud.example.com : gitea "},
-			map[string][]string{
-				"git.example.com": {gitea}, "cloud.example.com": {gitea},
-			},
-		},
-		{
-			environment{apps: "cloud.example.com:gitea,wordpress"},
-			map[string][]string{"cloud.example.com": {gitea, "wordpress"}},
-		},
-	} {
-		cfg := fromEnvironment(t, tc.env)
-		if !reflect.DeepEqual(cfg.Apps, tc.want) {
-			t.Errorf("%v gave %#v, want %#v", tc.env, cfg.Apps, tc.want)
-		}
-	}
-}
-
-// Unset and set empty, SWW_APPS has opposite effects, so it logs differently.
-func TestUnsetAndEmptyAppsAreLoggedDifferently(t *testing.T) {
-	t.Parallel()
-
-	for _, tc := range []struct {
-		env  environment
-		want string
-	}{
-		{environment{}, `"` + apps + `":null`},
-		{environment{apps: ""}, `"` + apps + `":""`},
-	} {
-		var out bytes.Buffer
-
-		slog.New(slog.NewJSONHandler(&out, nil)).
-			Info("starting", "settings", fromEnvironment(t, tc.env))
-
-		if !strings.Contains(out.String(), tc.want) {
-			t.Errorf("%v logged %s, want %s in it", tc.env, out.String(), tc.want)
-		}
-	}
-}
-
-func TestInvalidAppsStopsTheStart(t *testing.T) {
-	t.Parallel()
-
-	const (
-		notEntry = " is not a hostname, : and a list of apps, " +
-			"such as git.example.com:gitea"
-		notHostname = " is not a DNS name such as git.example.com"
-	)
-
-	for _, tc := range []struct{ value, want string }{
-		{"git.example.com:gitea;blog.example.com", `"blog.example.com"` + notEntry},
-		{"git.example.com:gitea;", `""` + notEntry},
-		{":gitea", `":gitea"` + notEntry},
-		{"git.example.com:", `"git.example.com:"` + notEntry},
-		{"*.example.com:gitea", `"*.example.com"` + notHostname},
-		{"git.example.com/:gitea", `"git.example.com/"` + notHostname},
-		{"git example.com:gitea", `"git example.com"` + notHostname},
-		{
-			"git.example.com:gitea,jellyfin",
-			`"jellyfin" is not an app sww has fixes for; ` +
-				`the apps are drupal, gitea, mastodon, mediawiki, nextcloud, ` +
-				`vaultwarden, wordpress`,
-		},
-		{
-			"git.example.com:gitea;GIT.example.com:gitea",
-			`"git.example.com" is listed twice`,
-		},
-		{"git.example.com:gitea,gitea", `"gitea" is listed twice`},
-		{"git.example.com:gitea,", `"gitea," has an empty item in its list`},
-	} {
-		t.Run(tc.value, func(t *testing.T) {
-			t.Parallel()
-
-			_, err := config.FromEnvironment(environment{apps: tc.value}.lookupEnv)
-
-			want := apps + ": " + tc.want
-			if err == nil || err.Error() != want {
-				t.Errorf("error %v, want %s", err, want)
-			}
-		})
-	}
-}
-
 func TestWAFBodyLimitOf64K(t *testing.T) {
 	t.Parallel()
 
@@ -2398,6 +2306,7 @@ func TestInvalidBanOrStateValueStopsTheStart(t *testing.T) {
 		{stateDir, ""}, {stateDir, "state"}, {stateDir, "./var/lib/sww"},
 		{stateWriteDelay, off}, {stateWriteDelay, "0s"},
 		{stateCounterInterval, off}, {stateCounterInterval, "15"},
+		{cacheDir, ""}, {cacheDir, "cache"}, {cacheDir, "./var/cache/sww"},
 	})
 }
 
@@ -2670,6 +2579,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		stateDir:             "/var/lib/sww",
 		stateWriteDelay:      "10s",
 		stateCounterInterval: "15m",
+		cacheDir:             "/var/cache/sww",
 		adminToken:           "",
 		metricsToken:         "",
 		metricsTopN:          "50",
@@ -2685,7 +2595,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		wafExemptPaths:       "",
 		wafBodyLimit:         off,
 		exclusionsFile:       "",
-		apps:                 "", // logged as null, which decodes as ""
 		trapPaths:            "",
 		errorBurstThreshold:  "30",
 		logRemoteURL:         "",
@@ -2771,8 +2680,8 @@ func wantLookupSettings(t *testing.T, got *config.Config, want config.Config) {
 	}
 }
 
-// wantBanSettings checks the settings for bans, the state files, the
-// metrics and the rule files.
+// wantBanSettings checks the settings for bans, the state and cache
+// files, the metrics and the rule files.
 func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
 	t.Helper()
 
@@ -2791,11 +2700,7 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
 			got.RulesDir, got.RulesEnabled, want.RulesDir, want.RulesEnabled)
 	}
 
-	if got.StateDir != want.StateDir ||
-		got.StateWriteDelay != want.StateWriteDelay ||
-		got.StateCounterInterval != want.StateCounterInterval {
-		t.Errorf("state settings\n%+v\nwant\n%+v", got, want)
-	}
+	wantStateSettings(t, got, want)
 
 	if got.MetricsToken != want.MetricsToken || got.MetricsTopN != want.MetricsTopN {
 		t.Errorf("metrics token %q and top %d, want %q and %d",
@@ -2803,6 +2708,18 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) {
 	}
 }
 
+// wantStateSettings checks the settings for the state and cache files.
+func wantStateSettings(t *testing.T, got *config.Config, want config.Config) {
+	t.Helper()
+
+	if got.StateDir != want.StateDir ||
+		got.StateWriteDelay != want.StateWriteDelay ||
+		got.StateCounterInterval != want.StateCounterInterval ||
+		got.CacheDir != want.CacheDir {
+		t.Errorf("state settings\n%+v\nwant\n%+v", got, want)
+	}
+}
+
 // wantNetblocks checks a list of netblocks.
 func wantNetblocks(t *testing.T, got []netip.Prefix, want ...string) {
 	t.Helper()

internal/coreruleset/coreruleset.go

@@ -1,9 +1,9 @@
 // Package coreruleset runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
 // method, the URL with its query and the headers of a request, and on its
-// body while SWW_WAF_BODY_LIMIT is set, with the five changes sww
-// makes to it, the fixes for the apps SWW_APPS names, and the exclusions in
-// SWW_CORE_RULE_SET_EXCLUSIONS_FILE, as "Attack detection" under "Settings"
-// in SPEC.md describes them. It reads no response.
+// body while SWW_WAF_BODY_LIMIT is set, with the fourteen changes sww
+// makes to it and the exclusions in SWW_CORE_RULE_SET_EXCLUSIONS_FILE, as
+// "Attack detection" under "Settings" in SPEC.md describes them. It reads no
+// response.
 //
 // sww writes only to its state directory, so Coraza is built with
 // its no_fs_access tag, as the Dockerfile and script/build build it: of a
@@ -16,8 +16,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"maps"
-	"net"
 	"net/http"
 	"net/netip"
 	"os"
@@ -34,8 +32,11 @@ import (
 
 // directives are the Core Rule Set as sww runs it, with the
 // paranoia level for %d, bodyDirectives for the first %s while
-// SWW_WAF_BODY_LIMIT is set, the exclusions file for the second, and the
-// directives of every app in apps for the third.
+// SWW_WAF_BODY_LIMIT is set, the exclusions file for the second,
+// giteaExclusions for the third, wordpressExclusions for the fourth,
+// drupalExclusions for the fifth, mediawikiExclusions for the sixth,
+// mastodonExclusions for the seventh, vaultwardenExclusions for the eighth,
+// and nextcloudExclusions for the ninth.
 // Each rule sww adds has an id from 900000 to 900999, the ids the
 // Core Rule Set keeps for the rules that set it up, which
 // SWW_WAF_DISABLED_RULES and the exclusions file refuse, so that no
@@ -83,24 +84,67 @@ SecRule ARGUMENTS_LIMIT_REACHED "@eq 1" "id:900300,phase:2,pass,\
 # before the rules of phase 1 too.
 %s
 
-# The apps' fixes come before the rules for the same reason.
+# The eighth change comes before the rules for the same reason.
 %s
 
-# The fifth: only the rules for requests are loaded, and no response is
+# The ninth change comes before the rules for the same reason.
+%s
+
+# So does the tenth.
+%s
+
+# So does the eleventh.
+%s
+
+# So does the twelfth.
+%s
+
+# So does the thirteenth.
+%s
+
+# And the fourteenth.
+%s
+
+# The sixth: only the rules for requests are loaded, and no response is
 # inspected.
 Include @owasp_crs/REQUEST-*.conf
 
 # The third: redirect_uri is not checked for a URL naming an IP address or
-# localhost. Coraza matches a parameter name here without regard to case.
-# ARGS holds the fields of a form data or multipart body Coraza reads as
-# well as the query parameters, so a field of this name is left out too.
+# localhost. Coraza matches a parameter name here, and in the fourth,
+# without regard to case. ARGS holds the fields of a form data or multipart
+# body Coraza reads as well as the query parameters, so a field of one of
+# these names is left out too.
 SecRuleUpdateTargetById 931100 "!ARGS:redirect_uri"
 SecRuleUpdateTargetById 934110 "!ARGS:redirect_uri"
 
-# The fourth: Referer is not checked for a Unix command without arguments
-# or for Java starting a process.
+# The fourth: the query parameters in which gitea sends names within a
+# repository or its own records, or a page of its own site, are not
+# checked against the lists of system files, shell paths and command
+# names. Coraza takes one rule id per directive.
+SecRuleUpdateTargetById 930120 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\
+    !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\
+    !ARGS:artifactName|!ARGS:redirect_to"
+SecRuleUpdateTargetById 932160 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\
+    !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\
+    !ARGS:artifactName|!ARGS:redirect_to"
+SecRuleUpdateTargetById 932260 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\
+    !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\
+    !ARGS:artifactName|!ARGS:redirect_to"
+
+# The fifth, for Referer: it is not checked for a Unix command without
+# arguments, for Java starting a process, or for a Java class name. 944130
+# refused every page opened from the page of java.lang.String.md. The
+# cookies are left out in Inspect.
 SecRuleUpdateTargetById 932340 "!REQUEST_HEADERS:Referer"
 SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer"
+SecRuleUpdateTargetById 944130 "!REQUEST_HEADERS:Referer"
+
+# The seventh: a container image push that reuses a layer names the image
+# it takes the layer from in the query parameter from, which is not
+# checked for command names. 932260 refused the push of a layer from
+# owner/python3-app.
+SecRule REQUEST_FILENAME "@rx ^/v2/.+/blobs/uploads/$" \
+    "id:900500,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932260;ARGS:from"
 `
 
 // bodyDirectives have the Core Rule Set read the part of a request body
@@ -145,61 +189,61 @@ SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:900450,phase:2,pass,\
     severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'"
 `
 
-// app is an app whose ordinary requests the Core Rule Set would refuse,
-// with the fixes that let them pass.
-type app struct {
-	// directives are Coraza exclusions, loaded before the Core Rule Set's
-	// rules. Their first rule skips to the marker they end with, unless
-	// Inspect sets the transaction variable named for the app, such as
-	// app_gitea. The skip reaches only phase 1, so a rule of a later phase
-	// must check that variable itself.
-	directives string
-	// cookies are the cookies the Core Rule Set reads a request without.
-	cookies []string
-}
-
-//go:embed gitea.conf
-var giteaDirectives string
-
-//go:embed wordpress.conf
-var wordpressDirectives string
-
-//go:embed drupal.conf
-var drupalDirectives string
-
-//go:embed mediawiki.conf
-var mediawikiDirectives string
-
-//go:embed mastodon.conf
-var mastodonDirectives string
-
-//go:embed vaultwarden.conf
-var vaultwardenDirectives string
-
-//go:embed nextcloud.conf
-var nextcloudDirectives string
-
-// apps are the apps whose fixes sww has, by the name SWW_APPS gives each.
+// giteaExclusions are the eighth change: the fields of gitea's own
+// requests that skip the rules that refused them there, as Coraza
+// directives.
 //
-//nolint:gochecknoglobals // a constant cannot be a map
-var apps = map[string]app{
-	"drupal": {directives: drupalDirectives},
-	"gitea": {
-		directives: giteaDirectives,
-		cookies:    []string{"gitea_flash", "redirect_to"},
-	},
-	"mastodon":    {directives: mastodonDirectives},
-	"mediawiki":   {directives: mediawikiDirectives},
-	"nextcloud":   {directives: nextcloudDirectives},
-	"vaultwarden": {directives: vaultwardenDirectives},
-	"wordpress":   {directives: wordpressDirectives},
-}
+//go:embed gitea.conf
+var giteaExclusions string
 
-// AppNames returns the names of the apps whose fixes sww has, in
-// alphabetical order.
-func AppNames() []string {
-	return slices.Sorted(maps.Keys(apps))
-}
+// wordpressExclusions are the ninth change: the fields of WordPress's own
+// requests that skip the rules that refused them there, as Coraza
+// directives.
+//
+//go:embed wordpress.conf
+var wordpressExclusions string
+
+// drupalExclusions are the tenth change: the fields of Drupal's own
+// requests that skip the rules that refused them there, as Coraza
+// directives.
+//
+//go:embed drupal.conf
+var drupalExclusions string
+
+// mediawikiExclusions are the eleventh change: the fields of MediaWiki's
+// own requests that skip the rules that refused them there, as Coraza
+// directives.
+//
+//go:embed mediawiki.conf
+var mediawikiExclusions string
+
+// mastodonExclusions are the twelfth change: the fields of Mastodon's own
+// requests that skip the rules that refused them there, as Coraza
+// directives.
+//
+//go:embed mastodon.conf
+var mastodonExclusions string
+
+// vaultwardenExclusions are the thirteenth change: the fields of
+// Vaultwarden's own requests that skip the rules that refused them there, as
+// Coraza directives.
+//
+//go:embed vaultwarden.conf
+var vaultwardenExclusions string
+
+// nextcloudExclusions are the fourteenth change: the fields of Nextcloud's
+// own requests that skip the rules that refused them there, the WebDAV
+// methods it uses, its cookie of the account's name, and its uploads,
+// whose bodies are not read, as Coraza directives.
+//
+//go:embed nextcloud.conf
+var nextcloudExclusions string
+
+// cookiesNotRead are the cookies the Core Rule Set reads a request
+// without, the rest of the fifth change.
+//
+//nolint:gochecknoglobals // a constant cannot be a list
+var cookiesNotRead = []string{"gitea_flash", "redirect_to"}
 
 // Params are what New needs.
 type Params struct {
@@ -215,10 +259,6 @@ type Params struct {
 	// Core Rule Set's rules (SWW_CORE_RULE_SET_EXCLUSIONS_FILE), "" while
 	// there is none.
 	ExclusionsFile string
-	// Apps are the names of the apps each hostname serves, by hostname in
-	// lowercase (SWW_APPS). A request gets the fixes of its hostname's
-	// apps only, or of every app while Apps is nil.
-	Apps map[string][]string
 }
 
 // The ids of sww's own rules, which an exclusions file may not
@@ -310,15 +350,13 @@ func checkExclusions(exclusions string) error {
 type CoreRuleSet struct {
 	engine    coraza.WAF
 	bodyLimit int64
-	apps      map[string][]string
 }
 
-// New returns the Core Rule Set with the five changes and the apps' fixes,
-// at params' paranoia level, without the rules it switches off, reading
-// request bodies up to params' limit, and with params' exclusions file
-// loaded before the Core Rule Set's rules. An exclusions file is an error
-// if it cannot be read or does not parse, or if checkExclusions refuses
-// it.
+// New returns the Core Rule Set with the fourteen changes, at params'
+// paranoia level, without the rules it switches off, reading request
+// bodies up to params' limit, and with params' exclusions file loaded
+// before the Core Rule Set's rules. An exclusions file is an error if it
+// cannot be read or does not parse, or if checkExclusions refuses it.
 func New(params Params) (*CoreRuleSet, error) {
 	body := ""
 	if params.BodyLimit > 0 {
@@ -341,13 +379,10 @@ func New(params Params) (*CoreRuleSet, error) {
 		}
 	}
 
-	var fixes strings.Builder
-	for _, name := range AppNames() {
-		fixes.WriteString(apps[name].directives + "\n")
-	}
-
 	text := fmt.Sprintf(directives, params.ParanoiaLevel, body, exclusions,
-		fixes.String())
+		giteaExclusions, wordpressExclusions, drupalExclusions,
+		mediawikiExclusions, mastodonExclusions, vaultwardenExclusions,
+		nextcloudExclusions)
 
 	if len(params.DisabledRules) > 0 {
 		ids := make([]string, len(params.DisabledRules))
@@ -365,9 +400,7 @@ func New(params Params) (*CoreRuleSet, error) {
 		return nil, fmt.Errorf("load the Core Rule Set: %w", err)
 	}
 
-	return &CoreRuleSet{
-		engine: engine, bodyLimit: params.BodyLimit, apps: params.Apps,
-	}, nil
+	return &CoreRuleSet{engine: engine, bodyLimit: params.BodyLimit}, nil
 }
 
 // Result is what the Core Rule Set found in a request.
@@ -379,12 +412,11 @@ type Result struct {
 	Score int
 }
 
-// Inspect runs the Core Rule Set on r, a request from client, with the
-// fixes of the apps appsFor names for r's hostname: on its method, its URL
-// with the query, and its headers, the Cookie header without those apps'
-// cookies, and on body, r's body as the caller has it, as readBody reads
-// it. It returns what it found, what it read of body, and the error that
-// ended the reading early, if one did.
+// Inspect runs the Core Rule Set on r, a request from client: on its
+// method, its URL with the query, and its headers, the Cookie header
+// without the cookies in cookiesNotRead, and on body, r's body as the
+// caller has it, as readBody reads it. It returns what it found, what it
+// read of body, and the error that ended the reading early, if one did.
 func (c *CoreRuleSet) Inspect(
 	r *http.Request, client netip.Addr, body io.Reader,
 ) (Result, []byte, error) {
@@ -392,24 +424,13 @@ func (c *CoreRuleSet) Inspect(
 	// Closing would remove the files Coraza wrote, and it writes none.
 	defer func() { _ = tx.Close() }()
 
-	// The transaction's variables are set only through Coraza's interface
-	// for plugins.
-	state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is
-
-	var cookiesNotRead []string
-
-	for _, name := range c.appsFor(r.Host) {
-		state.Variables().TX().Set("app_"+name, []string{"1"})
-		cookiesNotRead = append(cookiesNotRead, apps[name].cookies...)
-	}
-
 	tx.ProcessConnection(client.String(), 0, "", 0)
 	tx.ProcessURI(r.URL.String(), r.Method, r.Proto)
 
 	for name, values := range r.Header {
 		for _, value := range values {
 			if name == "Cookie" {
-				value = withoutCookies(value, cookiesNotRead)
+				value = withoutCookiesNotRead(value)
 				if value == "" {
 					continue // it held those cookies alone
 				}
@@ -502,26 +523,9 @@ func score(tx types.Transaction) int {
 	return n
 }
 
-// appsFor returns the names of the apps whose fixes a request for host,
-// its Host header, gets: every app while SWW_APPS is unset, and otherwise
-// those SWW_APPS lists for host's name, without the port and in
-// lowercase.
-func (c *CoreRuleSet) appsFor(host string) []string {
-	if c.apps == nil {
-		return AppNames()
-	}
-
-	hostname, _, err := net.SplitHostPort(host)
-	if err != nil {
-		hostname = host // it has no port
-	}
-
-	return c.apps[strings.ToLower(hostname)]
-}
-
-// withoutCookies returns value, a Cookie header's, without the cookies
-// named in cookiesNotRead.
-func withoutCookies(value string, cookiesNotRead []string) string {
+// withoutCookiesNotRead returns value, a Cookie header's, without the
+// cookies in cookiesNotRead.
+func withoutCookiesNotRead(value string) string {
 	var kept []string
 
 	for cookie := range strings.SplitSeq(value, ";") {

internal/coreruleset/coreruleset_test.go

@@ -38,14 +38,12 @@ var loaded = struct {
 }{loaders: map[string]func() (*coreruleset.CoreRuleSet, error){}}
 
 // loaderFor returns the function that loads the Core Rule Set with params
-// once, making it the first time params are asked for. %#v writes a nil
-// Apps, which gives every app's fixes, apart from an empty one, which
-// gives none.
+// once, making it the first time params are asked for.
 func loaderFor(params coreruleset.Params) func() (*coreruleset.CoreRuleSet, error) {
 	loaded.Lock()
 	defer loaded.Unlock()
 
-	settings := fmt.Sprintf("%#v", params)
+	settings := fmt.Sprintf("%+v", params)
 
 	load, found := loaded.loaders[settings]
 	if !found {
@@ -159,10 +157,9 @@ func inspectBody(
 	}
 
 	for _, header := range r.headers {
-		// Go's server keeps Host and Transfer-Encoding out of the headers,
-		// whatever their case.
+		// Go's server keeps Host and Transfer-Encoding out of the headers.
 		name, value, _ := strings.Cut(header, ": ")
-		switch http.CanonicalHeaderKey(name) {
+		switch name {
 		case "Host":
 			req.Host = value
 		case "Transfer-Encoding":
@@ -207,8 +204,8 @@ func atDefaults(t *testing.T) *coreruleset.CoreRuleSet {
 }
 
 // wantChange checks that crs lets through passes, a request one of the
-// five changes or an app's fixes are for, and still finds result in
-// refused, a request like it that they are not for.
+// fourteen changes is for, and still finds result in refused, a request like
+// it that the change is not for.
 func wantChange(
 	t *testing.T, crs *coreruleset.CoreRuleSet, passes, refused request,
 	result coreruleset.Result,
@@ -469,131 +466,6 @@ func TestImagePushThatReusesALayerIsNotCheckedForCommandNames(t *testing.T) {
 		matched(930120, 932160))
 }
 
-// giteaFixed are gitea requests that only gitea's fixes let through, one
-// for each kind of fix, each with what the Core Rule Set finds in it
-// without them.
-//
-//nolint:gochecknoglobals // a constant cannot be a list
-var giteaFixed = []struct {
-	request request
-	result  coreruleset.Result
-}{
-	// A search of code, on the path of gitea's search.
-	{get("/explore/code?q=SELECT%20*%20FROM%20users%20WHERE"), matched(942100)},
-	// A file name, in a parameter gitea sends names in, on any path.
-	{get("/?path=.gitignore"), matched(930120)},
-	// A file name, in a cookie of gitea's.
-	{
-		get("/owner/repo", "Cookie: gitea_flash=success%3DFile%2Bpackage.json%2Bdeleted"),
-		matched(930120),
-	},
-	// A page opened from the page of a file named for a Java class.
-	{
-		get("/", "Referer: https://git.example/owner/repo/src/branch/main/"+
-			"java.lang.String.md"),
-		matched(944130),
-	},
-	// An image push that reuses a layer, naming the image the layer comes from.
-	{get("/v2/owner/app/blobs/uploads/?from=python3"), matched(932260)},
-}
-
-// The hostnames of the tests of SWW_APPS: the one that serves gitea, and
-// one that serves another app.
-const (
-	giteaHost = "git.example"
-	otherHost = "blog.example"
-)
-
-// gitea is gitea's name in SWW_APPS.
-const gitea = "gitea"
-
-// wordpress is WordPress's name in SWW_APPS.
-const wordpress = "wordpress"
-
-// withHost returns r sent to host.
-func withHost(r request, host string) request {
-	return request{r.method, r.target, append([]string{"Host: " + host}, r.headers...)}
-}
-
-// servingApps returns the Core Rule Set at the defaults, with SWW_APPS
-// giving apps.
-func servingApps(t *testing.T, apps map[string][]string) *coreruleset.CoreRuleSet {
-	t.Helper()
-
-	return shared(t, coreruleset.Params{
-		ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, Apps: apps,
-	})
-}
-
-func TestAppsUnsetGivesEveryHostnameEveryAppsFixes(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, nil)
-
-	for _, fixed := range giteaFixed {
-		for _, host := range []string{giteaHost, otherHost} {
-			wantResult(t, crs, withHost(fixed.request, host), coreruleset.Result{})
-		}
-	}
-}
-
-func TestListedHostnameGetsItsAppsFixes(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{giteaHost: {gitea}})
-
-	// The hostname is matched without the port and without regard to case.
-	for _, host := range []string{giteaHost, "GIT.Example:8443"} {
-		for _, fixed := range giteaFixed {
-			wantResult(t, crs, withHost(fixed.request, host), coreruleset.Result{})
-		}
-	}
-}
-
-func TestUnlistedHostnameGetsTheFullCheck(t *testing.T) {
-	t.Parallel()
-
-	for _, apps := range []map[string][]string{
-		{giteaHost: {gitea}},
-		// SWW_APPS set empty lists no hostname.
-		{},
-	} {
-		crs := servingApps(t, apps)
-
-		for _, host := range []string{otherHost, otherHost + ":8443", "git.example.org"} {
-			for _, fixed := range giteaFixed {
-				wantResult(t, crs, withHost(fixed.request, host), fixed.result)
-			}
-		}
-	}
-}
-
-func TestWordPressFixesApplyOnlyToAHostnameThatServesWordPress(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {wordpress},
-	})
-
-	// A search, which only WordPress's fixes let through.
-	search := get("/?s=python3")
-
-	wantResult(t, crs, withHost(search, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(search, giteaHost), matched(932260))
-}
-
-func TestHostnameListingTwoAppsGetsTheFixesOfBoth(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{giteaHost: {gitea, wordpress}})
-
-	// A file name that only gitea's fixes let through, and a search that only
-	// WordPress's fixes let through.
-	for _, target := range []string{"/?path=.gitignore", "/?s=python3"} {
-		wantResult(t, crs, withHost(get(target), giteaHost), coreruleset.Result{})
-	}
-}
-
 // withExclusions returns the Core Rule Set at the defaults, with the
 // exclusions file at path.
 func withExclusions(path string) (*coreruleset.CoreRuleSet, error) {

internal/coreruleset/drupal.conf

@@ -1,4 +1,4 @@
-# Drupal's fixes: on Drupal's paths, some fields skip the rules that refused
+# The tenth change: on Drupal's paths, some fields skip the rules that refused
 # ordinary Drupal requests there. They are the fields where Drupal sends a
 # search, a name, a file name or an address. Each exclusion runs with the
 # request, before the rules it switches off, so this file is loaded before the
@@ -25,13 +25,6 @@
 #   932230: ls -la
 #   932235: python3
 
-# Unless SWW_APPS lists drupal for the request's hostname, every rule below is
-# skipped. Inspect sets tx.app_drupal when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_drupal itself.
-SecRule &TX:app_drupal "@eq 0" \
-    "id:900790,phase:1,pass,nolog,skipAfter:END_DRUPAL"
-
 # Every admin page's filters and autocompletes, and the page to return to:
 #   title: /admin/content?title=python3
 #   name: /admin/content/media?name=python3
@@ -330,5 +323,3 @@ SecRule REQUEST_FILENAME "@rx ^(?:/[a-z]{2,3}(?:-[a-z]+)?)?/user/password$" \
     SecRule &ARGS_GET:json.mail "@eq 0" \
         "ctl:ruleRemoveTargetById=932120;ARGS:json.mail,\
         ctl:ruleRemoveTargetById=932260;ARGS:json.mail"
-
-SecMarker END_DRUPAL

internal/coreruleset/drupal_test.go

@@ -5,8 +5,6 @@ import (
 	"net/url"
 	"slices"
 	"testing"
-
-	"sneak.berlin/go/smallwebwaf/internal/coreruleset"
 )
 
 // drupalThreshold is SWW_WAF_ANOMALY_THRESHOLD's default.
@@ -39,20 +37,6 @@ const drupalEditAccount = "/user/2/edit"
 // drupalName is the field for a person's name, as the start of a form body.
 const drupalName = "name="
 
-func TestDrupalFixesApplyOnlyToAHostnameThatServesDrupal(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {"drupal"},
-	})
-
-	// A search, which only Drupal's fixes let through.
-	search := get("/search?keys=python3")
-
-	wantResult(t, crs, withHost(search, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(search, giteaHost), matched(932260))
-}
-
 // drupalAttacks are an SQL injection, a path traversal and a script
 // injection, as a query value.
 //

internal/coreruleset/gitea.conf

@@ -1,55 +1,8 @@
-# gitea's fixes: some fields skip the rules that refused ordinary gitea requests.
-# They are the fields where gitea sends a search, a file name or a branch name.
-# Each exclusion runs with the request, before the rules it switches off, those
-# of phase 1 included, so this file is loaded before the Core Rule Set's rules.
-# Each comment names a request the rule refused.
-
-# Unless SWW_APPS lists gitea for the request's hostname, every rule below is
-# skipped. Inspect sets tx.app_gitea when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_gitea itself.
-SecRule &TX:app_gitea "@eq 0" \
-    "id:900590,phase:1,pass,nolog,skipAfter:END_GITEA"
-
-# The query parameters in which gitea sends names within a repository or its
-# own records, or a page of its own site, are not checked against the lists of
-# system files, shell paths and command names, on any path:
-#   930120: .gitignore
-#   932160: bin/docker-entrypoint
-#   932260: docker-build
-# ARGS holds the fields of a form data or multipart body Coraza reads as well
-# as the query parameters, so a field of these names is left out too.
-SecAction "id:900591,phase:1,pass,nolog,\
-    ctl:ruleRemoveTargetById=930120;ARGS:path,\
-    ctl:ruleRemoveTargetById=930120;ARGS:files,\
-    ctl:ruleRemoveTargetById=930120;ARGS:skip-to,\
-    ctl:ruleRemoveTargetById=930120;ARGS:sub_path,\
-    ctl:ruleRemoveTargetById=930120;ARGS:ref,\
-    ctl:ruleRemoveTargetById=930120;ARGS:sha,\
-    ctl:ruleRemoveTargetById=930120;ARGS:branch,\
-    ctl:ruleRemoveTargetById=930120;ARGS:workflow,\
-    ctl:ruleRemoveTargetById=930120;ARGS:artifactName,\
-    ctl:ruleRemoveTargetById=930120;ARGS:redirect_to,\
-    ctl:ruleRemoveTargetById=932160;ARGS:path,\
-    ctl:ruleRemoveTargetById=932160;ARGS:files,\
-    ctl:ruleRemoveTargetById=932160;ARGS:skip-to,\
-    ctl:ruleRemoveTargetById=932160;ARGS:sub_path,\
-    ctl:ruleRemoveTargetById=932160;ARGS:ref,\
-    ctl:ruleRemoveTargetById=932160;ARGS:sha,\
-    ctl:ruleRemoveTargetById=932160;ARGS:branch,\
-    ctl:ruleRemoveTargetById=932160;ARGS:workflow,\
-    ctl:ruleRemoveTargetById=932160;ARGS:artifactName,\
-    ctl:ruleRemoveTargetById=932160;ARGS:redirect_to,\
-    ctl:ruleRemoveTargetById=932260;ARGS:path,\
-    ctl:ruleRemoveTargetById=932260;ARGS:files,\
-    ctl:ruleRemoveTargetById=932260;ARGS:skip-to,\
-    ctl:ruleRemoveTargetById=932260;ARGS:sub_path,\
-    ctl:ruleRemoveTargetById=932260;ARGS:ref,\
-    ctl:ruleRemoveTargetById=932260;ARGS:sha,\
-    ctl:ruleRemoveTargetById=932260;ARGS:branch,\
-    ctl:ruleRemoveTargetById=932260;ARGS:workflow,\
-    ctl:ruleRemoveTargetById=932260;ARGS:artifactName,\
-    ctl:ruleRemoveTargetById=932260;ARGS:redirect_to"
+# The eighth change: on gitea's paths, some fields skip the rules that refused
+# ordinary gitea requests there. They are the fields where gitea sends a search,
+# a file name or a branch name. Each exclusion runs with the request, before the
+# rules it switches off, those of phase 1 included, so this file is loaded before
+# the Core Rule Set's rules. Each comment names a request the rule refused.
 
 # gitea's searches of code, of issues and pull requests, and of commits send the
 # text searched for in q, which skips these rules there:
@@ -210,8 +163,8 @@ SecRule REQUEST_FILENAME \
 #   942140: information_schema.sql
 #   944110: java/lang/Runtime.java
 #   944130: java.lang.String.md
-# Each request skips the rules that refused it. Rule 900591 already has path and
-# redirect_to skip 930120, 932160 and 932260 on every path.
+# Each request skips the rules that refused it. The fourth change already has
+# path and redirect_to skip 930120, 932160 and 932260 on every path.
 
 # A file's commit graph, in file.
 SecRule REQUEST_FILENAME "@rx ^/[^/]+/[^/]+/graph$" \
@@ -383,16 +336,3 @@ SecRule REQUEST_FILENAME "@rx ^/user/(?:login|sign_up|link_account_signin|link_a
         ctl:ruleRemoveTargetById=942540;ARGS:/^(?:password|retype|old_password|json\.password)$/,\
         ctl:ruleRemoveTargetById=942550;ARGS:/^(?:password|retype|old_password|json\.password)$/,\
         ctl:ruleRemoveTargetById=944150;ARGS:/^(?:password|retype|old_password|json\.password)$/"
-
-# Referer skips the rule on Java class names (944130). It refused every page
-# opened from the page of java.lang.String.md.
-SecAction "id:900641,phase:1,pass,nolog,\
-    ctl:ruleRemoveTargetById=944130;REQUEST_HEADERS:Referer"
-
-# A container image push that reuses a layer names the image it takes the layer
-# from in the query parameter from, which skips the list of command names
-# (932260) there. It refused the push of a layer from owner/python3-app.
-SecRule REQUEST_FILENAME "@rx ^/v2/.+/blobs/uploads/$" \
-    "id:900500,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932260;ARGS:from"
-
-SecMarker END_GITEA

internal/coreruleset/mastodon.conf

@@ -1,4 +1,4 @@
-# Mastodon's fixes: on Mastodon's paths, some fields skip the rules that
+# The twelfth change: on Mastodon's paths, some fields skip the rules that
 # refused ordinary Mastodon requests there. They are the fields where Mastodon
 # sends a search, a name, a title, a file name, an address or free text. Each
 # exclusion runs with the request, before the rules it switches off, so this
@@ -17,13 +17,6 @@
 #   932250: ls -la
 #   932260: python3
 
-# Unless SWW_APPS lists mastodon for the request's hostname, every rule below is
-# skipped. Inspect sets tx.app_mastodon when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_mastodon itself.
-SecRule &TX:app_mastodon "@eq 0" \
-    "id:900940,phase:1,pass,nolog,skipAfter:END_MASTODON"
-
 # The API's searches and names, in the query:
 #   q: /api/v2/search?q=python3, /api/v1/accounts/search?q=ls+-la
 #   acct: /api/v1/accounts/lookup?acct=python3
@@ -304,5 +297,3 @@ SecRule REQUEST_FILENAME "@rx ^/auth(?:/|$)" \
         "ctl:ruleRemoveTargetById=930120;ARGS:/^user\[email\]$/,\
         ctl:ruleRemoveTargetById=932120;ARGS:/^user\[email\]$/,\
         ctl:ruleRemoveTargetById=932260;ARGS:/^user\[email\]$/"
-
-SecMarker END_MASTODON

internal/coreruleset/mastodon_test.go

@@ -55,20 +55,6 @@ const (
 	mastodonStatus   = "status="
 )
 
-func TestMastodonFixesApplyOnlyToAHostnameThatServesMastodon(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {"mastodon"},
-	})
-
-	// A search, which only Mastodon's fixes let through.
-	search := get("/api/v2/search?q=python3")
-
-	wantResult(t, crs, withHost(search, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(search, giteaHost), matched(932260))
-}
-
 // mastodonQueryFields sends each field that skips 930120, 932120, 932250
 // and 932260 in the query, on a path where it skips them, ending in "=".
 //

internal/coreruleset/mediawiki.conf

@@ -1,4 +1,4 @@
-# MediaWiki's fixes: on MediaWiki's paths, some fields skip the rules that
+# The eleventh change: on MediaWiki's paths, some fields skip the rules that
 # refused ordinary MediaWiki requests there. They are the fields where MediaWiki
 # sends a search, a title, a name, a file name, an address, a summary or a
 # reason, and the fields that hold wikitext. Each exclusion runs with the
@@ -21,13 +21,6 @@
 #   932250: ls -la
 #   932260: python3
 
-# Unless SWW_APPS lists mediawiki for the request's hostname, every rule below
-# is skipped. Inspect sets tx.app_mediawiki when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_mediawiki itself.
-SecRule &TX:app_mediawiki "@eq 0" \
-    "id:900890,phase:1,pass,nolog,skipAfter:END_MEDIAWIKI"
-
 # The pages and their titles, the searches, and the special pages' fields
 # for a page, a user, a file or an address:
 #   title: /index.php?title=Python3_notes&action=history
@@ -215,12 +208,10 @@ SecRule REQUEST_FILENAME "@rx ^/(?:w/)?api\.php$" \
 # Spezial:. The title follows /index.php/ or /wiki/ in the path, or is the title
 # of /index.php. A title on any other path does not count, so another app's
 # paths under /index.php/ keep every rule. These rules are in phase 2, because
-# the forms send authAction in the body. They check tx.app_mediawiki, since the
-# skip above does not reach phase 2.
+# the forms send authAction in the body.
 SecRule REQUEST_FILENAME "@rx ^/(?:w/)?index\.php$|^/(?:(?:w/)?index\.php|wiki)/[^/:]+:" \
     "id:900919,phase:2,pass,nolog,chain"
     SecRule REQUEST_FILENAME|ARGS_GET:title "@rx ^(?:/(?:(?:w/)?index\.php|wiki)/)?[^/:]+:" "chain"
-    SecRule &TX:app_mediawiki "@eq 1" "chain"
     SecRule &ARGS_GET:/^(?:password|retype)$/ "@eq 0" "chain"
     SecRule ARGS:authAction "@streq change" \
         "ctl:ruleRemoveTargetById=920270;ARGS:/^(?:password|retype)$/,\
@@ -262,7 +253,6 @@ SecRule REQUEST_FILENAME "@rx ^/(?:w/)?index\.php$|^/(?:(?:w/)?index\.php|wiki)/
 SecRule REQUEST_FILENAME "@rx ^/(?:w/)?index\.php$|^/(?:(?:w/)?index\.php|wiki)/[^/:]+:" \
     "id:900920,phase:2,pass,nolog,chain"
     SecRule REQUEST_FILENAME|ARGS_GET:title "@rx ^(?:/(?:(?:w/)?index\.php|wiki)/)?[^/:]+:" "chain"
-    SecRule &TX:app_mediawiki "@eq 1" "chain"
     SecRule &ARGS_GET:retype "@eq 0" "chain"
     SecRule ARGS:authAction "@rx ^(?:create|login-continue)$" \
         "ctl:ruleRemoveTargetById=920270;ARGS:retype,\
@@ -391,12 +381,10 @@ SecRule REQUEST_FILENAME "@rx ^/(?:w/)?index\.php(?:$|/.)|^/wiki/" \
 # This rule is in phase 2, because a phase 1 rule runs before the body is read
 # and cannot see what wpTitles holds. It checks that the body sends wpTitles
 # once, since it would otherwise skip these rules for a second wpTitles
-# holding a command. It checks tx.app_mediawiki, since the skip above does not
-# reach phase 2.
+# holding a command.
 SecRule REQUEST_FILENAME "@rx ^/(?:w/)?index\.php(?:$|/.)|^/wiki/" \
     "id:900916,phase:2,pass,nolog,chain"
     SecRule REQUEST_FILENAME|ARGS_GET_NAMES "@rx ^/(?:w/)?index\.php/.|^/wiki/|^title$" "chain"
-    SecRule &TX:app_mediawiki "@eq 1" "chain"
     SecRule &ARGS_GET:wpTitles "@eq 0" "chain"
     SecRule &ARGS_POST:wpTitles "@eq 1" "chain"
     SecRule ARGS_POST:wpTitles "!@rx [;|&`{]" \
@@ -405,5 +393,3 @@ SecRule REQUEST_FILENAME "@rx ^/(?:w/)?index\.php(?:$|/.)|^/wiki/" \
         ctl:ruleRemoveTargetById=932235;ARGS:wpTitles,\
         ctl:ruleRemoveTargetById=932260;ARGS:wpTitles,\
         ctl:ruleRemoveTargetById=932380;ARGS:wpTitles"
-
-SecMarker END_MEDIAWIKI

internal/coreruleset/mediawiki_test.go

@@ -5,16 +5,11 @@ import (
 	"slices"
 	"strings"
 	"testing"
-
-	"sneak.berlin/go/smallwebwaf/internal/coreruleset"
 )
 
 // mediawikiThreshold is SWW_WAF_ANOMALY_THRESHOLD's default.
 const mediawikiThreshold = 5
 
-// mediawiki is MediaWiki's name in SWW_APPS.
-const mediawiki = "mediawiki"
-
 // mediawikiEdit is the address MediaWiki's edit form posts to.
 const mediawikiEdit = "/index.php?title=Example&action=submit"
 
@@ -54,77 +49,6 @@ const (
 	mediawikiUploadPage = "/index.php/Special:Upload"
 )
 
-func TestMediaWikiFixesApplyOnlyToAHostnameThatServesMediaWiki(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {mediawiki},
-	})
-
-	// A search, which only MediaWiki's fixes let through.
-	search := get("/index.php?title=Special:Search&search=python3")
-
-	wantResult(t, crs, withHost(search, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(search, giteaHost), matched(932260))
-}
-
-func TestMediaWikiWatchlistFixAppliesOnlyToAHostnameThatServesMediaWiki(t *testing.T) {
-	t.Parallel()
-
-	crs := shared(t, coreruleset.Params{
-		ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
-		Apps: map[string][]string{giteaHost: {gitea}, otherHost: {mediawiki}},
-	})
-
-	// A watchlist that only the phase 2 rule 900916 lets through.
-	r := request{http.MethodPost, mediawikiWatchlist.target, []string{formData}}
-	body := mediawikiWatchlist.field + "Lighthouse%20notes%0D%0APython3%20notes"
-
-	got, _ := inspectBody(t, crs, withHost(r, otherHost), body)
-	if got.Score >= mediawikiThreshold {
-		t.Errorf("POST %s %s on %s refused: %+v", r.target, body, otherHost, got)
-	}
-
-	got, _ = inspectBody(t, crs, withHost(r, giteaHost), body)
-	if got.Score < mediawikiThreshold {
-		t.Errorf("POST %s %s on %s let through: %+v", r.target, body, giteaHost, got)
-	}
-}
-
-func TestMediaWikiPasswordFixesApplyOnlyToAHostnameThatServesMediaWiki(t *testing.T) {
-	t.Parallel()
-
-	crs := shared(t, coreruleset.Params{
-		ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
-		Apps: map[string][]string{giteaHost: {gitea}, otherHost: {mediawiki}},
-	})
-
-	// Passwords that only the phase 2 rules 900919 and 900920 let through,
-	// each sent with the authAction of its form.
-	for _, sent := range []struct{ target, body string }{
-		{
-			mediawikiAccountChange,
-			formBody("authAction", "change") + "&" + formBody(passwordKey, "' OR '1'='1"),
-		},
-		{
-			mediawikiSignUp,
-			formBody("authAction", "create") + "&" + formBody(retypeKey, "' OR '1'='1"),
-		},
-	} {
-		r := request{http.MethodPost, sent.target, []string{formData}}
-
-		got, _ := inspectBody(t, crs, withHost(r, otherHost), sent.body)
-		if got.Score >= mediawikiThreshold {
-			t.Errorf("POST %s %s on %s refused: %+v", r.target, sent.body, otherHost, got)
-		}
-
-		got, _ = inspectBody(t, crs, withHost(r, giteaHost), sent.body)
-		if got.Score < mediawikiThreshold {
-			t.Errorf("POST %s %s on %s let through: %+v", r.target, sent.body, giteaHost, got)
-		}
-	}
-}
-
 // mediawikiQueryFields and mediawikiBodyFields together hold every field that
 // skips 930120, 932120, 932250 and 932260, other than wikitext, wpTitles,
 // the passwords and the JSON fields of /rest.php/. mediawikiQueryFields sends

internal/coreruleset/nextcloud.conf

@@ -1,4 +1,4 @@
-# Nextcloud's fixes: on Nextcloud's paths, some fields skip the rules that
+# The fourteenth change: on Nextcloud's paths, some fields skip the rules that
 # refused ordinary Nextcloud requests there. They are the fields where Nextcloud
 # sends a search, a name, a file name, a folder, a comment, a note, a status or
 # a document's text. Each exclusion runs with the request, before the rules it
@@ -29,13 +29,6 @@
 # "; wget http://example.com/x.sh" reaches Nextcloud in these fields. SQL
 # injection, path traversal and script injection are still refused there.
 
-# Unless SWW_APPS lists nextcloud for the request's hostname, every rule below
-# is skipped. Inspect sets tx.app_nextcloud when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_nextcloud itself.
-SecRule &TX:app_nextcloud "@eq 0" \
-    "id:900090,phase:1,pass,nolog,skipAfter:END_NEXTCLOUD"
-
 # The sync client and the Files app use these WebDAV methods besides those
 # the first change allows, on /remote.php/dav/ and /public.php/dav/ and on the
 # older /remote.php/webdav/ and /public.php/webdav/. 911100 refused each:
@@ -162,10 +155,8 @@ SecRule REQUEST_FILENAME "@rx ^/(?:index\.php/)?(?:login/confirm|lostpassword/se
 # change: key=password&value=..., or the same in JSON. This rule is in phase 2,
 # because a phase 1 rule runs before the body is read. It checks that the body
 # sends key once, so a second key cannot skip these rules for another field.
-# It checks tx.app_nextcloud, since the skip above does not reach phase 2.
 SecRule REQUEST_FILENAME "@rx ^/ocs/v[12]\.php/cloud/users/[^/]+$" \
     "id:900127,phase:2,pass,nolog,chain"
-    SecRule &TX:app_nextcloud "@eq 1" "chain"
     SecRule &ARGS_GET:/^(?:json\.)?(?:key|value)$/ "@eq 0" "chain"
     SecRule &ARGS_POST:/^(?:json\.)?key$/ "@eq 1" "chain"
     SecRule ARGS_POST:/^(?:json\.)?key$/ "@streq password" \
@@ -450,5 +441,3 @@ SecRule REQUEST_FILENAME "@rx ^/remote\.php/dav/addressbooks/" \
     SecRule REQUEST_METHOD "@streq MKCOL" \
         "ctl:ruleRemoveTargetById=932125;XML:/*,\
         ctl:ruleRemoveTargetById=932230;XML:/*"
-
-SecMarker END_NEXTCLOUD

internal/coreruleset/nextcloud_test.go

@@ -51,66 +51,6 @@ const (
 // nextcloudOther is a JSON field no exclusion names.
 const nextcloudOther = `{"other":"python3"}`
 
-// nextcloud is Nextcloud's name in SWW_APPS.
-const nextcloud = "nextcloud"
-
-func TestNextcloudFixesApplyOnlyToAHostnameThatServesNextcloud(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {nextcloud},
-	})
-
-	// A search, which only Nextcloud's fixes let through.
-	search := get(nextcloudUsers + "?search=python3")
-
-	wantResult(t, crs, withHost(search, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(search, giteaHost), matched(932260))
-}
-
-func TestNextcloudWebDAVMethodsPassOnlyOnHostnamesThatServeNextcloud(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {nextcloud},
-	})
-
-	// Listing a folder, as the sync client does.
-	list := request{nextcloudPropFind, nextcloudFiles, nil}
-
-	wantResult(t, crs, withHost(list, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(list, giteaHost), matched(911100))
-
-	// Making a calendar, as a CalDAV client does.
-	calendar := request{nextcloudMkcalendar, nextcloudCalendars + "work/", nil}
-
-	wantResult(t, crs, withHost(calendar, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(calendar, giteaHost), matched(911100))
-}
-
-func TestNextcloudPasswordInValuePassesOnlyOnHostnamesThatServeNextcloud(t *testing.T) {
-	t.Parallel()
-
-	crs := shared(t, coreruleset.Params{
-		ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
-		Apps: map[string][]string{giteaHost: {gitea}, otherHost: {nextcloud}},
-	})
-
-	// A password that only the phase 2 rule 900127 lets through.
-	r := request{http.MethodPut, nextcloudUsers + "/alice", []string{formData}}
-	body := "key=password&value=" + url.QueryEscape("' OR '1'='1")
-
-	got, _ := inspectBody(t, crs, withHost(r, otherHost), body)
-	if got.Score >= nextcloudThreshold {
-		t.Errorf("PUT %s %s on %s refused: %+v", r.target, body, otherHost, got)
-	}
-
-	got, _ = inspectBody(t, crs, withHost(r, giteaHost), body)
-	if got.Score < nextcloudThreshold {
-		t.Errorf("PUT %s %s on %s let through: %+v", r.target, body, giteaHost, got)
-	}
-}
-
 // nextcloudQueryFields are the query fields that skip 930120, 932120, 932250
 // and 932260, each on a path where it skips them, ending in "=".
 //

internal/coreruleset/recorded_test.go

@@ -13,8 +13,6 @@ import (
 	"slices"
 	"strings"
 	"testing"
-
-	"sneak.berlin/go/smallwebwaf/internal/coreruleset"
 )
 
 // giteaRequests returns the recorded ordinary gitea requests in testdata.
@@ -54,10 +52,10 @@ func TestOrdinaryGiteaRequestsPass(t *testing.T) {
 
 	const threshold = 5 // SWW_WAF_ANOMALY_THRESHOLD's default
 
-	crs := servingApps(t, map[string][]string{giteaHost: {gitea}})
+	crs := atDefaults(t)
 
 	for _, r := range giteaRequests(t) {
-		if got := inspect(t, crs, withHost(r, giteaHost)); got.Score >= threshold {
+		if got := inspect(t, crs, r); got.Score >= threshold {
 			t.Errorf("%s %q refused: %+v", r.target, r.headers[1:], got)
 		}
 	}
@@ -104,25 +102,17 @@ func recordedRequests(t *testing.T, name string) []recordedRequest {
 }
 
 // wantRecordedRequestsPass checks that none of the recorded ordinary
-// requests in the file name in testdata is refused on the hostname where
-// they were recorded, otherHost for WordPress and example.com for the other
-// apps, while SWW_APPS lists only app for it. They are checked at the
-// defaults and while bodies are read. With SWW_APPS unset, every app's fixes
-// apply. These only switch checks off, so the requests pass then too. A
-// request with neither a body nor a Content-Type header is checked only at
-// the defaults: with neither, the set that reads bodies has nothing more to
-// check. Each request runs as its own parallel subtest.
-func wantRecordedRequestsPass(t *testing.T, name, app string) {
+// requests in the file name in testdata is refused, at the defaults and
+// while bodies are read. A request with neither a body nor a Content-Type
+// header is checked only at the defaults: with neither, the set that reads
+// bodies has nothing more to check. Each request runs as its own parallel
+// subtest.
+func wantRecordedRequestsPass(t *testing.T, name string) {
 	t.Helper()
 
 	const threshold = 5 // SWW_WAF_ANOMALY_THRESHOLD's default
 
-	apps := map[string][]string{"example.com": {app}, otherHost: {app}}
-	defaults := servingApps(t, apps)
-	bodies := shared(t, coreruleset.Params{
-		ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
-		Apps: apps,
-	})
+	defaults, bodies := atDefaults(t), readingBodies(t)
 
 	for _, recorded := range recordedRequests(t, name) {
 		t.Run(recorded.Method+" "+recorded.Target, func(t *testing.T) {
@@ -156,35 +146,35 @@ func wantRecordedRequestsPass(t *testing.T, name, app string) {
 func TestOrdinaryDrupalRequestsPass(t *testing.T) {
 	t.Parallel()
 
-	wantRecordedRequestsPass(t, "drupal-requests.txt", "drupal")
+	wantRecordedRequestsPass(t, "drupal-requests.txt")
 }
 
 func TestOrdinaryMastodonRequestsPass(t *testing.T) {
 	t.Parallel()
 
-	wantRecordedRequestsPass(t, "mastodon-requests.txt", "mastodon")
+	wantRecordedRequestsPass(t, "mastodon-requests.txt")
 }
 
 func TestOrdinaryMediaWikiRequestsPass(t *testing.T) {
 	t.Parallel()
 
-	wantRecordedRequestsPass(t, "mediawiki-requests.txt", mediawiki)
+	wantRecordedRequestsPass(t, "mediawiki-requests.txt")
 }
 
 func TestOrdinaryNextcloudRequestsPass(t *testing.T) {
 	t.Parallel()
 
-	wantRecordedRequestsPass(t, "nextcloud-requests.txt", nextcloud)
+	wantRecordedRequestsPass(t, "nextcloud-requests.txt")
 }
 
 func TestOrdinaryVaultwardenRequestsPass(t *testing.T) {
 	t.Parallel()
 
-	wantRecordedRequestsPass(t, "vaultwarden-requests.txt", "vaultwarden")
+	wantRecordedRequestsPass(t, "vaultwarden-requests.txt")
 }
 
 func TestOrdinaryWordPressRequestsPass(t *testing.T) {
 	t.Parallel()
 
-	wantRecordedRequestsPass(t, "wordpress-requests.txt", wordpress)
+	wantRecordedRequestsPass(t, "wordpress-requests.txt")
 }

internal/coreruleset/vaultwarden.conf

@@ -1,4 +1,4 @@
-# Vaultwarden's fixes: on Vaultwarden's paths, some fields skip the rules that
+# The thirteenth change: on Vaultwarden's paths, some fields skip the rules that
 # refused ordinary Vaultwarden requests there. They are the fields where
 # Vaultwarden receives a name, a password hint or an email address in plain
 # text, and the names of uploaded files. Each exclusion runs with the request,
@@ -17,13 +17,6 @@
 #   932250: ls -la
 #   932260: python3
 
-# Unless SWW_APPS lists vaultwarden for the request's hostname, every rule below
-# is skipped. Inspect sets tx.app_vaultwarden when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_vaultwarden itself.
-SecRule &TX:app_vaultwarden "@eq 0" \
-    "id:900140,phase:1,pass,nolog,skipAfter:END_VAULTWARDEN"
-
 # Names and password hints, in JSON bodies, skip the four rules:
 #   name: signing up, editing the account's profile, and creating or renaming
 #   an organization or a group, "python3 .htaccess team"
@@ -131,5 +124,3 @@ SecRule REQUEST_FILENAME "@rx ^/admin/?$" \
         ctl:ruleRemoveTargetById=942540;ARGS:token,\
         ctl:ruleRemoveTargetById=942550;ARGS:token,\
         ctl:ruleRemoveTargetById=944150;ARGS:token"
-
-SecMarker END_VAULTWARDEN

internal/coreruleset/vaultwarden_test.go

@@ -9,8 +9,6 @@ import (
 	"slices"
 	"strings"
 	"testing"
-
-	"sneak.berlin/go/smallwebwaf/internal/coreruleset"
 )
 
 // vaultwardenThreshold is SWW_WAF_ANOMALY_THRESHOLD's default.
@@ -31,21 +29,6 @@ const (
 // vaultwardenEmail is a JSON body that sends an address in email.
 const vaultwardenEmail = `{"email":"VALUE"}`
 
-func TestVaultwardenFixesApplyOnlyToAHostnameThatServesVaultwarden(t *testing.T) {
-	t.Parallel()
-
-	crs := servingApps(t, map[string][]string{
-		giteaHost: {gitea}, otherHost: {"vaultwarden"},
-	})
-
-	// An address for the data breach report, which only Vaultwarden's fixes
-	// let through.
-	report := get(vaultwardenBreachReport + "python3%40example.com")
-
-	wantResult(t, crs, withHost(report, otherHost), coreruleset.Result{})
-	wantResult(t, crs, withHost(report, giteaHost), matched(932260))
-}
-
 // vaultwardenField is a field Vaultwarden receives in plain text: the
 // request that sends it, and its body, with VALUE where the field's value
 // goes.

internal/coreruleset/wordpress.conf

@@ -1,17 +1,10 @@
-# WordPress's fixes: on WordPress's paths, some fields skip the rules that refused
+# The ninth change: on WordPress's paths, some fields skip the rules that refused
 # ordinary WordPress requests there. Each exclusion runs with the request, before
 # the rules it switches off, so this file is loaded before the Core Rule Set's
 # rules. Each comment names a request the rule refused. An exclusion on a field
 # of a body applies only when the query has no parameter of that name, so it
 # matters only while SWW_WAF_BODY_LIMIT is set.
 
-# Unless SWW_APPS lists wordpress for the request's hostname, every rule below is
-# skipped. Inspect sets tx.app_wordpress when it does.
-# The skip reaches only phase 1, so a rule of a later phase must check
-# tx.app_wordpress itself.
-SecRule &TX:app_wordpress "@eq 0" \
-    "id:900690,phase:1,pass,nolog,skipAfter:END_WORDPRESS"
-
 # WordPress takes a call to its REST API at any of four addresses:
 #   /wp-json/wp/v2/posts/1
 #   /index.php/wp-json/wp/v2/posts/1
@@ -322,5 +315,3 @@ SecRule REQUEST_FILENAME "@rx ^/(?:(?:index\.php/)?page/[0-9]+/?|index\.php)?$"
 # so skips 932340 there.
 SecRule REQUEST_FILENAME "@streq /wp-admin/upload.php" \
     "id:900711,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932340;ARGS:_wp_http_referer"
-
-SecMarker END_WORDPRESS

internal/lookup/file.go

@@ -26,7 +26,7 @@ type FileParams struct {
 	// Path is the lookup database, the IPinfo Lite file in its .mmdb form
 	// (SWW_LOOKUP_DB_PATH).
 	Path string
-	// Now tells the time, normally time.Now.
+	// Now tells the time, normally time.Now in UTC.
 	Now func() time.Time
 	// ProcessLog receives each reading of the file, and why a replacement
 	// of it cannot be read.

internal/lookup/lookup.go

@@ -74,7 +74,7 @@ type Params struct {
 	// Answered, unless nil, is given each answer GeoJS gives, once it is
 	// kept.
 	Answered func(Answer)
-	// Now tells the time, normally time.Now.
+	// Now tells the time, normally time.Now in UTC.
 	Now func() time.Time
 	// ProcessLog receives GeoJS's failures.
 	ProcessLog *slog.Logger

internal/metrics/metrics.go

@@ -120,17 +120,19 @@ func New(topN int, instanceName string) *Metrics {
 				"went on without it, because GeoJS had not given it in time.",
 		}),
 		stateFileWrites: counterVec("sww_state_file_writes_total",
-			"Writes of each state file.", byFile),
+			"Writes of each state or cache file.", byFile),
 		stateFileWriteFailures: counterVec("sww_state_file_write_failures_total",
-			"Writes of each state file that failed.", byFile),
+			"Writes of each state or cache file that failed.", byFile),
 		stateFileLastWrite: gaugeVec("sww_state_file_last_write_timestamp_seconds",
-			"When each state file was last written, in seconds since 1970.", byFile),
+			"When each state or cache file was last written, in seconds since 1970.",
+			byFile),
 		stateFileSize: gaugeVec("sww_state_file_size_bytes",
-			"The size of each state file, as it was last written.", byFile),
+			"The size of each state or cache file, as it was last written.", byFile),
 		stateFileEditsTakenIn: counterVec("sww_state_file_edits_taken_in_total",
-			"Edits of each state file taken in while running.", byFile),
+			"Edits of each state or cache file taken in while running.", byFile),
 		stateFileEditsSetAside: counterVec("sww_state_file_edits_set_aside_total",
-			"Edits of each state file renamed to <name>.bad because they did not parse.",
+			"Edits of each state or cache file renamed to <name>.bad because they "+
+				"did not parse.",
 			byFile),
 	}
 
@@ -447,8 +449,8 @@ func (m *Metrics) CoreRuleSetMatched(mode string, id int) {
 	m.coreRuleSetMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc()
 }
 
-// StateFileWritten counts a write of the state file name, of size bytes,
-// that ended with err.
+// StateFileWritten counts a write of the state or cache file name, of size
+// bytes, that ended with err.
 func (m *Metrics) StateFileWritten(name string, size int, err error) {
 	m.stateFileWrites.WithLabelValues(name).Inc()
 
@@ -466,13 +468,13 @@ func (m *Metrics) StateFileWritten(name string, size int, err error) {
 	m.stateFileSize.WithLabelValues(name).Set(float64(size))
 }
 
-// StateFileEditTakenIn counts an admin's edit of the state file name
+// StateFileEditTakenIn counts an admin's edit of the state or cache file name
 // taken in while sww runs.
 func (m *Metrics) StateFileEditTakenIn(name string) {
 	m.stateFileEditsTakenIn.WithLabelValues(name).Inc()
 }
 
-// StateFileEditSetAside counts an admin's edit of the state file name
+// StateFileEditSetAside counts an admin's edit of the state or cache file name
 // renamed to name.bad because it did not parse.
 func (m *Metrics) StateFileEditSetAside(name string) {
 	m.stateFileEditsSetAside.WithLabelValues(name).Inc()

internal/reputation/abuseipdb.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 )
@@ -27,8 +28,6 @@ const (
 	AbuseIPDBSource = "abuseipdb"
 	// maxAnswerBytes is the most of an answer of AbuseIPDB that is read.
 	maxAnswerBytes = 64 << 10
-	// day is the length of the day the checks are counted in, in UTC.
-	day = 24 * time.Hour
 )
 
 var (
@@ -37,22 +36,21 @@ var (
 		"checks spent; none is made until the day ends at 00:00 UTC")
 )
 
-// Score is what AbuseIPDB said about a client, as reputation.json holds
-// it: the client, an IPv4 address or an IPv6 group, its abuse confidence
-// score, from 0 to 100, and when AbuseIPDB answered.
+// Score is what AbuseIPDB said about a client, as abuseipdb-scores.json
+// holds it: the client, an IPv4 address or an IPv6 group, its abuse
+// confidence score, from 0 to 100, and when AbuseIPDB answered.
 type Score struct {
 	Client  netip.Prefix `json:"client"`
 	Score   int64        `json:"score"`
 	Fetched time.Time    `json:"fetched"`
 }
 
-// Checks are what reputation.json keeps of the checks of clients with
-// AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the
-// first, and the scores still in use.
-type Checks struct {
-	Day    time.Time `json:"day,omitzero"`
-	Spent  int       `json:"spent"`
-	Scores []Score   `json:"scores"`
+// ChecksSpent is what abuseipdb-checks-spent.json keeps of the checks of
+// clients with AbuseIPDB: the day, in UTC, of the checks Spent counts,
+// zero before the first.
+type ChecksSpent struct {
+	Day   civil.Date
+	Spent int
 }
 
 // AbuseIPDBParams are what NewAbuseIPDB needs.
@@ -93,7 +91,7 @@ type AbuseIPDB struct {
 	// checking are the clients whose check is under way.
 	checking map[netip.Prefix]bool
 	// day is the day, in UTC, of the checks spent counts.
-	day   time.Time
+	day   civil.Date
 	spent int
 	// checks and failures count the checks made and those that failed,
 	// and retryAt is when a client may be checked again after the last
@@ -143,7 +141,7 @@ func (a *AbuseIPDB) Hit(
 		return kept.Score, kept.Score >= a.params.MinScore
 	}
 
-	if today := now.Truncate(day); !a.day.Equal(today) {
+	if today := civil.DateOf(now.UTC()); a.day != today {
 		a.day, a.spent = today, 0
 	}
 
@@ -190,42 +188,42 @@ func (a *AbuseIPDB) BudgetLeft() int {
 	a.mu.Lock()
 	defer a.mu.Unlock()
 
-	if !a.day.Equal(a.params.Now().Truncate(day)) {
+	if a.day != civil.DateOf(a.params.Now().UTC()) {
 		return a.params.DailyBudget
 	}
 
 	return max(a.params.DailyBudget-a.spent, 0)
 }
 
-// Snapshot returns the checks spent and every score still in use, sorted
-// by client, as reputation.json keeps them.
-func (a *AbuseIPDB) Snapshot() Checks {
+// Snapshot returns every score still in use, sorted by client, as
+// abuseipdb-scores.json keeps them.
+func (a *AbuseIPDB) Snapshot() []Score {
 	a.mu.Lock()
 
 	now := a.params.Now()
-	checks := Checks{Day: a.day, Spent: a.spent, Scores: make([]Score, 0, a.scores.Len())}
+	scores := make([]Score, 0, a.scores.Len())
 
 	for _, kept := range a.scores.Values() {
 		if now.Sub(kept.Fetched) < a.params.CacheTTL {
-			checks.Scores = append(checks.Scores, kept)
+			scores = append(scores, kept)
 		}
 	}
 
 	a.mu.Unlock()
 
-	slices.SortFunc(checks.Scores, func(x, y Score) int {
+	slices.SortFunc(scores, func(x, y Score) int {
 		return x.Client.Compare(y.Client)
 	})
 
-	return checks
+	return scores
 }
 
-// Load keeps checks, read from reputation.json, in place of those it
-// keeps, but for the scores past maxVerdicts, those fetched longest ago.
-// One fetched CacheTTL ago or more is neither used nor written, as for any
-// score.
-func (a *AbuseIPDB) Load(checks Checks) {
-	scores := slices.Clone(checks.Scores)
+// Load keeps scores, read from abuseipdb-scores.json, in place of those
+// it keeps, but for the scores past maxVerdicts, those fetched longest
+// ago. One fetched CacheTTL ago or more is neither used nor written, as
+// for any score.
+func (a *AbuseIPDB) Load(scores []Score) {
+	scores = slices.Clone(scores)
 	slices.SortStableFunc(scores, func(x, y Score) int {
 		return x.Fetched.Compare(y.Fetched)
 	})
@@ -233,7 +231,6 @@ func (a *AbuseIPDB) Load(checks Checks) {
 	a.mu.Lock()
 	defer a.mu.Unlock()
 
-	a.day, a.spent = checks.Day, checks.Spent
 	a.scores.Purge()
 
 	for _, kept := range scores {
@@ -241,6 +238,24 @@ func (a *AbuseIPDB) Load(checks Checks) {
 	}
 }
 
+// ChecksSpent returns the checks spent, as abuseipdb-checks-spent.json
+// keeps them.
+func (a *AbuseIPDB) ChecksSpent() ChecksSpent {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	return ChecksSpent{Day: a.day, Spent: a.spent}
+}
+
+// LoadChecksSpent keeps checks, read from abuseipdb-checks-spent.json, in
+// place of the checks spent.
+func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	a.day, a.spent = checks.Day, checks.Spent
+}
+
 // check checks client with AbuseIPDB by addr, one of its addresses, keeps
 // the score as client's, and notes the check as no longer under way. A
 // check that fails gives no score: it is counted, logged and raised as a

internal/reputation/abuseipdb_test.go

@@ -17,6 +17,7 @@ import (
 	"testing/synctest"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -179,11 +180,13 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) {
 			t.Errorf("logged\n%s\nwant the budget used up", log.String())
 		}
 
-		// Restarted with what reputation.json keeps, it uses the scores, and
-		// checks no client until the day ends.
+		// Restarted with what abuseipdb-scores.json and
+		// abuseipdb-checks-spent.json keep, it uses the scores, and checks
+		// no client until the day ends.
 		restarted := &abuseIPDBStandIn{}
 		again := newAbuseIPDB(restarted, p)
 		again.Load(checker.Snapshot())
+		again.LoadChecksSpent(checker.ChecksSpent())
 
 		wantScore(t, again, suspect, true, 100, true)
 		wantBudgetLeft(t, again, 0)
@@ -202,6 +205,39 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) {
 	})
 }
 
+func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		// A clock 14 hours ahead of UTC, whose day starts at 10:00 UTC.
+		ahead := time.FixedZone("UTC+14", 14*60*60)
+		p := abuseIPDBParams()
+		p.DailyBudget = 1
+		p.Now = func() time.Time { return time.Now().In(ahead) }
+		abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}}
+		checker := newAbuseIPDB(abuseIPDB, p)
+
+		// At 09:00 UTC the one check is spent.
+		time.Sleep(9 * time.Hour)
+		hitFrom(t, checker, suspect, true)
+		synctest.Wait()
+		wantBudgetLeft(t, checker, 0)
+
+		// At 11:00 UTC the clock's day has changed, but the UTC day has not.
+		time.Sleep(2 * time.Hour)
+		wantBudgetLeft(t, checker, 0)
+
+		want := civil.Date{Year: 2000, Month: 1, Day: 1}
+		if day := checker.ChecksSpent().Day; day != want {
+			t.Errorf("checks spent on %s, want on %s", day, want)
+		}
+
+		// At 00:00 UTC the budget is whole again.
+		time.Sleep(13 * time.Hour)
+		wantBudgetLeft(t, checker, 1)
+	})
+}
+
 func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) {
 	t.Parallel()
 
@@ -265,7 +301,7 @@ func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) {
 				wantChecked(t, abuseIPDB, suspect, other)
 				wantFailures(t, checker, 2)
 
-				if scores := checker.Snapshot().Scores; len(scores) != 0 {
+				if scores := checker.Snapshot(); len(scores) != 0 {
 					t.Errorf("scores %+v, want none", scores)
 				}
 
@@ -362,7 +398,7 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
 
 		for name, shown := range map[string]string{
 			"the log": log.String(), "the alerts": string(alerted),
-			"the metrics": scrapeMetrics(t, m), "reputation.json": string(kept),
+			"the metrics": scrapeMetrics(t, m), "abuseipdb-scores.json": string(kept),
 		} {
 			if strings.Contains(shown, key) {
 				t.Errorf("%s shows the key:\n%s", name, shown)
@@ -425,12 +461,12 @@ func TestScoreFetchedATTLAgoIsNeitherUsedNorKept(t *testing.T) {
 		Fetched: now.Add(-cacheTTL),
 	}
 
-	checker.Load(reputation.Checks{Scores: []reputation.Score{stale, inUse}})
+	checker.Load([]reputation.Score{stale, inUse})
 
 	wantScore(t, checker, suspect, false, 100, true)
 	wantScore(t, checker, other, false, 0, false)
 
-	got := checker.Snapshot().Scores
+	got := checker.Snapshot()
 	if !reflect.DeepEqual(got, []reputation.Score{inUse}) {
 		t.Errorf("scores %+v, want only %+v", got, inUse)
 	}
@@ -444,7 +480,7 @@ func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T)
 	p.Now = func() time.Time { return now }
 	checker := reputation.NewAbuseIPDB(p)
 
-	// 100,001 scores, listed by client, as reputation.json lists them, each
+	// 100,001 scores, listed by client, as abuseipdb-scores.json lists them, each
 	// fetched a millisecond before the one before it: the last is one too
 	// many.
 	const count = 100001
@@ -460,9 +496,9 @@ func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T)
 		addr = addr.Next()
 	}
 
-	checker.Load(reputation.Checks{Scores: scores})
+	checker.Load(scores)
 
-	got := checker.Snapshot().Scores
+	got := checker.Snapshot()
 	if len(got) != count-1 || !slices.Contains(got, scores[0]) ||
 		slices.Contains(got, scores[count-1]) {
 		t.Errorf("%d scores kept, want all but the one fetched longest ago", len(got))

internal/reputation/crowdsec_test.go

@@ -291,7 +291,7 @@ func crowdSecFailures() []crowdSecFailure {
 // other than its decision list, such as the one a redirect names, and that
 // the key is in none of what the fetches leave behind: log, the process
 // log, the alerts waiting in queue, and the copies of lists, which
-// reputation.json keeps.
+// lists.json keeps.
 func wantKeyNotShown(
 	t *testing.T, e *engine, log string, lists *reputation.Lists, queue *alerts.Queue,
 ) {
@@ -326,7 +326,7 @@ func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo(t *testing
 		lists := start(t, e, crowdSecParams())
 		kept := lists.Snapshot()
 
-		// Restarted half an hour later with what reputation.json keeps, and
+		// Restarted half an hour later with what lists.json keeps, and
 		// the engine down, the decision still bans, until the end it had at
 		// the fetch, half an hour on.
 		time.Sleep(30 * time.Minute)

internal/reputation/dnsbl.go

@@ -40,7 +40,7 @@ var (
 	errNotListing = errors.New("the answer is outside 127.0.0.0/8")
 )
 
-// Verdict is what a zone said about a client, as reputation.json holds
+// Verdict is what a zone said about a client, as dnsbl.json holds
 // it: the zone, the client's address, whether the zone lists it, and when
 // the zone answered.
 type Verdict struct {
@@ -187,7 +187,7 @@ func (d *DNSBL) Failures(zone string) int {
 }
 
 // Snapshot returns every verdict still in use, sorted by client, then by
-// zone, as reputation.json lists them.
+// zone, as dnsbl.json lists them.
 func (d *DNSBL) Snapshot() []Verdict {
 	d.mu.Lock()
 
@@ -209,7 +209,7 @@ func (d *DNSBL) Snapshot() []Verdict {
 	return verdicts
 }
 
-// Load keeps verdicts, read from reputation.json, in place of those it
+// Load keeps verdicts, read from dnsbl.json, in place of those it
 // keeps, but for those of a zone SWW_DNSBL_ZONES does not name, and,
 // past maxVerdicts, those fetched longest ago. One fetched CacheTTL ago or
 // more is neither used nor written, as for any verdict.

internal/reputation/dnsbl_test.go

@@ -389,7 +389,7 @@ func TestVerdictsKeptAcrossARestart(t *testing.T) {
 			t.Errorf("verdicts %+v, want %+v", kept, want)
 		}
 
-		// Restarted an hour later with what reputation.json keeps, it uses
+		// Restarted an hour later with what dnsbl.json keeps, it uses
 		// the verdicts, and asks the zone nothing, until the TTL has passed
 		// since they were fetched.
 		time.Sleep(time.Hour)
@@ -446,7 +446,7 @@ func TestAtMost100000VerdictsKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.
 	p.Now = func() time.Time { return now }
 	dnsbl := reputation.NewDNSBL(p)
 
-	// 100,001 verdicts, listed by client, as reputation.json lists them,
+	// 100,001 verdicts, listed by client, as dnsbl.json lists them,
 	// each fetched a millisecond before the one before it: the last is one
 	// too many.
 	const count = 100001

internal/reputation/reputation.go

@@ -7,8 +7,8 @@
 // and when each was last tried. It also asks the DNSBL zones of
 // SWW_DNSBL_ZONES about clients, and keeps their verdicts, and checks
 // clients with AbuseIPDB, and keeps their scores and the checks spent
-// today. The state package writes all of these to reputation.json and
-// reads them from it, so that a restart keeps them too.
+// today. The state package writes all of these to files and reads them
+// from there, so that a restart keeps them too.
 package reputation
 
 import (
@@ -54,7 +54,7 @@ var (
 		"does not give an address or a netblock and a duration, such as 4h0m0s")
 )
 
-// List is a list as reputation.json holds it: the URL it is fetched from,
+// List is a list as lists.json holds it: the URL it is fetched from,
 // when it was last tried, the fetch failed or not, and its last good copy:
 // when that was fetched, and its lines, as fetched, comment lines
 // included, both left out while no fetch of it has succeeded.
@@ -106,7 +106,7 @@ type Lists struct {
 	lists map[string]*list
 }
 
-// list is one list: what reputation.json keeps of it, its last try, zero
+// list is one list: what lists.json keeps of it, its last try, zero
 // before the first, and its last good copy, what that copy says, and how
 // many fetches of it failed.
 type list struct {
@@ -272,7 +272,7 @@ func (l *Lists) Failures(listURL string) int {
 
 // Run fetches each list once it is due, as due tells, until ctx is done. A
 // list never tried is fetched at once, and so is one that is due by its
-// last try or copy read from reputation.json.
+// last try or copy read from lists.json.
 func (l *Lists) Run(ctx context.Context) {
 	if len(l.lists) == 0 {
 		return
@@ -292,7 +292,7 @@ func (l *Lists) Run(ctx context.Context) {
 }
 
 // Snapshot returns each list that has been tried, with its copy, if it
-// has one, sorted by URL, as reputation.json lists them.
+// has one, sorted by URL, as lists.json lists them.
 func (l *Lists) Snapshot() []List {
 	l.mu.Lock()
 
@@ -313,7 +313,7 @@ func (l *Lists) Snapshot() []List {
 	return tried
 }
 
-// Load puts lists, read from reputation.json, in place of the last tries
+// Load puts lists, read from lists.json, in place of the last tries
 // and copies held. A list Params does not name is dropped. A copy with a
 // line that parse refuses is an error, and then nothing changes.
 func (l *Lists) Load(lists []List) error {

internal/reputation/reputation_test.go

@@ -260,7 +260,7 @@ func TestKeptCopyIsFetchedAgainOnceRefreshHasPassedSinceItWasFetched(t *testing.
 		lists.SetTransport(servers)
 
 		// drop.txt was fetched an hour ago, and tor.txt a refresh ago, as
-		// reputation.json says at start.
+		// lists.json says at start.
 		err := lists.Load([]reputation.List{
 			{URL: dropURL, Fetched: time.Now().Add(-time.Hour), Lines: []string{"203.0.113.9"}},
 			{URL: torURL, Fetched: time.Now().Add(-refresh), Lines: []string{"198.51.100.9"}},
@@ -297,7 +297,7 @@ func TestRestartWaitsRefreshAfterTheLastTryEvenOneThatFailed(t *testing.T) {
 		time.Sleep(refresh)
 		wantFetches(t, servers, 2)
 
-		// Restarted with what reputation.json keeps, it waits a refresh
+		// Restarted with what lists.json keeps, it waits a refresh
 		// after the failed try, as it does while it runs.
 		restarted := &standIn{lists: map[string]string{dropURL: "198.51.100.2\n"}}
 		again := reputation.New(params(dropURL))
@@ -353,7 +353,7 @@ func TestFetchCutOffAsItStopsIsNoFailureButARestartWaitsForIt(t *testing.T) {
 				waiting(queue))
 		}
 
-		// Restarted an hour later with what reputation.json keeps, it fetches
+		// Restarted an hour later with what lists.json keeps, it fetches
 		// tor.txt, never tried, at once, and drop.txt a refresh after its
 		// cut-off try.
 		time.Sleep(time.Hour)

internal/requestlog/requestlog.go

@@ -5,6 +5,7 @@
 package requestlog
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -252,14 +253,20 @@ func WriteRProxyLine(w io.Writer, line *RProxyLine) error {
 	return writeJSONLine(w, line)
 }
 
-// writeJSONLine writes line to w as one JSON line.
+// writeJSONLine writes line to w as one JSON line. It writes "&", "<" and
+// ">" as they are, so a query string reads as it was sent.
 func writeJSONLine(w io.Writer, line any) error {
-	encoded, err := json.Marshal(line)
+	var encoded bytes.Buffer
+
+	encoder := json.NewEncoder(&encoded)
+	encoder.SetEscapeHTML(false)
+
+	err := encoder.Encode(line)
 	if err != nil {
 		return fmt.Errorf("encode the request log line: %w", err)
 	}
 
-	_, err = w.Write(append(encoded, '\n'))
+	_, err = w.Write(encoded.Bytes())
 	if err != nil {
 		return fmt.Errorf("write the request log line: %w", err)
 	}

internal/requestlog/requestlog_test.go

@@ -67,6 +67,44 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) {
 	}
 }
 
+func TestRequestLinesHoldTheQueryAndRefererAsSent(t *testing.T) {
+	t.Parallel()
+
+	var out bytes.Buffer
+
+	err := requestlog.Write(&out, &requestlog.Line{
+		Query:   "a=1&b=<2>",
+		Referer: "https://example.com/?x=1&y=<2>",
+	})
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	err = requestlog.WriteRProxyLine(&out, &requestlog.RProxyLine{Query: "a=1&b=<2>"})
+	if err != nil {
+		t.Fatalf("write the rproxy line: %v", err)
+	}
+
+	lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("wrote %q, want two lines", out.String())
+	}
+
+	for _, line := range lines {
+		if !json.Valid([]byte(line)) {
+			t.Errorf("%s is not valid JSON", line)
+		}
+
+		if !strings.Contains(line, `"query":"a=1&b=<2>"`) {
+			t.Errorf("%s does not hold the query as sent", line)
+		}
+	}
+
+	if !strings.Contains(lines[0], `"referer":"https://example.com/?x=1&y=<2>"`) {
+		t.Errorf("%s does not hold the referer as sent", lines[0])
+	}
+}
+
 func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
 	t.Parallel()
 

internal/routewatch/routewatch.go

@@ -1,6 +1,8 @@
 // Package routewatch asks a routewatch instance about client addresses, in the
-// background, and keeps its answers in memory. A routewatch instance knows, for
-// an address, the netblock it is announced in and the AS that originates it.
+// background, and keeps its answers. A routewatch instance knows, for an
+// address, the netblock it is announced in and the AS that originates it. The
+// answers are kept in memory, for at most 100,000 clients and for an hour each,
+// and are written to routewatch.json and read from it by the state package.
 package routewatch
 
 import (
@@ -13,6 +15,7 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
+	"slices"
 	"sync"
 	"time"
 
@@ -65,6 +68,17 @@ type Route struct {
 	ASN      uint32
 }
 
+// Answer is what routewatch answered about a client, as routewatch.json holds
+// it: the netblock the client's address is announced in, none for an address
+// without a live route, the AS that originates it, and when routewatch
+// answered.
+type Answer struct {
+	Client   netip.Prefix `json:"client"`
+	Netblock netip.Prefix `json:"netblock"`
+	ASN      uint32       `json:"asn"`
+	Answered time.Time    `json:"answered"`
+}
+
 // Params are what New needs.
 type Params struct {
 	// URL is where routewatch is (SWW_ROUTEWATCH_URL).
@@ -84,23 +98,16 @@ type Client struct {
 	mu sync.Mutex
 	// answers are by client. Each is added as it comes and never moved up,
 	// so that the one that came longest ago is the first dropped.
-	answers *simplelru.LRU[netip.Prefix, answer]
+	answers *simplelru.LRU[netip.Prefix, Answer]
 	// asking are the clients whose query is under way.
 	asking map[netip.Prefix]bool
 	// retryAt is when routewatch may be asked again after a query failed.
 	retryAt time.Time
 }
 
-// answer is one kept answer: the route, or why there is none, and when it came.
-type answer struct {
-	route   Route
-	missing Missing
-	came    time.Time
-}
-
 // New returns a Client with no answer yet.
 func New(params Params) *Client {
-	answers, err := simplelru.NewLRU[netip.Prefix, answer](maxAnswers, nil)
+	answers, err := simplelru.NewLRU[netip.Prefix, Answer](maxAnswers, nil)
 	if err != nil {
 		panic(err) // NewLRU fails only for a size below one
 	}
@@ -128,8 +135,12 @@ func (c *Client) Route(
 	now := c.params.Now()
 
 	kept, found := c.answers.Peek(client)
-	if found && now.Sub(kept.came) < answerKept {
-		return kept.route, kept.missing
+	if found && now.Sub(kept.Answered) < answerKept {
+		if !kept.Netblock.IsValid() {
+			return Route{}, NoRoute
+		}
+
+		return Route{Netblock: kept.Netblock, ASN: kept.ASN}, ""
 	}
 
 	if now.Before(c.retryAt) {
@@ -145,12 +156,58 @@ func (c *Client) Route(
 	return Route{}, NoAnswerYet
 }
 
+// Snapshot returns every answer still in use, sorted by client, as
+// routewatch.json lists them.
+func (c *Client) Snapshot() []Answer {
+	c.mu.Lock()
+
+	now := c.params.Now()
+	answers := make([]Answer, 0, c.answers.Len())
+
+	for _, kept := range c.answers.Values() {
+		if now.Sub(kept.Answered) < answerKept {
+			answers = append(answers, kept)
+		}
+	}
+
+	c.mu.Unlock()
+
+	slices.SortFunc(answers, func(a, b Answer) int {
+		return a.Client.Compare(b.Client)
+	})
+
+	return answers
+}
+
+// Load keeps answers, read from routewatch.json, in place of those it keeps,
+// but for those that came answerKept ago or more and, past maxAnswers, those
+// that came longest ago.
+func (c *Client) Load(answers []Answer) {
+	answers = slices.Clone(answers)
+	slices.SortStableFunc(answers, func(a, b Answer) int {
+		return a.Answered.Compare(b.Answered)
+	})
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.answers.Purge()
+
+	now := c.params.Now()
+
+	for _, kept := range answers {
+		if now.Sub(kept.Answered) < answerKept {
+			c.answers.Add(kept.Client, kept)
+		}
+	}
+}
+
 // ask asks routewatch about addr, keeps the answer as client's, and notes the
 // query as no longer under way. A query that fails gives no answer, and
 // routewatch is not asked again for failureKept. Only the first failure of
 // those under way is logged.
 func (c *Client) ask(ctx context.Context, client netip.Prefix, addr netip.Addr) {
-	route, missing, err := c.query(ctx, addr)
+	route, err := c.query(ctx, addr)
 	now := c.params.Now()
 
 	c.mu.Lock()
@@ -160,7 +217,9 @@ func (c *Client) ask(ctx context.Context, client netip.Prefix, addr netip.Addr)
 	firstFailure := err != nil && !now.Before(c.retryAt)
 
 	if err == nil {
-		c.answers.Add(client, answer{route: route, missing: missing, came: now})
+		c.answers.Add(client, Answer{
+			Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
+		})
 	} else {
 		c.retryAt = now.Add(failureKept)
 	}
@@ -174,19 +233,19 @@ func (c *Client) ask(ctx context.Context, client netip.Prefix, addr netip.Addr)
 
 // query asks routewatch's IP info API about addr. An answer other than 200, one
 // that cannot be read, and none within queryTimeout, fail. A result with no
-// netblock is an address without a live route.
-func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, Missing, error) {
+// netblock is an address without a live route, and gives a Route without one.
+func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, error) {
 	ctx, cancel := context.WithTimeout(ctx, queryTimeout)
 	defer cancel()
 
 	endpoint, err := url.JoinPath(c.params.URL, "api/v1/ip", addr.String())
 	if err != nil {
-		return Route{}, "", fmt.Errorf("make the URL: %w", err)
+		return Route{}, fmt.Errorf("make the URL: %w", err)
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
 	if err != nil {
-		return Route{}, "", fmt.Errorf("make the request: %w", err)
+		return Route{}, fmt.Errorf("make the request: %w", err)
 	}
 
 	req.Header.Set("Accept", "application/json")
@@ -195,7 +254,7 @@ func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, Missing, er
 	if err != nil {
 		// Do's error names the URL, which holds the client's address, which
 		// is not to be logged: only what went wrong is kept.
-		return Route{}, "", fmt.Errorf("ask routewatch: %w", errors.Unwrap(err))
+		return Route{}, fmt.Errorf("ask routewatch: %w", errors.Unwrap(err))
 	}
 
 	defer func() {
@@ -203,14 +262,14 @@ func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, Missing, er
 	}()
 
 	if res.StatusCode != http.StatusOK {
-		return Route{}, "", fmt.Errorf("%w %s", errStatus, res.Status)
+		return Route{}, fmt.Errorf("%w %s", errStatus, res.Status)
 	}
 
 	return readAnswer(io.LimitReader(res.Body, maxAnswerBytes))
 }
 
 // readAnswer reads routewatch's answer about one address.
-func readAnswer(body io.Reader) (Route, Missing, error) {
+func readAnswer(body io.Reader) (Route, error) {
 	var decoded struct {
 		Data struct {
 			Results []struct {
@@ -222,22 +281,22 @@ func readAnswer(body io.Reader) (Route, Missing, error) {
 
 	err := json.NewDecoder(body).Decode(&decoded)
 	if err != nil {
-		return Route{}, "", fmt.Errorf("read the answer: %w", err)
+		return Route{}, fmt.Errorf("read the answer: %w", err)
 	}
 
 	if len(decoded.Data.Results) == 0 {
-		return Route{}, "", errNoResult
+		return Route{}, errNoResult
 	}
 
 	result := decoded.Data.Results[0]
 	if result.Netblock == "" {
-		return Route{}, NoRoute, nil
+		return Route{}, nil
 	}
 
 	netblock, err := netip.ParsePrefix(result.Netblock)
 	if err != nil {
-		return Route{}, "", fmt.Errorf("%w: %q", errNotNetblock, result.Netblock)
+		return Route{}, fmt.Errorf("%w: %q", errNotNetblock, result.Netblock)
 	}
 
-	return Route{Netblock: netblock.Masked(), ASN: result.ASN}, "", nil
+	return Route{Netblock: netblock.Masked(), ASN: result.ASN}, nil
 }

internal/routewatch/routewatch_test.go

@@ -7,10 +7,12 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
+	"testing/synctest"
 	"time"
 
 	"sneak.berlin/go/smallwebwaf/internal/routewatch"
@@ -113,6 +115,28 @@ func answering(body string) http.HandlerFunc {
 	}
 }
 
+// silentURL is where a routewatch that never answers is: a query to it waits,
+// without the network, until it is abandoned. TestMain registers it with Go's
+// default transport, through which the Client asks.
+const silentURL = "silent://routewatch/"
+
+func TestMain(m *testing.M) {
+	transport, _ := http.DefaultTransport.(*http.Transport)
+	transport.RegisterProtocol("silent", silentRoutewatch{})
+
+	m.Run()
+}
+
+// silentRoutewatch is the routewatch at silentURL.
+type silentRoutewatch struct{}
+
+// RoundTrip waits until req is abandoned.
+func (silentRoutewatch) RoundTrip(req *http.Request) (*http.Response, error) {
+	<-req.Context().Done()
+
+	return nil, req.Context().Err()
+}
+
 func (f *fixture) route() (routewatch.Route, routewatch.Missing) {
 	addr := netip.MustParseAddr(clientAddr)
 	client := netip.PrefixFrom(addr, addr.BitLen())
@@ -287,20 +311,29 @@ func TestFailureIsKeptForAMinuteSoRoutewatchIsNotAskedPerRequest(t *testing.T) {
 func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
 	t.Parallel()
 
-	f := newFixture(t, func(_ http.ResponseWriter, r *http.Request) {
-		<-r.Context().Done()
+	// In a synctest bubble, time passes only while every goroutine in it waits,
+	// so Route's wait is measured exactly, however busy the host is. routewatch
+	// is the one at silentURL: one reached over the network would keep the
+	// bubble's clock from moving.
+	synctest.Test(t, func(t *testing.T) {
+		// Without newFixture's stand-in, which is reached over the network.
+		f := &fixture{t: t, log: &syncBuffer{}}
+		f.client = routewatch.New(routewatch.Params{
+			URL: silentURL, Now: time.Now,
+			ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
+		})
+
+		began := time.Now()
+		_, missing := f.route()
+
+		if waited := time.Since(began); missing != routewatch.NoAnswerYet || waited != 0 {
+			t.Errorf("Route gave %q after %s while routewatch hangs, want %q at once",
+				missing, waited, routewatch.NoAnswerYet)
+		}
+
+		f.waitFor(routewatch.RequestFailed)
+		f.wantFailureLoggedWithoutTheClient()
 	})
-
-	start := time.Now()
-
-	f.route()
-
-	if elapsed := time.Since(start); elapsed > time.Second {
-		t.Errorf("Route took %s while routewatch hangs", elapsed)
-	}
-
-	f.waitFor(routewatch.RequestFailed)
-	f.wantFailureLoggedWithoutTheClient()
 }
 
 func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
@@ -336,3 +369,96 @@ func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
 		t.Errorf("%d queries after the answer expired, want 2", got)
 	}
 }
+
+func TestSnapshotHoldsTheAnswerUntilItExpires(t *testing.T) {
+	t.Parallel()
+
+	f := newFixture(t, answering(routedAnswer))
+
+	f.route()
+	f.waitFor("")
+
+	want := []routewatch.Answer{{
+		Client:   netip.MustParsePrefix(clientAddr + "/32"),
+		Netblock: netip.MustParsePrefix("192.0.2.0/24"), ASN: 64496,
+		Answered: f.clk.Now(),
+	}}
+	if got := f.client.Snapshot(); !slices.Equal(got, want) {
+		t.Errorf("snapshot %+v, want %+v", got, want)
+	}
+
+	f.clk.advance(time.Hour)
+
+	if got := f.client.Snapshot(); len(got) != 0 {
+		t.Errorf("snapshot %+v an hour after the answer, want none", got)
+	}
+}
+
+func TestLoadedAnswersAreUsedForAnHourFromWhenTheyCame(t *testing.T) {
+	t.Parallel()
+
+	f := newFixture(t, answering(routedAnswer))
+	now := f.clk.Now()
+	kept := routewatch.Answer{
+		Client:   netip.MustParsePrefix(clientAddr + "/32"),
+		Netblock: netip.MustParsePrefix("198.51.100.0/24"), ASN: 64497,
+		Answered: now.Add(-time.Hour + time.Second),
+	}
+	stale := routewatch.Answer{
+		Client:   netip.MustParsePrefix("192.0.2.2/32"),
+		Netblock: netip.MustParsePrefix("192.0.2.0/24"), ASN: 64496,
+		Answered: now.Add(-time.Hour),
+	}
+
+	f.client.Load([]routewatch.Answer{kept, stale})
+
+	if got := f.client.Snapshot(); !slices.Equal(got, []routewatch.Answer{kept}) {
+		t.Errorf("kept %+v, want only the answer that came less than an hour ago", got)
+	}
+
+	route, missing := f.route()
+
+	want := routewatch.Route{Netblock: kept.Netblock, ASN: kept.ASN}
+	if route != want || missing != "" {
+		t.Errorf("route %+v (%q), want %+v", route, missing, want)
+	}
+
+	if got := f.queries.Load(); got != 0 {
+		t.Errorf("%d queries for a client with an answer loaded, want none", got)
+	}
+}
+
+func TestLoadDropsTheAnswerThatCameLongestAgoFirst(t *testing.T) {
+	t.Parallel()
+
+	const maxAnswers = 100000
+
+	f := newFixture(t, answering(routedAnswer))
+	now := f.clk.Now()
+
+	// routewatch.json lists the answers by client. Here each came a
+	// millisecond before the one listed before it, so the last listed is
+	// the one that came longest ago, and the one dropped.
+	answers := make([]routewatch.Answer, maxAnswers+1)
+	addr := netip.MustParseAddr("10.0.0.0")
+
+	for i := range answers {
+		answers[i] = routewatch.Answer{
+			Client:   netip.PrefixFrom(addr, addr.BitLen()),
+			Answered: now.Add(-time.Duration(i) * time.Millisecond),
+		}
+		addr = addr.Next()
+	}
+
+	f.client.Load(answers)
+
+	got := f.client.Snapshot()
+	if len(got) != maxAnswers {
+		t.Fatalf("%d answers kept, want %d", len(got), maxAnswers)
+	}
+
+	if got[0] != answers[0] || got[maxAnswers-1] != answers[maxAnswers-1] {
+		t.Errorf("answers kept from %s to %s, want from %s to %s", got[0].Client,
+			got[maxAnswers-1].Client, answers[0].Client, answers[maxAnswers-1].Client)
+	}
+}

internal/rproxy/client.go

@@ -105,15 +105,14 @@ func setForwardedHeaders(in, out *http.Request, peer netip.Addr, trusted bool) {
 		return
 	}
 
-	// ReverseProxy has removed Forwarded and the three set below; these are the
-	// other headers in which traefik tells the app about the visitor and its
-	// request.
-	for _, name := range []string{
-		"X-Forwarded-Port", "X-Forwarded-Server", "X-Forwarded-Uri",
-		"X-Forwarded-Method", "X-Forwarded-Prefix", "X-Forwarded-Tls-Client-Cert",
-		"X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip",
-	} {
-		out.Header.Del(name)
+	// An app behind CGI or FastCGI reads X_Forwarded_Ssl as X-Forwarded-Ssl, so
+	// a name is matched with each _ read as - and in any letter case.
+	for name := range out.Header {
+		spelt := strings.ToLower(strings.ReplaceAll(name, "_", "-"))
+		if strings.HasPrefix(spelt, "x-forwarded-") ||
+			spelt == "forwarded" || spelt == "x-real-ip" {
+			delete(out.Header, name)
+		}
 	}
 
 	out.Header.Set("X-Forwarded-For", forwardedFor)

internal/rproxy/limits_test.go

@@ -382,6 +382,20 @@ type deadlineRecorder struct {
 	// stalled, if set, sends the part of the request body that never arrives. A
 	// read deadline that has passed ends a read of it, as on a connection.
 	stalled *io.PipeWriter
+	// flushError, if set, is what every flush returns, and nothing is flushed.
+	flushError error
+}
+
+// FlushError, which http.ResponseController calls to flush, returns flushError
+// if it is set, and otherwise flushes.
+func (w *deadlineRecorder) FlushError() error {
+	if w.flushError != nil {
+		return w.flushError
+	}
+
+	w.Flush()
+
+	return nil
 }
 
 // SetReadDeadline notes deadline.
@@ -447,6 +461,45 @@ func TestABodyLeftUnreadIsReadOnlyUntilTheRequestTimeLimit(t *testing.T) {
 	}
 }
 
+func TestABodyLeftUnreadIsReadNoMoreOnceTheTimeToSendTheAnswerRunsOut(t *testing.T) {
+	t.Parallel()
+
+	standIn := startWAF(t, allow)
+	app := startApp(t, answerBeforeTheBody)
+	_, _, server := startServer(t, standIn.URL, app.URL, nil)
+
+	// The visitor sends the first part and a little more, and then nothing.
+	stalled, stop := io.Pipe()
+
+	t.Cleanup(func() { _ = stop.Close() })
+
+	req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/",
+		io.MultiReader(bytes.NewReader(make([]byte, questionBodyMaxBytes+1)), stalled))
+	req.ContentLength = 2 * questionBodyMaxBytes
+	req.Header.Set(contentLengthHeader, strconv.Itoa(2*questionBodyMaxBytes))
+
+	// Every flush fails, as once the time to send the answer has run out. The
+	// default time limits are far longer than the test, so rproxy records no
+	// refusal.
+	w := &deadlineRecorder{
+		ResponseRecorder: httptest.NewRecorder(),
+		stalled:          stop,
+		flushError:       os.ErrDeadlineExceeded,
+	}
+
+	before := time.Now()
+
+	server.Handler.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Errorf("status %d, want the app's 200", w.Code)
+	}
+
+	if w.readDeadline.Before(before) || w.readDeadline.After(time.Now()) {
+		t.Errorf("the reading of the body ends at %s, want now", w.readDeadline)
+	}
+}
+
 func TestAnAnswerRProxyGivesItselfHasTheResponseTimeToBeSent(t *testing.T) {
 	t.Parallel()
 
@@ -807,8 +860,10 @@ func TestAVisitorThatStopsSendingAfterTheAppHasAnsweredGetsTheAnswer(t *testing.
 	app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
 		_, _ = io.WriteString(w, greeting)
 	})
+	// The request time limit is far longer than dial's deadline, so that a busy
+	// host cannot use it up before the app's answer arrives.
 	addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
-		maxRequestTime: "3000",
+		maxRequestTime: "120000",
 	})
 
 	// Far more announced than sent, so that Go's server answers for the app
@@ -869,11 +924,11 @@ func TestAVisitorThatStopsALittleShortGetsTheAppsAnswerAtOnce(t *testing.T) {
 
 	standIn := startWAF(t, allow)
 	app := startApp(t, answerBeforeTheBody)
-	// The request time limit is longer than the 10 seconds the status and headers
-	// of an answer have to be sent, and than dial's deadline.
+	// Both time limits are far longer than dial's deadline, so that the answer
+	// has to come before either runs out, and a busy host cannot use them up.
 	addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
-		maxRequestTime:  "15000",
-		maxResponseTime: "300",
+		maxRequestTime:  "120000",
+		maxResponseTime: "60000",
 	})
 
 	conn := dial(t, addr)
@@ -910,6 +965,77 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(t *testing.
 	}
 }
 
+func TestAVisitorThatStopsALittleShortHasTheConnectionClosedWhenTheAnswerIsCutOff(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	// rproxy passes on a part sent at once well within this limit, even on a busy
+	// host.
+	const limit = 2 * time.Second
+
+	// The app sends the parts in atOnce with its status and headers. It sends the
+	// parts in late only once rproxy has ended the request to it, which rproxy
+	// does when it cuts off the answer, so that those never reach the visitor.
+	for _, tc := range []struct {
+		name         string
+		atOnce, late []string
+	}{
+		{"with the body in one part", nil, []string{greeting}},
+		{"with the body in two parts", []string{greeting[:5]}, []string{greeting[5:]}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			standIn := startWAF(t, allow)
+			app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
+				rc := http.NewResponseController(w)
+				_ = rc.EnableFullDuplex()
+
+				w.Header().Set(contentLengthHeader, strconv.Itoa(len(greeting)))
+
+				for _, part := range tc.atOnce {
+					_, _ = io.WriteString(w, part)
+				}
+
+				_ = rc.Flush()
+
+				// The request's body ends when rproxy ends the request to the app.
+				_, _ = io.Copy(io.Discard, r.Body)
+
+				for _, part := range tc.late {
+					_, _ = io.WriteString(w, part)
+					_ = rc.Flush()
+				}
+			})
+
+			// The request time limit is far longer than dial's deadline.
+			addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
+				maxRequestTime:  "120000",
+				maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
+			})
+
+			conn := dial(t, addr)
+			sendAllButALittle(t, conn)
+
+			res, err := http.ReadResponse(bufio.NewReader(conn), nil)
+			if err != nil {
+				t.Fatalf("read the answer: %v", err)
+			}
+
+			// If the connection stays open, dial's deadline ends this read with a
+			// timeout.
+			body, err := io.ReadAll(res.Body)
+			_ = res.Body.Close()
+
+			want := strings.Join(tc.atOnce, "")
+			if string(body) != want || !errors.Is(err, io.ErrUnexpectedEOF) {
+				t.Errorf("read %q (%v), want %q and the connection closed", body, err, want)
+			}
+		})
+	}
+}
+
 func TestAVisitorThatStopsSendingWhileTheAppWorksGetsNoAnswer(t *testing.T) {
 	t.Parallel()
 

internal/rproxy/request.go

@@ -272,6 +272,10 @@ func (rq *request) modifyResponse(res *http.Response) error {
 
 	rq.closeAfterAnswer()
 
+	// An answer that arrives before the app has been sent the whole request ends
+	// the request, and SWRP_MAX_RESPONSE_TIME_MS starts.
+	rq.startResponseTimer(requestEnd)
+
 	res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
 	rq.out.afterStatus = rq.setResponseDeadline
 
@@ -384,8 +388,23 @@ func (rq *request) refuse(status int) {
 func (rq *request) finish(ctx context.Context) {
 	rq.stopTimers()
 
-	if rq.refused.Load() == 0 {
-		rq.stopPassingOnBody()
+	// After a cut-off answer, the reading of a body that has not arrived whole
+	// ends now. Go's server would otherwise keep the connection open while it
+	// reads the rest. modifyResponse sets afterStatus only when it passes the
+	// app's answer on. Any refusal after that has cut the answer off, since a
+	// refusal ends the request to the app. Any answer is cut off when the time to
+	// send it runs out: a flush then fails, and every flush after a failed one
+	// fails too. ReverseProxy ignores the failure, but panics if it was still
+	// waiting for more of the app's answer; finish runs either way. Before a
+	// status, a flush would send a 200; after a 101, ReverseProxy has taken over
+	// the connection.
+	switch {
+	case rq.out.afterStatus != nil && rq.refused.Load() != 0,
+		rq.out.status != 0 && rq.out.status != http.StatusSwitchingProtocols &&
+			errors.Is(rq.rc.Flush(), os.ErrDeadlineExceeded):
+		rq.stopPassingOnBody(time.Now())
+	case rq.refused.Load() == 0:
+		rq.stopPassingOnBody(rq.requestDeadline())
 	}
 
 	requestEnd := rq.endRequest()
@@ -473,10 +492,10 @@ func (rq *request) stopReadingBody(deadline time.Time) {
 }
 
 // stopPassingOnBody ends the transport's reading of a visitor's body that has
-// not arrived whole, and lets Go's server read the rest until
-// SWRP_MAX_REQUEST_TIME_MS runs out. Go's server reads it once the handler
-// returns, but with no deadline at all if another read of it is still waiting.
-func (rq *request) stopPassingOnBody() {
+// not arrived whole, and lets Go's server read the rest until deadline. Go's
+// server reads it once the handler returns, but with no deadline at all if
+// another read of it is still waiting.
+func (rq *request) stopPassingOnBody(deadline time.Time) {
 	if rq.body == nil || rq.body.received.Load() {
 		return
 	}
@@ -489,7 +508,7 @@ func (rq *request) stopPassingOnBody() {
 	rq.body.stopped = true
 	rq.body.mu.Unlock()
 
-	rq.stopReadingBody(rq.requestDeadline())
+	rq.stopReadingBody(deadline)
 }
 
 // startRequestTimer starts SWRP_MAX_REQUEST_TIME_MS, which runs from the
@@ -542,19 +561,29 @@ func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) {
 	requestEnd := rq.endRequest()
 
 	rq.mu.Lock()
-	defer rq.mu.Unlock()
+	stopTimer(rq.requestTimer)
+	rq.mu.Unlock()
 
-	if rq.timersStopped {
+	rq.startResponseTimer(requestEnd)
+}
+
+// startResponseTimer starts SWRP_MAX_RESPONSE_TIME_MS, which runs from
+// requestEnd, unless it has started already.
+func (rq *request) startResponseTimer(requestEnd time.Time) {
+	limit := rq.h.config.MaxResponseTime
+	if limit == 0 {
 		return
 	}
 
-	stopTimer(rq.requestTimer)
+	rq.mu.Lock()
+	defer rq.mu.Unlock()
 
-	limit := rq.h.config.MaxResponseTime
-	if limit > 0 {
-		rq.responseTimer = time.AfterFunc(time.Until(requestEnd.Add(limit)),
-			rq.responseTimedOut)
+	if rq.timersStopped || rq.responseTimer != nil {
+		return
 	}
+
+	rq.responseTimer = time.AfterFunc(time.Until(requestEnd.Add(limit)),
+		rq.responseTimedOut)
 }
 
 // responseTimedOut is called when SWRP_MAX_RESPONSE_TIME_MS runs out before the

internal/rproxy/rproxy_test.go

@@ -559,6 +559,9 @@ func TestAPeerNotTrustedHasItsForwardedHeadersReplaced(t *testing.T) {
 		"X-Forwarded-Port", "X-Forwarded-Server", "X-Forwarded-Uri",
 		"X-Forwarded-Method", "X-Forwarded-Prefix", "X-Forwarded-Tls-Client-Cert",
 		"X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip",
+		"X-Forwarded-Scheme", "X-Forwarded-Ssl", "X-Forwarded-Protocol",
+		"X-Forwarded-Path", "X-Forwarded-Client-Cert",
+		"X_Forwarded_Ssl", "X_Forwarded_Proto", "X_Real_IP",
 	}
 
 	standIn := startWAF(t, allow)
@@ -582,6 +585,9 @@ func TestAPeerNotTrustedHasItsForwardedHeadersReplaced(t *testing.T) {
 		req.Header.Set(name, "claimed")
 	}
 
+	// A made-up name, sent in lower case.
+	req.Header["x-forwarded-made-up"] = []string{"claimed"}
+
 	wantGreeting(t, visit(t, req))
 
 	header := <-seen
@@ -596,7 +602,7 @@ func TestAPeerNotTrustedHasItsForwardedHeadersReplaced(t *testing.T) {
 		}
 	}
 
-	for _, name := range removed {
+	for _, name := range append(removed, "X-Forwarded-Made-Up") {
 		if got := header.Values(name); len(got) != 0 {
 			t.Errorf("the app saw %s %q, want none", name, got)
 		}
@@ -622,6 +628,8 @@ func TestATrustedProxysForwardedHeadersPassOn(t *testing.T) {
 		"X-Forwarded-Host":   "public.example",
 		forwardedProtoHeader: "https",
 		"X-Real-Ip":          "198.51.100.1",
+		"X-Forwarded-Ssl":    "on",
+		"X_Forwarded_Ssl":    "on",
 	}
 
 	req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
@@ -1023,7 +1031,9 @@ func TestAnAnswerOf1MiBGoesToTheVisitor(t *testing.T) {
 		w.WriteHeader(http.StatusInternalServerError)
 		_, _ = w.Write(make([]byte, size))
 	})
-	addr, _ := startRProxy(t, standIn.URL, nowhere, nil)
+	addr, _ := startRProxy(t, standIn.URL, nowhere, map[string]string{
+		timeoutSetting: "60000",
+	})
 
 	got := get(t, addr, "/")
 	if got.status != http.StatusInternalServerError || len(got.body) != size {
@@ -1117,6 +1127,7 @@ func TestAnAnswerThatCannotBePassedOnIsAnswered502(t *testing.T) {
 					t.Error("the app was asked")
 				})
 				addr, out := startRProxy(t, backend(t), app.URL, map[string]string{
+					timeoutSetting:     "60000",
 					defaultDisposition: setting,
 				})
 
@@ -1140,7 +1151,9 @@ func TestAnAnswerThatCannotBePassedOnGetsNoReport(t *testing.T) {
 	t.Parallel()
 
 	standIn := startWAF(t, overOneMiB(dispositionDeny))
-	addr, out := startRProxy(t, standIn.URL, nowhere, nil)
+	addr, out := startRProxy(t, standIn.URL, nowhere, map[string]string{
+		timeoutSetting: "60000",
+	})
 
 	if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
 		t.Errorf("answered %+v, want 502", got)

internal/sww/healthcheck_test.go

@@ -41,7 +41,7 @@ func TestHealthCheckInWAFsContainer(t *testing.T) {
 		exited <- run(ctx, settings, out)
 	}()
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
 	_, port, _ := net.SplitHostPort(addr)
 
 	// The container's settings: an address to listen on with an empty

internal/sww/rproxy_test.go

@@ -71,6 +71,8 @@ func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
 		}, out)
 	}()
 
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
+
 	t.Cleanup(func() {
 		stop()
 
@@ -86,8 +88,6 @@ func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
 		out.line(t, "msg", "stopped")
 	})
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
-
 	return "http://" + addr, out
 }
 
@@ -105,7 +105,7 @@ func startWAF(t *testing.T, env map[string]string) (string, *output, func()) {
 		exited <- run(ctx, env, out)
 	}()
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
 
 	return addr, out, func() {
 		stop()
@@ -507,9 +507,10 @@ func TestAStatusWAFDoesNotTakeIsReportedAs599(t *testing.T) {
 }
 
 // startRProxyProcess runs `sww rproxy` in a process of its own, with the
-// settings in env, each as NAME=value. It returns where rproxy listens, the
-// process, and what rproxy writes.
-func startRProxyProcess(t *testing.T, env ...string) (string, *exec.Cmd, *output) {
+// settings in env, each as NAME=value. It returns where rproxy listens and a
+// function that sends rproxy SIGTERM and fails the test unless it exits with
+// status 0 within waitLimit.
+func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
 	t.Helper()
 
 	binary, err := os.Executable()
@@ -517,11 +518,8 @@ func startRProxyProcess(t *testing.T, env ...string) (string, *exec.Cmd, *output
 		t.Fatalf("find the test binary: %v", err)
 	}
 
-	ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit)
-	t.Cleanup(cancel)
-
 	out := &output{}
-	cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary
+	cmd := exec.CommandContext(t.Context(), binary) //nolint:gosec // this test binary
 	cmd.Stdout = out
 
 	cmd.Env = append(os.Environ(),
@@ -529,19 +527,25 @@ func startRProxyProcess(t *testing.T, env ...string) (string, *exec.Cmd, *output
 		"SWRP_LISTEN_ADDR="+localhost+":0")
 	cmd.Env = append(cmd.Env, env...)
 
-	err = cmd.Start()
-	if err != nil {
-		t.Fatalf("start: %v", err)
+	exited := startProcess(t, cmd)
+
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
+
+	return addr, func() {
+		err := cmd.Process.Signal(syscall.SIGTERM)
+		if err != nil {
+			t.Fatalf("signal: %v", err)
+		}
+
+		select {
+		case status := <-exited:
+			if status != 0 {
+				t.Errorf("exit status %d, want 0; output:\n%s", status, out.text())
+			}
+		case <-time.After(waitLimit):
+			t.Error("still running after being told to stop")
+		}
 	}
-
-	t.Cleanup(func() {
-		_ = cmd.Process.Kill()
-		_ = cmd.Wait()
-	})
-
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
-
-	return addr, cmd, out
 }
 
 func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
@@ -572,7 +576,7 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
 		t.Fatalf("write the certificate: %v", err)
 	}
 
-	addr, _, _ := startRProxyProcess(t,
+	addr, _ := startRProxyProcess(t,
 		wafBackend+"="+backend.URL,
 		"SWRP_UPSTREAM_URL="+startApp(t),
 		"SSL_CERT_FILE="+certificates)
@@ -651,7 +655,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
 		exited <- runRProxy(ctx, env, out)
 	}()
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
 
 	wantStatus(t, "http://"+addr+"/", placed, http.StatusOK)
 
@@ -750,7 +754,7 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
 		exited <- runRProxy(ctx, env, out)
 	}()
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
 	conn := sendAndKeepOpen(t, addr, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
 
 	select {
@@ -845,7 +849,7 @@ func TestMainRunsRProxy(t *testing.T) {
 	t.Parallel()
 
 	// Nothing listens there; the health check does not ask.
-	addr, cmd, out := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
+	addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
 
 	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
 		"http://"+addr+waf.HealthPath, http.NoBody)
@@ -868,13 +872,5 @@ func TestMainRunsRProxy(t *testing.T) {
 		t.Errorf("health check answered %q (%v)", body, err)
 	}
 
-	err = cmd.Process.Signal(syscall.SIGTERM)
-	if err != nil {
-		t.Fatalf("signal: %v", err)
-	}
-
-	err = cmd.Wait()
-	if err != nil {
-		t.Errorf("exit: %v; output:\n%s", err, out.text())
-	}
+	stop()
 }

internal/sww/sww.go

@@ -1,9 +1,9 @@
 // Package sww runs the sww process, as the command it is given.
 // `sww waf` reads the settings, the rule files, the lookup database
-// and the state files, answers rproxy's questions until it is told to stop, and
-// then stops in an orderly way, writing the state files. `sww rproxy`
-// reads rproxy's settings alone, and runs the reverse proxy in front of one app
-// that asks waf about each request. `sww healthcheck` is the
+// and the state and cache files, answers rproxy's questions until it is told to
+// stop, and then stops in an orderly way, writing the state and cache files.
+// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
+// front of one app that asks waf about each request. `sww healthcheck` is the
 // container's health check.
 package sww
 
@@ -101,8 +101,8 @@ func printUsage(w io.Writer) int {
 }
 
 // RunWAF is `sww waf`: it reads the settings, the rule files,
-// the lookup database and the state files, then serves requests until ctx is
-// done. It returns the process's exit status, 1 when waf cannot start.
+// the lookup database and the state and cache files, then serves requests until
+// ctx is done. It returns the process's exit status, 1 when waf cannot start.
 func RunWAF(ctx context.Context, params Params) int {
 	// Until the settings are read, the one message is an invalid setting's
 	// error, which every SWW_LOG_LEVEL lets through.
@@ -171,7 +171,7 @@ func RunWAF(ctx context.Context, params Params) int {
 
 	files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
 	if err != nil {
-		processLog.Error("cannot use the state files", "error", err.Error())
+		processLog.Error("cannot use the state and cache files", "error", err.Error())
 
 		return 1
 	}
@@ -236,14 +236,15 @@ func newServer(
 	return server, nil
 }
 
-// loadStateFiles reads the state files into the parts of server and into
-// alertQueue, as state.Load does.
+// loadStateFiles reads the state and cache files into the parts of server and
+// into alertQueue, as state.Load does.
 func loadStateFiles(
 	cfg *config.Config, server *waf.Server, alertQueue *alerts.Queue,
 	now func() time.Time, processLog *slog.Logger,
 ) (*state.Files, error) {
 	return state.Load(state.Params{
 		Dir:             cfg.StateDir,
+		CacheDir:        cfg.CacheDir,
 		WriteDelay:      cfg.StateWriteDelay,
 		CounterInterval: cfg.StateCounterInterval,
 		Ledger:          server.Ledger,
@@ -252,6 +253,7 @@ func loadStateFiles(
 		Lists:           server.Lists,
 		DNSBL:           server.DNSBL,
 		AbuseIPDB:       server.AbuseIPDB,
+		RouteWatch:      server.RouteWatch,
 		Alerts:          alertQueue,
 		Anomalies:       server.Anomalies,
 		Now:             now,
@@ -317,13 +319,13 @@ func startSending(
 	}
 }
 
-// serve serves requests on listener, writes the state files as they are
-// due, takes in an admin's edits of them, reads the rule files again as
+// serve serves requests on listener, writes the state and cache files as they
+// are due, takes in an admin's edits of them, reads the rule files again as
 // they change, and the lookup database when it is replaced, fetches the
 // lists the settings name by URL as they are due, and sends the alerts,
 // until ctx is done. Then it gives the requests in progress
-// shutdownTimeout to finish, and writes every state file, alerts.json with
-// the alerts still waiting.
+// shutdownTimeout to finish, and writes every state and cache file,
+// alerts.json with the alerts still waiting.
 func serve(
 	ctx context.Context, server *waf.Server, listener net.Listener,
 	files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
@@ -393,7 +395,7 @@ func serve(
 
 	err = files.WriteAll()
 	if err != nil {
-		processLog.Error("writing the state files failed", "error", err.Error())
+		processLog.Error("writing the state and cache files failed", "error", err.Error())
 
 		return 1
 	}

internal/sww/sww_test.go

@@ -38,6 +38,7 @@ const (
 	stateDir             = "SWW_STATE_DIR"
 	stateWriteDelay      = "SWW_STATE_WRITE_DELAY"
 	stateCounterInterval = "SWW_STATE_COUNTER_INTERVAL"
+	cacheDir             = "SWW_CACHE_DIR"
 	rateLimitPerDay      = "SWW_RATE_LIMIT_PER_DAY"
 	rateLimitExemptNets  = "SWW_RATE_LIMIT_EXEMPT_NETS"
 	rulesDir             = "SWW_RULES_DIR"
@@ -92,6 +93,31 @@ func (o *output) line(t *testing.T, key, value string) map[string]any {
 	return nil
 }
 
+// lineBeforeExit returns the first line whose msg is msg, waiting for it
+// while sww runs. It waits as long as that takes, so that a slow start
+// cannot fail the test, and fails the test if sww exits first.
+func (o *output) lineBeforeExit(
+	t *testing.T, msg string, exited <-chan int,
+) map[string]any {
+	t.Helper()
+
+	for {
+		fields := o.findLine(t, func(fields map[string]any) bool {
+			return fields["msg"] == msg
+		})
+		if fields != nil {
+			return fields
+		}
+
+		select {
+		case status := <-exited:
+			t.Fatalf("exit status %d while waiting for a line with msg %q; output:\n%s",
+				status, msg, o.text())
+		case <-time.After(pollInterval):
+		}
+	}
+}
+
 // findLine returns the first line written so far for which matches is true, or
 // nil.
 func (o *output) findLine(
@@ -135,6 +161,12 @@ func run(ctx context.Context, env map[string]string, out *output) int {
 				return "off", true
 			}
 
+			// The cache files go beside the state files unless the test
+			// says otherwise.
+			if !ok && name == cacheDir {
+				return env[stateDir], env[stateDir] != ""
+			}
+
 			return value, ok
 		},
 		Stdout: out,
@@ -236,11 +268,12 @@ func TestServesUntilToldToStop(t *testing.T) {
 	}()
 
 	// The default rule file is read.
-	if rules := out.line(t, "msg", "read the rule files")["rules"]; rules != 12.0 {
+	readRules := out.lineBeforeExit(t, "read the rule files", exited)
+	if rules := readRules["rules"]; rules != 12.0 {
 		t.Errorf("read %v rules from the default rule file, want 12", rules)
 	}
 
-	starting := out.line(t, "msg", "starting")
+	starting := out.lineBeforeExit(t, "starting", exited)
 	wantStartingLine(t, starting, dir)
 
 	addr, _ := starting["address"].(string)
@@ -286,7 +319,7 @@ func TestLogLevelHoldsBackTheLessSevereProcessLines(t *testing.T) {
 		}, out)
 	}()
 
-	out.line(t, "msg", "fetching a list failed")
+	out.lineBeforeExit(t, "fetching a list failed", exited)
 	stop()
 
 	select {
@@ -673,7 +706,7 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
 	}
 }
 
-func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T) {
+func TestBlocklistTriesAndCopiesKeptInListsJSONAcrossRestarts(t *testing.T) {
 	t.Parallel()
 
 	const (
@@ -706,10 +739,11 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
 	t.Cleanup(server.Close)
 
 	torURL, dropURL := server.URL+torPath, server.URL+dropPath
-	dir := t.TempDir()
+	cache := t.TempDir()
 	env := map[string]string{
 		listenAddr:           localhost + ":0",
-		stateDir:             dir,
+		stateDir:             t.TempDir(),
+		cacheDir:             cache,
 		rulesDir:             t.TempDir(),
 		metricsToken:         token,
 		instanceName:         instance,
@@ -729,7 +763,7 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
 	})
 
 	// Restarted with drop.txt named after it and the server answering, tor.txt
-	// waits SWW_BLOCKLIST_REFRESH after its failed try, kept in reputation.json,
+	// waits SWW_BLOCKLIST_REFRESH after its failed try, kept in lists.json,
 	// while drop.txt, never tried, is fetched at once. Lists are fetched in the
 	// order named, so once drop.txt refuses the client, tor.txt has had its turn.
 	failing.Store(false)
@@ -747,8 +781,8 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
 	}
 
 	// After another restart, with the server failing, the copy of drop.txt
-	// kept in reputation.json, its copyright line included, refuses the
-	// client from the first request.
+	// kept in lists.json, its copyright line included, refuses the client
+	// from the first request.
 	failing.Store(true)
 
 	out = runUntilStopped(t, env, func(url string) {
@@ -756,16 +790,16 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
 	})
 	wantDeniedByList(t, out.line(t, "type", "request"), dropURL)
 
-	path := filepath.Join(dir, "reputation.json")
+	path := filepath.Join(cache, "lists.json")
 
 	kept, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
 	if err != nil || !strings.Contains(string(kept), `"`+copyright+`"`) {
-		t.Errorf("reputation.json holds\n%s\nwant the copy with %q (%v)", kept, copyright,
+		t.Errorf("lists.json holds\n%s\nwant the copy with %q (%v)", kept, copyright,
 			err)
 	}
 }
 
-func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
+func TestCrowdSecDecisionListKeptInListsJSONAcrossARestart(t *testing.T) {
 	t.Parallel()
 
 	const bouncerKey = "crowdsec-key-0123456789abcdef"
@@ -788,10 +822,11 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
 		}))
 	t.Cleanup(engine.Close)
 
-	dir := t.TempDir()
+	dir, cache := t.TempDir(), t.TempDir()
 	env := map[string]string{
 		listenAddr:              localhost + ":0",
 		stateDir:                dir,
+		cacheDir:                cache,
 		rulesDir:                t.TempDir(),
 		"SWW_CROWDSEC_LAPI_URL": engine.URL,
 		"SWW_CROWDSEC_LAPI_KEY": bouncerKey,
@@ -810,7 +845,7 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
 		t.Errorf("bans.json holds %v, want the ban for crowdsec on %s", ban, placed)
 	}
 
-	// Restarted with the engine down, the copy kept in reputation.json bans
+	// Restarted with the engine down, the copy kept in lists.json bans
 	// another client in the netblock from its first request.
 	down.Store(true)
 
@@ -818,22 +853,25 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
 		wantStatus(t, url, "203.0.113.10", http.StatusForbidden)
 	})
 
-	// The key is in neither run's output, nor in a state file.
-	files := []string{"bans.json", "reputation.json", "clients.json"}
+	// The key is in neither run's output, nor in a state or cache file.
+	files := []string{
+		filepath.Join(dir, "bans.json"), filepath.Join(dir, "clients.json"),
+		filepath.Join(cache, "lists.json"),
+	}
 	shown := make([]string, 0, len(files)+2)
 	shown = append(shown, first.text(), second.text())
 
-	for _, name := range files {
-		data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // the test's
+	for _, path := range files {
+		data, err := os.ReadFile(path) //nolint:gosec // the test's
 		if err != nil {
-			t.Fatalf("read %s: %v", name, err)
+			t.Fatalf("read %s: %v", path, err)
 		}
 
 		shown = append(shown, string(data))
 	}
 
 	if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) {
-		t.Errorf("the key is shown in the output or the state files:\n%s", all)
+		t.Errorf("the key is shown in the output or the state and cache files:\n%s", all)
 	}
 }
 
@@ -1181,6 +1219,29 @@ func TestUnwritableStateDirStopsTheStart(t *testing.T) {
 		"SWW_STATE_DIR cannot be written: ")
 }
 
+func TestUnwritableCacheDirStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	ctx, stop := context.WithTimeout(t.Context(), waitLimit)
+	defer stop()
+
+	out := &output{}
+
+	status := run(ctx, map[string]string{
+		listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
+		cacheDir: filepath.Join(t.TempDir(), "missing"),
+	}, out)
+	if status != 1 {
+		t.Fatalf("exit status %d, want 1", status)
+	}
+
+	line := out.line(t, "msg", "cannot use the state and cache files")
+	if message, _ := line["error"].(string); !strings.HasPrefix(message,
+		"SWW_CACHE_DIR cannot be written: ") {
+		t.Errorf("start refused with %q, want SWW_CACHE_DIR named", message)
+	}
+}
+
 // wantStartRefused runs sww with its state files in dir, and
 // checks that it stops at start, with an error that starts with want. If
 // it starts instead, it is stopped after waitLimit.
@@ -1199,7 +1260,7 @@ func wantStartRefused(t *testing.T, dir, want string) {
 		t.Fatalf("exit status %d, want 1", status)
 	}
 
-	line := out.line(t, "msg", "cannot use the state files")
+	line := out.line(t, "msg", "cannot use the state and cache files")
 	message, _ := line["error"].(string)
 
 	if !strings.HasPrefix(message, want) {
@@ -1261,7 +1322,7 @@ func runUntilStopped(
 		exited <- run(ctx, env, out)
 	}()
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
 	use("http://" + addr + "/")
 	stop()
 

internal/sww/waf_test.go

@@ -9,10 +9,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
+	// The time zone database, so that the process TestMainRunsWAF starts
+	// finds the zone TZ names on any host.
+	_ "time/tzdata"
 
 	"sneak.berlin/go/smallwebwaf/internal/sww"
 	"sneak.berlin/go/smallwebwaf/internal/waf"
@@ -34,49 +38,116 @@ func TestMain(m *testing.M) {
 	os.Exit(sww.Main(testVersion))
 }
 
+// startProcess starts cmd and returns a channel that gets its exit status. As
+// the test ends, cmd is killed and the test waits for it to exit.
+func startProcess(t *testing.T, cmd *exec.Cmd) <-chan int {
+	t.Helper()
+
+	err := cmd.Start()
+	if err != nil {
+		t.Fatalf("start: %v", err)
+	}
+
+	exited := make(chan int, 1)
+	waitReturned := make(chan struct{})
+
+	go func() {
+		_ = cmd.Wait()
+		exited <- cmd.ProcessState.ExitCode()
+
+		close(waitReturned)
+	}()
+
+	t.Cleanup(func() {
+		_ = cmd.Process.Kill()
+
+		<-waitReturned
+	})
+
+	return exited
+}
+
 func TestMainRunsWAF(t *testing.T) {
 	t.Parallel()
 
+	// Kiritimati is 14 hours ahead of UTC; every time sww gives stays in UTC.
+	const zone = "Pacific/Kiritimati"
+
+	_, err := time.LoadLocation(zone)
+	if err != nil {
+		t.Fatalf("load %s: %v", zone, err)
+	}
+
 	binary, err := os.Executable()
 	if err != nil {
 		t.Fatalf("find the test binary: %v", err)
 	}
 
-	ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit)
-	defer cancel()
-
+	dir := t.TempDir()
 	out := &output{}
-	cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary
+	cmd := exec.CommandContext(t.Context(), binary) //nolint:gosec // this test binary
 	cmd.Stdout = out
 
 	cmd.Env = append(os.Environ(),
+		"TZ="+zone,
 		commandForMain+"="+wafCommand,
 		listenAddr+"="+localhost+":0",
-		stateDir+"="+t.TempDir(),
+		stateDir+"="+dir,
+		cacheDir+"="+dir,
 		rulesDir+"="+t.TempDir(),
 		lookupSource+"=off")
 
-	err = cmd.Start()
-	if err != nil {
-		t.Fatalf("start: %v", err)
-	}
+	exited := startProcess(t, cmd)
 
-	t.Cleanup(func() {
-		_ = cmd.Process.Kill()
-		_ = cmd.Wait()
-	})
-
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	starting := out.lineBeforeExit(t, "starting", exited)
+	addr, _ := starting["address"].(string)
 	ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
 
+	// The files are written as sww stops.
 	err = cmd.Process.Signal(syscall.SIGTERM)
 	if err != nil {
 		t.Fatalf("signal: %v", err)
 	}
 
-	err = cmd.Wait()
+	select {
+	case status := <-exited:
+		if status != 0 {
+			t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text())
+		}
+	case <-time.After(waitLimit):
+		t.Fatal("still running after being told to stop")
+	}
+
+	wantUTC(t, "the process line", starting["time"])
+	wantUTC(t, "the request line", out.line(t, "type", "request")["time"])
+
+	path := filepath.Join(dir, "clients.json")
+
+	clients, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
 	if err != nil {
-		t.Errorf("exit: %v; output:\n%s", err, out.text())
+		t.Fatalf("read clients.json: %v", err)
+	}
+
+	stamp := regexp.MustCompile(`"\d{4}-\d\d-\d\dT[^"]*"`)
+
+	times := stamp.FindAllString(string(clients), -1)
+	if len(times) == 0 {
+		t.Errorf("clients.json holds no time:\n%s", clients)
+	}
+
+	for _, stored := range times {
+		wantUTC(t, "clients.json", strings.Trim(stored, `"`))
+	}
+}
+
+// wantUTC checks that value, a time read from the log line or file that
+// where names, is in UTC.
+func wantUTC(t *testing.T, where string, value any) {
+	t.Helper()
+
+	text, _ := value.(string)
+	if !strings.HasSuffix(text, "Z") {
+		t.Errorf("%s gives the time %v, want one in UTC", where, value)
 	}
 }
 
@@ -167,7 +238,7 @@ func runWAF(t *testing.T, env map[string]string, use func(addr string)) {
 		exited <- run(ctx, env, out)
 	}()
 
-	addr, _ := out.line(t, "msg", "starting")["address"].(string)
+	addr, _ := out.lineBeforeExit(t, "starting", exited)["address"].(string)
 	use(addr)
 	stop()
 

internal/waf/answertime_test.go

@@ -1,21 +1,27 @@
 package waf_test
 
 import (
+	"encoding/binary"
 	"fmt"
 	"maps"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"testing/synctest"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/reputation"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -28,6 +34,8 @@ const (
 	answerTimeLimit = 2 * time.Second
 	// defaultLookupWait is the default of SWW_LOOKUP_TIMEOUT.
 	defaultLookupWait = 800 * time.Millisecond
+	// defaultReputationWait is the default of SWW_REPUTATION_TIMEOUT.
+	defaultReputationWait = 2 * time.Second
 	// The header limits SPEC.md gives.
 	cookieLimit      = 50
 	headerFieldLimit = 100
@@ -216,93 +224,199 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
 	}
 }
 
-func TestRequestsOverTheHeaderLimitsCountTowardTheRateLimits(t *testing.T) {
+func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
 	t.Parallel()
 
 	s := startWAF(t, "", map[string]string{rateLimitPerMinute: "1"})
-	ask := func() answer {
+
+	for range 2 {
 		req := s.question(siteA, client, "/")
 		req.Header.Set(cookieHeader, cookies(cookieLimit+1))
-
-		return do(t, req)
+		wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
 	}
 
-	wantDisposition(t, ask(), tooLarge, waf.DispositionDeny)
-	wantDisposition(t, ask(), http.StatusForbidden, waf.DispositionDeny)
-
-	lines := s.out.requestLines(t, 2)
-	wantLine(t, lines[1], http.StatusForbidden, requestlog.ActionClientIsRateLimited)
+	wantDisposition(t, s.ask(client, "/"), http.StatusOK, waf.DispositionAllow)
 }
 
-func TestTheHeaderLimitsComeAfterTheTrapPathsAndTheRuleFiles(t *testing.T) {
+func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
 	t.Parallel()
 
+	// The lookup database places client in Germany.
+	lookupDB := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, lookupDB, map[string]lookuptest.Network{
+		client + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"},
+	})
+
 	for _, tc := range []struct {
-		name, path, action string
+		name string
+		env  map[string]string
+		path string
+		// prepare gets the server ready, at now, before the question.
+		prepare func(t *testing.T, server *waf.Server, now time.Time)
 	}{
-		{"a trap path", "/wp-login.php", requestlog.ActionTemporarilyBanned},
-		{"a block rule", "/blocked", requestlog.ActionBlockedByBlockRule},
+		{"SWW_DENY_NETS", map[string]string{denyNets: client}, "/", leaveAsItIs},
+		{"a ban", nil, "/", banClient},
+		{
+			"the country list", map[string]string{
+				lookupSource: fileSource, lookupDBPath: lookupDB, allowedCountries: "FR",
+			},
+			"/", leaveAsItIs,
+		},
+		{"a blocklist", map[string]string{blocklistURLs: dropURL}, "/", listClient},
+		{
+			"the CrowdSec decision list",
+			map[string]string{crowdSecURL: lapi, crowdSecKey: bouncerKey}, "/", decideOnClient,
+		},
+		{
+			"a DNS blocklist", map[string]string{
+				dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionDeny,
+			},
+			"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
+				loadVerdicts(server, map[string][]string{client: {dnsblZone}})
+			},
+		},
+		{
+			"AbuseIPDB",
+			map[string]string{abuseIPDBKey: accountKey, reputationAction: actionDeny},
+			"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
+				// Over SWW_ABUSEIPDB_MIN_SCORE, 75 by default.
+				loadScores(server, map[string]int64{client: 80})
+			},
+		},
+		{
+			"SWW_TAG_DENY", map[string]string{tagLists: "vpn=" + vpnList, tagDeny: "vpn"},
+			"/", func(t *testing.T, server *waf.Server, _ time.Time) {
+				t.Helper()
+				loadLists(t, server, map[string][]string{vpnList: {client}})
+			},
+		},
+		{"the rate limits", map[string]string{rateLimitPerMinute: "1"}, "/", countClient},
+		{
+			"a trap path", map[string]string{trapPaths: trapPathList}, "/wp-login.php",
+			leaveAsItIs,
+		},
+		{
+			"a block rule", map[string]string{rulesDir: writeRules(t, testRules)}, "/blocked",
+			leaveAsItIs,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			s := startWAF(t, "", map[string]string{
-				trapPaths: trapPathList, rulesDir: writeRules(t, testRules),
-			})
+			s := startWAF(t, "", tc.env)
+			tc.prepare(t, s.server, s.clock.Now())
+
 			req := s.question(siteA, client, tc.path)
 			req.Header.Set(cookieHeader, cookies(cookieLimit+1))
 
-			wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
-			wantLine(t, s.out.requestLine(t), http.StatusForbidden, tc.action)
+			wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
+			wantLine(t, s.out.requestLine(t), tooLarge, requestlog.ActionRequestHeaderTooLarge)
 		})
 	}
 }
 
+func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
+	t.Parallel()
+
+	// Each check after the header limits leaves a mark on the log line of a
+	// request it reads: a tag, routewatch_tags_missing, a rule id, the
+	// counts, or the Core Rule Set's time.
+	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
+	lookuptest.Write(t, path, map[string]lookuptest.Network{
+		client + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"},
+	})
+	s := startWAF(t, "", map[string]string{
+		lookupSource: fileSource, lookupDBPath: path,
+		tagLists:      "vpn=" + vpnList,
+		routeWatchURL: startRouteWatch(t, map[string]standInRoute{client: {prefixDE, 64496}}),
+		dnsblZones:    dnsblZone, reputationAction: actionLog,
+		rulesDir: writeRules(t, "noted path log ^/\n"),
+		wafMode:  block,
+	})
+	loadLists(t, s.server, map[string][]string{vpnList: {client}})
+	loadVerdicts(s.server, map[string][]string{client: {dnsblZone}})
+
+	over := s.question(siteA, client, "/")
+	over.Header.Set(cookieHeader, cookies(cookieLimit+1))
+	wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny)
+
+	within := s.question(siteA, client, "/")
+	within.Header.Set(cookieHeader, cookies(cookieLimit))
+	wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow)
+
+	lines := s.out.requestLines(t, 2)
+
+	wantTags(t, lines[0])
+	wantTagsMissing(t, lines[0], "")
+
+	if lines[0].RuleIDs != nil || lines[0].Counts != (ratelimit.Counts{}) ||
+		lines[0].DurationCoreRuleSetMS != nil {
+		t.Errorf("rule ids %v, counts %+v and Core Rule Set time %v, want none",
+			lines[0].RuleIDs, lines[0].Counts, lines[0].DurationCoreRuleSetMS)
+	}
+
+	// The request within the limits shows that each check leaves its mark.
+	wantTags(t, lines[1],
+		"asn=64496", "country=DE", "dnsbl="+dnsblZone, "exploitsignature", "vpn")
+	wantTagsMissing(t, lines[1], "no_answer_yet")
+
+	if lines[1].RuleIDs == nil || lines[1].Counts.Minute != 1 ||
+		lines[1].DurationCoreRuleSetMS == nil {
+		t.Errorf("rule ids %v, counts %+v and Core Rule Set time %v, want each",
+			lines[1].RuleIDs, lines[1].Counts, lines[1].DurationCoreRuleSetMS)
+	}
+}
+
 func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
 	t.Parallel()
 
+	// The client's next request takes it over the whole limit, so it does
+	// not wait.
 	geojsURL, _ := startGeoJS(t)
 	s := startWAF(t, geojsURL, map[string]string{
-		unknownLimitPercent: "0", lookupTimeout: "1h",
+		unknownLimitPercent: "0", lookupTimeout: "1h", rateLimitPerMinute: "1",
 	})
-	req := s.question(siteA, fromDE, "/")
-	req.Header.Set(cookieHeader, cookies(cookieLimit+1))
+	countClient(t, s.server, s.clock.Now())
 
-	wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
+	wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden, waf.DispositionDeny)
 
 	line := s.out.requestLine(t)
-	wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
+	wantLine(t, line, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
 
 	if line.RateLimitPercent != nil {
 		t.Errorf("limit percentage %d, want none", *line.RateLimitPercent)
 	}
-
-	// The client is under no ban, and GeoJS places it in a country.
-	wantDisposition(t, s.ask(fromDE, "/"), http.StatusOK, waf.DispositionAllow)
 }
 
 func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(t *testing.T) {
 	t.Parallel()
 
+	// The client's next request takes it over the whole limit, so it does
+	// not wait.
 	geojsURL, _ := startGeoJS(t)
-	s := startWAF(t, geojsURL, map[string]string{unknownLimitPercent: "0"})
+	s := startWAF(t, geojsURL, map[string]string{
+		unknownLimitPercent: "0", rateLimitPerMinute: "1",
+	})
 	s.server.GeoJS.Load([]lookup.Answer{{
 		Client: netip.MustParsePrefix(client + "/32"), ASN: "AS64512",
 		Answered: s.clock.Now(), Used: s.clock.Now(),
 	}})
+	countClient(t, s.server, s.clock.Now())
 
-	req := s.question(siteA, client, "/")
-	req.Header.Set(cookieHeader, cookies(cookieLimit+1))
+	wantDisposition(t, s.ask(client, "/"), http.StatusForbidden, waf.DispositionDeny)
 
-	wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
-	wantLine(t, s.out.requestLine(t), http.StatusForbidden,
-		requestlog.ActionClientIsRateLimited)
+	line := s.out.requestLine(t)
+	wantLine(t, line, http.StatusForbidden, requestlog.ActionClientIsRateLimited)
+
+	if line.RateLimitPercent == nil || *line.RateLimitPercent != 0 {
+		t.Errorf("limit percentage %v, want 0", line.RateLimitPercent)
+	}
 }
 
-func TestObserveModeWaitsForGeoJSForARequestOverTheHeaderLimits(t *testing.T) {
+func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(t *testing.T) {
 	t.Parallel()
 
-	geojsURL, _ := startGeoJS(t)
+	geojsURL, asked := startGeoJS(t)
 	s := startWAF(t, geojsURL, map[string]string{
 		mode: observe, addLookupHeaders: on, lookupTimeout: "1h",
 	})
@@ -312,10 +426,9 @@ func TestObserveModeWaitsForGeoJSForARequestOverTheHeaderLimits(t *testing.T) {
 	got := do(t, req)
 	wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
 
-	asn, country := got.header.Get("X-Client-Asn"), got.header.Get("X-Client-Country")
-	if asn != asnDE || country != "DE" {
-		t.Errorf("answered with the AS number %q and the country %q, want %q and DE",
-			asn, country, asnDE)
+	if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
+		t.Errorf("GeoJS was asked about %v, and the answer gave the AS number %q, "+
+			"want neither", asked(), got.header.Get("X-Client-Asn"))
 	}
 }
 
@@ -521,15 +634,9 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
 ) {
 	t.Parallel()
 
-	held, askedGeoJS, _ := startHeldGeoJS(t)
-
-	// A resolver that takes every query and answers none.
-	resolver, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", localhost+":0")
-	if err != nil {
-		t.Fatalf("listen: %v", err)
-	}
-
-	t.Cleanup(func() { _ = resolver.Close() })
+	// The lists' stand-in: one for GeoJS that answers nothing.
+	held, _, _ := startHeldGeoJS(t)
+	resolver, releaseResolver := startHeldResolver(t)
 
 	server, out, _ := newWAFWith(t, func(
 		params waf.Params,
@@ -537,42 +644,117 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
 		params.AbuseIPDBURL = silentAbuseIPDBURL
 
 		return waf.New(params)
-	}, held, time.Now, map[string]string{
+	}, silentGeoJSURL, time.Now, map[string]string{
 		tagDeny: "country=KP", wafMode: block,
-		dnsblZones: dnsblZone, dnsblResolver: resolver.LocalAddr().String(),
+		dnsblZones: dnsblZone, dnsblResolver: resolver,
 		abuseIPDBKey:  accountKey,
 		blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
 		crowdSecURL: held, crowdSecKey: bouncerKey,
 	})
 
-	// AbuseIPDB is asked only about a client with an offence.
-	group := netip.MustParsePrefix(client + "/32")
-	server.Limiter.AddToHistory(group, time.Now(), ratelimit.Request{Attack: true})
-
+	// The lists are fetched outside the synctest bubble below: a wait on the
+	// network inside it would keep its clock from moving on.
 	go server.Lists.Run(t.Context())
 
-	status, took := askDirectly(t, server, func(req *http.Request) {
-		req.Header.Set("User-Agent", "test")
-		req.Header.Set("Accept", "*/*")
-		req.Host = siteA
+	// In the bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays says, the
+	// answer takes exactly as long as waf waits, however busy the host is.
+	// GeoJS and AbuseIPDB never answer there, without the network. The query to
+	// the DNS blocklist waits on the network, which stops the bubble's clock,
+	// but it starts only after the wait for GeoJS.
+	synctest.Test(t, func(t *testing.T) {
+		// AbuseIPDB is asked only about a client with an offence.
+		group := netip.MustParsePrefix(client + "/32")
+		server.Limiter.AddToHistory(group, time.Now(), ratelimit.Request{Attack: true})
+
+		status, took := askDirectly(t, server, func(req *http.Request) {
+			req.Header.Set("User-Agent", "test")
+			req.Header.Set("Accept", "*/*")
+			req.Host = siteA
+		})
+
+		// The wait for GeoJS decides when the answer comes.
+		if status != http.StatusOK || took != defaultLookupWait || took >= answerTimeLimit {
+			t.Errorf("answered %d after %s, want %d after %s, within %s; output:\n%s",
+				status, took, http.StatusOK, defaultLookupWait, answerTimeLimit, out.text())
+		}
+
+		// Each outside source was asked, and none has answered. GeoJS is asked
+		// only about the clients waiting for it, here client alone.
+		if requests := testutil.ToFloat64(server.Metrics.GeoJSRequests); requests != 1 {
+			t.Errorf("%v requests to GeoJS, want 1", requests)
+		}
+
+		// The resolver holds the query until it is released below, so a waf
+		// that waited for the query would answer only once it had failed.
+		queries, failures := server.DNSBL.Queries(dnsblZone), server.DNSBL.Failures(dnsblZone)
+		if queries != 1 || failures != 0 {
+			t.Errorf("%d queries to the DNS blocklist, %d failed, want 1, none failed",
+				queries, failures)
+		}
+
+		if checked := server.AbuseIPDB.Checked(); checked != 1 {
+			t.Errorf("%d checks with AbuseIPDB, want 1", checked)
+		}
+
+		// The query is answered, and the check with AbuseIPDB abandoned,
+		// before the bubble ends.
+		releaseResolver()
+		time.Sleep(defaultReputationWait)
 	})
+}
 
-	if status != http.StatusOK || took >= answerTimeLimit {
-		t.Errorf("answered %d after %s, want %d within %s; output:\n%s",
-			status, took, http.StatusOK, answerTimeLimit, out.text())
+// How the stand-in for the resolver reads a query and makes it its reply.
+const (
+	// maxQueryLength is the longest query it reads.
+	maxQueryLength = 512
+	// flagsAt is where a DNS message's flags start, after its id.
+	flagsAt = 2
+	// noSuchNameFlags mark a reply, to a query that asked for recursion,
+	// which is available, that no such name exists.
+	noSuchNameFlags = 0x8183
+)
+
+// startHeldResolver starts a stand-in for the resolver the DNSBL zones are
+// asked through, and returns its address and release. Each query waits until
+// release is called, and is then answered that no such name exists, so that
+// the zone does not list the client.
+func startHeldResolver(t *testing.T) (string, func()) {
+	t.Helper()
+
+	conn, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", localhost+":0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
 	}
+
+	t.Cleanup(func() { _ = conn.Close() })
+
+	var (
+		released = make(chan struct{})
+		once     sync.Once
+	)
+
+	go func() {
+		query := make([]byte, maxQueryLength)
+
+		for {
+			n, from, err := conn.ReadFrom(query)
+			if err != nil {
+				return // conn is closed
+			}
 
-	// Each outside source was asked, and none has answered. The stand-in for
-	// GeoJS is the lists' too, which name no address.
-	if !slices.Contains(askedGeoJS(), client) {
-		t.Errorf("GeoJS was asked about %q, not %s", askedGeoJS(), client)
-	}
+			<-released
+
+			// The query back, with the flags of the reply in place of its own.
+			reply := slices.Clone(query[:n])
+			binary.BigEndian.PutUint16(reply[flagsAt:], noSuchNameFlags)
+
+			_, _ = conn.WriteTo(reply, from)
+		}
+	}()
 
-	if queries := server.DNSBL.Queries(dnsblZone); queries != 1 {
-		t.Errorf("%d queries to the DNS blocklist, want 1", queries)
-	}
+	release := func() { once.Do(func() { close(released) }) }
+	// Run before conn is closed, so that no query is left waiting.
+	t.Cleanup(release)
 
-	if checked := server.AbuseIPDB.Checked(); checked != 1 {
-		t.Errorf("%d checks with AbuseIPDB, want 1", checked)
-	}
+	return conn.LocalAddr().String(), release
 }

internal/waf/coreruleset_test.go

@@ -12,7 +12,6 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
-	"sneak.berlin/go/smallwebwaf/internal/waf"
 )
 
 // The Core Rule Set's settings the tests set, besides SWW_WAF_MODE, and
@@ -22,7 +21,6 @@ const (
 	wafDisabledRules    = "SWW_WAF_DISABLED_RULES"
 	wafExemptPaths      = "SWW_WAF_EXEMPT_PATHS"
 	wafBodyLimit        = "SWW_WAF_BODY_LIMIT"
-	apps                = "SWW_APPS"
 	block               = "block"
 	detect              = "detect"
 )
@@ -292,27 +290,6 @@ func TestDisabledRulesSwitchOffWhatGiteaWouldBeRefused(t *testing.T) {
 	}
 }
 
-func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) {
-	t.Parallel()
-
-	// Only gitea's fixes let a file name through in path, a parameter
-	// gitea sends file names in.
-	const fileName = "/?path=.gitignore"
-
-	s := startWAF(t, "", map[string]string{
-		wafMode: block, apps: siteA + ":gitea",
-	})
-
-	wantDisposition(t, do(t, s.question(siteA, client, fileName)),
-		http.StatusOK, waf.DispositionAllow)
-	wantDisposition(t, do(t, s.question(siteB, client, fileName)),
-		http.StatusForbidden, waf.DispositionDeny)
-
-	lines := s.out.requestLines(t, 2)
-	wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
-	wantLine(t, lines[1], http.StatusForbidden, requestlog.ActionBlockedByCoreRuleSet)
-}
-
 func TestAttackInAFormBodyIsRefusedOnlyWhileBodiesAreRead(t *testing.T) {
 	t.Parallel()
 

internal/waf/crowdsec_test.go

@@ -16,7 +16,7 @@ import (
 
 // The CrowdSec settings, and the tests' engine, which is never asked: each
 // test puts in the copy of its decision list, at decisionsURL, that it
-// needs, as reputation.json would at start.
+// needs, as lists.json would at start.
 const (
 	crowdSecURL  = "SWW_CROWDSEC_LAPI_URL"
 	crowdSecKey  = "SWW_CROWDSEC_LAPI_KEY"
@@ -182,7 +182,7 @@ func TestObserveModeForwardsAClientTheCrowdSecDecisionListListsAndAlertsTheBan(
 
 // loadDecisions puts into server's lists the copy of the decision list at
 // decisionsURL, answer, the engine's answer, fetched at fetched, as
-// reputation.json would at start.
+// lists.json would at start.
 func loadDecisions(
 	t *testing.T, server *waf.Server, fetched time.Time, answer string,
 ) {

internal/waf/headerlimits.go

@@ -10,22 +10,23 @@ import (
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
-// The header limits: the most cookies and header fields a request may have
-// when the checks reach the Core Rule Set. Real clients never reach them.
+// The header limits: the most cookies and header fields a request may have.
+// Real clients never reach them.
 const (
 	maxCookies      = 50
 	maxHeaderFields = 100
 )
 
-// headerLimits refuses a request over the header limits, with 431. Its figures
-// are the request's header fields and cookies, and the limit of each.
+// headerLimits refuses a request with more than maxHeaderFields header fields
+// or more than maxCookies cookies, with 431. Its figures are the request's
+// header fields and cookies, and the limit of each.
 type headerLimits struct{}
 
 func (headerLimits) name() string { return "header_limits" }
 
 func (headerLimits) check(_ context.Context, rq *request, _ time.Time) *verdict {
 	fields, cookies := rq.headerCounts()
-	if !overHeaderLimits(fields, cookies) {
+	if fields <= maxHeaderFields && cookies <= maxCookies {
 		return nil
 	}
 
@@ -38,13 +39,6 @@ func (headerLimits) check(_ context.Context, rq *request, _ time.Time) *verdict
 	}
 }
 
-// overHeaderLimits reports whether a request with these counts of header fields
-// and cookies, as headerCounts counts them, is over the header limits: it has
-// more than maxHeaderFields fields or more than maxCookies cookies.
-func overHeaderLimits(fields, cookies int) bool {
-	return fields > maxHeaderFields || cookies > maxCookies
-}
-
 // headerCounts returns the request's header fields, each value of a header sent
 // twice being one, and its cookies, each part of a Cookie header between
 // semicolons being one unless it is empty or only spaces and tabs. The Core

internal/waf/heuristics.go

@@ -41,8 +41,11 @@ const (
 
 // beforeLookup are the heuristics that run before the client's AS number and
 // country are looked up, so that a client they refuse is not looked up.
+// header_limits comes first: it only counts, and a request it refuses causes no
+// other work.
 func beforeLookup() []heuristic {
 	return []heuristic{
+		headerLimits{},
 		tagLists{},
 		routeWatchTags{},
 		denyNets{},
@@ -64,7 +67,6 @@ func afterLookup() []heuristic {
 		rateLimits{},
 		trapPaths{},
 		ruleFiles{},
-		headerLimits{},
 		coreRuleSet{},
 		tagDeny{},
 	}

internal/waf/proxy.go

@@ -86,8 +86,8 @@ type Params struct {
 // Server is the server waf runs, with the parts whose state the state
 // files keep, the lookup database, nil unless SWW_LOOKUP_SOURCE is file, the
 // lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
-// AbuseIPDB's scores and checks spent, and the metrics. RouteWatch, nil while
-// SWW_ROUTEWATCH_URL is unset, keeps its answers in memory only.
+// AbuseIPDB's scores and checks spent, routewatch's answers, and the metrics.
+// RouteWatch is nil while SWW_ROUTEWATCH_URL is unset.
 type Server struct {
 	*http.Server
 
@@ -221,8 +221,7 @@ func newReputation(
 // NewCoreRuleSet returns the Core Rule Set at SWW_WAF_PARANOIA_LEVEL,
 // without the rules SWW_WAF_DISABLED_RULES switches off, reading bodies
 // up to SWW_WAF_BODY_LIMIT, with the exclusions in
-// SWW_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_APPS,
-// or nil while SWW_WAF_MODE is off. It
+// SWW_CORE_RULE_SET_EXCLUSIONS_FILE, or nil while SWW_WAF_MODE is off. It
 // reads at most QuestionBodyMaxBytes less one byte: a question's body of
 // QuestionBodyMaxBytes may be cut short, and the Core Rule Set does not read a
 // JSON or XML body over its limit. An exclusions file that coreruleset.New refuses is
@@ -237,7 +236,6 @@ func NewCoreRuleSet(cfg *config.Config) (*coreruleset.CoreRuleSet, error) {
 		ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules,
 		BodyLimit:      min(cfg.WAFBodyLimit, QuestionBodyMaxBytes-1),
 		ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
-		Apps:           cfg.Apps,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("SWW_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err)

internal/waf/reputation_test.go

@@ -39,7 +39,7 @@ const (
 )
 
 // The lists these tests name, which are never fetched: each test puts in
-// the copies it needs, as reputation.json would at start.
+// the copies it needs, as lists.json would at start.
 const (
 	dropURL = "https://lists.example/drop.txt"
 	torURL  = "https://lists.example/tor.txt"
@@ -374,7 +374,7 @@ const (
 
 // The DNSBL zones these tests name, which are never asked about the
 // clients the tests send requests from: each test puts in the verdicts it
-// needs, as reputation.json would at start. A query a test does start is
+// needs, as dnsbl.json would at start. A query a test does start is
 // sent to noResolver, where nothing listens, so that none leaves the host.
 const (
 	dnsblZone  = "dnsbl.example"
@@ -759,7 +759,7 @@ func TestOnlyAClientThatHasCommittedAnOffenceIsCheckedWithAbuseIPDB(t *testing.T
 	clk.advance(time.Hour)
 	wantReputation(t, s.get(fromDE, http.StatusOK, forward))
 	wantAbuseIPDBChecks(t, server, 1)
-	waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot().Scores) == 1 })
+	waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot()) == 1 })
 	wantReputation(t, s.get(fromDE, http.StatusOK, forward), abuseipdb)
 
 	s.get(fromKP, http.StatusOK, forward)
@@ -1072,7 +1072,7 @@ func verdictsFetched() time.Time {
 
 // loadVerdicts puts into server's DNSBL, for each client listedBy names,
 // a verdict of each zone SWW_DNSBL_ZONES names, fetched at
-// verdictsFetched, as reputation.json would at start: one that lists the
+// verdictsFetched, as dnsbl.json would at start: one that lists the
 // client from each zone listedBy gives for it, and one that does not from
 // each other zone.
 func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
@@ -1091,8 +1091,8 @@ func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
 }
 
 // loadScores puts into server's AbuseIPDB the score scores gives each
-// client, an IPv4 address, fetched at verdictsFetched, as reputation.json
-// would at start.
+// client, an IPv4 address, fetched at verdictsFetched, as
+// abuseipdb-scores.json would at start.
 func loadScores(server *waf.Server, scores map[string]int64) {
 	kept := make([]reputation.Score, 0, len(scores))
 	for client, score := range scores {
@@ -1102,7 +1102,7 @@ func loadScores(server *waf.Server, scores map[string]int64) {
 		})
 	}
 
-	server.AbuseIPDB.Load(reputation.Checks{Scores: kept})
+	server.AbuseIPDB.Load(kept)
 }
 
 // wantAbuseIPDBChecks checks how many clients server has checked with
@@ -1116,7 +1116,7 @@ func wantAbuseIPDBChecks(t *testing.T, server *waf.Server, want int) {
 }
 
 // loadLists puts copies of lists into server's lists, by URL, each with
-// its lines, fetched at listsFetched, as reputation.json would at start.
+// its lines, fetched at listsFetched, as lists.json would at start.
 func loadLists(t *testing.T, server *waf.Server, copies map[string][]string) {
 	t.Helper()
 

internal/waf/waf.go

@@ -143,10 +143,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // refusedWhateverTheAnswer reports whether the request is one that does not wait
 // for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
-// its client: the heuristic crowdsec, header_limits, blocklists or rate_limits
-// will refuse it, as a function in that heuristic's file says. In observe mode,
-// which refuses nothing, none is. rproxy lets a request through when
-// waf answers too late.
+// its client: the heuristic crowdsec, blocklists or rate_limits will refuse it,
+// as a function in that heuristic's file says. In observe mode, which refuses
+// nothing, none is. rproxy lets a request through when waf answers too late.
 func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
 	if rq.h.config.Observe {
 		return false
@@ -154,8 +153,7 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
 
 	_, decided := rq.crowdSecDecisionInForce(now)
 
-	return decided || overHeaderLimits(rq.headerCounts()) ||
-		rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
+	return decided || rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
 		rq.overAWholeRateLimit(now)
 }
 
@@ -215,10 +213,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
 
 // readQuestion returns the visitor's address and the body a question carries,
 // and whether waf takes the question. It answers a question it does not
-// take with 400 and no disposition, which rproxy treats as it treats no
-// answer. The id of a question it takes is held from here on, without a
-// decision yet, so that another question with that id is refused while this
-// one is being handled.
+// take with 400 and no disposition. rproxy passes that 400 to the visitor,
+// and SWRP_DEFAULT_DISPOSITION does not apply. The id of a question waf takes
+// is held from here on, without a decision yet, so that another question with
+// that id is refused while this one is being handled.
 //
 // The body is read before any check, since Go's server does not send an answer
 // until it has read the body. A question whose body stops arriving gets no

script/example-app

@@ -9,6 +9,7 @@
 #   - that a probe for /.env bans another client, which its next request bans
 #     for good;
 #   - that only waf's container changes the state directory's owner;
+#   - that waf's container makes the cache directory for the sww user;
 #   - that the app still answers through rproxy with waf stopped, and
 #     that once waf has started again, bans apply again;
 #   - that `sv stop` stops rproxy in order, and `docker stop` stops each
@@ -17,8 +18,8 @@
 #     client;
 #   - that a waf container with its state files on a new volume mounted at
 #     the directory SWW_STATE_DIR_FILE names becomes healthy;
-#   - that one whose SWW_STATE_DIR does not exist logs waf's message
-#     about it.
+#   - that the run script makes a SWW_STATE_DIR that does not exist, owned
+#     by sww with mode 0700, and that waf then becomes healthy.
 # waf runs with SWW_LOOKUP_SOURCE=off, so that no address is sent to
 # GeoJS. The containers, the network, the volumes and both images are removed
 # however the script ends. Building the app needs network access, for nixpkgs'
@@ -153,9 +154,15 @@ refused_from() {
     [ "$(status_from "$1" "$2")" = 403 ]
 }
 
-# owner_of_state_dir <container>: print who owns /var/lib/sww there.
-owner_of_state_dir() {
-    docker exec "$1" stat --format '%U' /var/lib/sww
+# owner_of <container> <dir>: print who owns dir there.
+owner_of() {
+    docker exec "$1" stat --format '%U' "$2"
+}
+
+# owner_and_mode_of <container> <dir>: print who owns dir there, and its
+# mode in octal.
+owner_and_mode_of() {
+    docker exec "$1" stat --format '%U %a' "$2"
 }
 
 main() {
@@ -173,11 +180,14 @@ main() {
     start_app
     echo "example-app: the health checks pass"
 
-    [ "$(owner_of_state_dir "$WAF")" = sww ] ||
+    [ "$(owner_of "$WAF" /var/lib/sww)" = sww ] ||
         fail "waf's container did not give the state directory to sww"
-    [ "$(owner_of_state_dir "$APP")" = root ] ||
+    [ "$(owner_of "$APP" /var/lib/sww)" = root ] ||
         fail "the app's container changed the state directory's owner"
     echo "example-app: only waf's container changes the state directory's owner"
+    [ "$(owner_of "$WAF" /var/cache/sww)" = sww ] ||
+        fail "waf's container did not make the cache directory for sww"
+    echo "example-app: waf's container makes the cache directory for sww"
 
     page="$(curl --fail --silent --show-error --max-time 10 "http://$address/")" ||
         fail "no answer on port 8080"
@@ -247,9 +257,11 @@ main() {
         --env SWW_STATE_DIR=/srv/missing \
         --env SWW_LOOKUP_SOURCE=off \
         "$IMAGE" >/dev/null
-    wait_for "waf did not report the missing state directory" \
-        logged "$WAF" "SWW_STATE_DIR cannot be written"
-    echo "example-app: waf reports a state directory that does not exist"
+    wait_for "waf's health check did not pass" healthy "$WAF"
+    [ "$(owner_and_mode_of "$WAF" /srv/missing)" = "sww 700" ] ||
+        fail "the run script did not make SWW_STATE_DIR for sww with mode 0700"
+    echo "example-app: the run script makes a missing SWW_STATE_DIR for sww" \
+        "with mode 0700"
 }
 
 main "$@"

script/run

@@ -6,8 +6,9 @@
 # waf listens on port 8090 unless SWW_LISTEN_ADDR says otherwise, and
 # rproxy asks it there unless SWRP_WAF_BACKEND says otherwise. Unless
 # SWW_STATE_DIR is set, the state files go in bin/state, beside the binary,
-# and unless SWW_RULES_DIR is set, the rule files are those of share/rules.d,
-# which the image ships.
+# and unless SWW_CACHE_DIR is set, the cache files go in bin/cache. Unless
+# SWW_RULES_DIR is set, the rule files are those of share/rules.d, which the
+# image ships.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -31,6 +32,11 @@ main() {
         export SWW_STATE_DIR
         mkdir -p "$SWW_STATE_DIR"
     fi
+    if [ -z "${SWW_CACHE_DIR+set}" ]; then
+        SWW_CACHE_DIR="$ROOT/bin/cache"
+        export SWW_CACHE_DIR
+        mkdir -p "$SWW_CACHE_DIR"
+    fi
     if [ -z "${SWW_RULES_DIR+set}" ]; then
         SWW_RULES_DIR="$ROOT/share/rules.d"
         export SWW_RULES_DIR

share/sww.run

@@ -11,7 +11,10 @@ main() {
     local command="${SWW_COMMAND:-rproxy}"
     case "$command" in
         rproxy) ;;
-        waf) give_state_dir_to_sww ;;
+        waf)
+            make_state_dir_for_sww
+            make_cache_dir_for_sww
+            ;;
         *)
             printf 'SWW_COMMAND: "%s" is not rproxy or waf\n' \
                 "$command" >&2
@@ -21,29 +24,43 @@ main() {
     exec chpst -u sww:sww /usr/local/bin/sww "$command"
 }
 
-# give_state_dir_to_sww gives the state directory and every file in
-# it to the sww user, so that a volume mounted there needs no change
-# of owner; chown -R changes a symbolic link itself, never what it points
-# to. When SWW_STATE_DIR, in either form, cannot be read or is not the
-# absolute path of a directory, no owner is changed, and waf starts and
-# stops with a message naming the setting. A change of owner that fails, as
-# on a volume mounted read-only, does not end the script either: waf
-# reports a directory it cannot write.
-give_state_dir_to_sww() {
+# make_state_dir_for_sww makes the state directory, SWW_STATE_DIR in
+# either form, when it is missing, gives it and every file in it to the
+# sww user, and sets its mode to 0700. So a volume mounted there needs no
+# change of owner; chown -R changes a symbolic link itself, never what it
+# points to. When the setting cannot be read or is not an absolute path,
+# nothing is changed, and waf starts and stops with a message naming the
+# setting. A step that fails, as on a volume mounted read-only, does not
+# end the script either: waf reports a directory it cannot write.
+make_state_dir_for_sww() {
     local dir
-    if dir="$(state_dir)" && [[ "$dir" == /* && -d "$dir" ]]; then
-        chown -R sww:sww "$dir" || true
+    if dir="$(setting SWW_STATE_DIR /var/lib/sww)" && [[ "$dir" == /* ]]; then
+        { mkdir -p -- "$dir" && chown -R sww:sww "$dir" &&
+            chmod 0700 "$dir"; } || true
     fi
 }
 
-# state_dir prints SWW_STATE_DIR, /var/lib/sww when it is unset,
-# or what is in the file SWW_STATE_DIR_FILE names, less every newline at
-# its end. It fails when both are set or the file cannot be read.
-state_dir() {
-    if [[ ! -v SWW_STATE_DIR_FILE ]]; then
-        printf '%s\n' "${SWW_STATE_DIR-/var/lib/sww}"
-    elif [[ ! -v SWW_STATE_DIR ]]; then
-        cat -- "$SWW_STATE_DIR_FILE" 2>/dev/null
+# make_cache_dir_for_sww makes the cache directory, SWW_CACHE_DIR in
+# either form, when it is missing, and gives it and every file in it to
+# the sww user. As for the state directory, a setting that cannot be read
+# or is not an absolute path changes nothing, and a failure does not end
+# the script.
+make_cache_dir_for_sww() {
+    local dir
+    if dir="$(setting SWW_CACHE_DIR /var/cache/sww)" && [[ "$dir" == /* ]]; then
+        { mkdir -p -- "$dir" && chown -R sww:sww "$dir"; } || true
+    fi
+}
+
+# setting prints the setting named $1, $2 when it is unset, or what is in
+# the file the setting $1_FILE names, less every newline at its end. It
+# fails when both are set or the file cannot be read.
+setting() {
+    local name="$1" file_name="$1_FILE"
+    if [[ ! -v "$file_name" ]]; then
+        printf '%s\n' "${!name-$2}"
+    elif [[ ! -v "$name" ]]; then
+        cat -- "${!file_name}" 2>/dev/null
     else
         return 1
     fi