rproxy and servewaf replace the one-process smallwebwaf (closes #172, closes #165)

`smallwebwaf` with no command now prints the commands and exits with status 2.
The image runs rproxy, or servewaf with SMALLWEBWAF_COMMAND=servewaf. The
passing-on code, its settings and its log fields and metric go. internal/proxy
keeps the checks, answers, reports and own endpoints; its tests run through the
real rproxy. The documents describe the pair.

Judgement call: clients.json keeps token_refused, since state files refuse unknown fields.
Deviation: script/gitea-requests sets SWWAF_LOG_REQUEST_HEADERS, to label requests.
Partially verified: make example-app and make gitea-requests never built the image, as Ubuntu's snapshot service answered 5xx.

Model: opus-5-5

script/example-app

@@ -1,19 +1,28 @@
 #!/bin/sh
 # script/example-app: build the image, and on it the example app in
-# deploy/example-app, then run the app's container with a volume for the
-# state files and check that the health check passes, that a request is
-# served through smallwebwaf, that a second one in a minute bans the
-# client, that a probe for /.env bans another client, which its next
-# request bans for good, that `sv stop` stops smallwebwaf in order, that
-# `docker stop` stops the container without having to kill it, that a
-# new container on the same volume still refuses the banned client, that
-# a container with its state files on a new volume mounted at the
-# directory SWWAF_STATE_DIR_FILE names becomes healthy, and that one whose
-# SWWAF_STATE_DIR does not exist logs smallwebwaf's message about it. The
-# containers run with SWWAF_LOOKUP_SOURCE=off, so that no address is sent
-# to GeoJS. The containers, the volumes and both images are removed
-# however the script ends. Building the app needs network access, for
-# nixpkgs' binary cache. script/check does not run this.
+# deploy/example-app, then run the app's container, with rproxy in it, and
+# servewaf in a container of its own with a volume for the state files. It
+# checks:
+#   - that the health checks pass;
+#   - that a request is served through rproxy, and a second one in a minute
+#     bans the client;
+#   - that a probe for /.env bans another client, which its next request bans
+#     for good;
+#   - that only servewaf's container changes the state directory's owner;
+#   - that the app still answers through rproxy with servewaf stopped, and
+#     that once servewaf has started again, bans apply again;
+#   - that `sv stop` stops rproxy in order, and `docker stop` stops each
+#     container without having to kill it;
+#   - that a new servewaf container on the same volume still refuses the banned
+#     client;
+#   - that a servewaf container with its state files on a new volume mounted at
+#     the directory SWWAF_STATE_DIR_FILE names becomes healthy;
+#   - that one whose SWWAF_STATE_DIR does not exist logs servewaf's message
+#     about it.
+# servewaf runs with SWWAF_LOOKUP_SOURCE=off, so that no address is sent to
+# GeoJS. The containers, the network, the volumes and both images are removed
+# however the script ends. Building the app needs network access, for nixpkgs'
+# binary cache. script/check does not run this.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -24,7 +33,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 NAME="$("$SCRIPT_DIR/projectname")-example-$$"
 IMAGE="$NAME-base"
 APP_IMAGE="$NAME-app"
-CONTAINER="$NAME"
+NETWORK="$NAME"
+APP="$NAME-app-container"
+SERVEWAF="$NAME-servewaf"
 VOLUME="$NAME-state"
 # The volumes of the container whose state directory SWWAF_STATE_DIR_FILE
 # names: its state files, and the file that names their directory.
@@ -32,15 +43,18 @@ MOVED_VOLUME="$NAME-moved-state"
 SETTINGS_VOLUME="$NAME-settings"
 
 cleanup() {
-    docker rm --force "$CONTAINER" >/dev/null 2>&1 || true
+    docker rm --force "$APP" "$SERVEWAF" >/dev/null 2>&1 || true
+    docker network rm "$NETWORK" >/dev/null 2>&1 || true
     docker volume rm --force "$VOLUME" "$MOVED_VOLUME" "$SETTINGS_VOLUME" \
         >/dev/null 2>&1 || true
     docker rmi --force "$APP_IMAGE" "$IMAGE" >/dev/null 2>&1 || true
 }
 
 fail() {
-    echo "example-app: $*; the container's output:" >&2
-    docker logs "$CONTAINER" >&2 || true
+    echo "example-app: $*; the app container's output:" >&2
+    docker logs "$APP" >&2 || true
+    echo "example-app: servewaf's output:" >&2
+    docker logs "$SERVEWAF" >&2 || true
     exit 1
 }
 
@@ -57,52 +71,73 @@ wait_for() {
     done
 }
 
+# healthy <container>: docker says the container's health check passes.
 healthy() {
-    status="$(docker inspect --format '{{.State.Health.Status}}' "$CONTAINER")"
+    status="$(docker inspect --format '{{.State.Health.Status}}' "$1")"
     [ "$status" = healthy ]
 }
 
-# logged <text>...: a line of the container's output holds every text,
-# in any order.
+# logged <container> <text>...: a line of the container's output holds every
+# text, in any order.
 logged() {
-    lines="$(docker logs "$CONTAINER" 2>&1)"
+    lines="$(docker logs "$1" 2>&1)"
+    shift
     for text in "$@"; do
         lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1
     done
 }
 
-# start_container: run the app's container, with the state files on the
-# volume, a rate limit of one request a minute and no client looked up,
-# and wait until it is healthy.
-start_container() {
-    docker run --detach --name "$CONTAINER" --publish 127.0.0.1::8080 \
+# start_servewaf: run servewaf from the image, with the state files on the
+# volume, a rate limit of one request a minute and no client looked up, and wait
+# until it is healthy.
+start_servewaf() {
+    docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
         --volume "$VOLUME:/var/lib/smallwebwaf" \
+        --env SMALLWEBWAF_COMMAND=servewaf \
         --env SWWAF_RATE_LIMIT_PER_MINUTE=1 \
         --env SWWAF_LOOKUP_SOURCE=off \
-        "$APP_IMAGE" >/dev/null
-    wait_for "the health check did not pass" healthy
-    address="$(docker port "$CONTAINER" 8080/tcp)"
+        "$IMAGE" >/dev/null
+    wait_for "servewaf's health check did not pass" healthy "$SERVEWAF"
 }
 
-# start_moved_container: run the app's container with the state files on
-# a new volume at /srv/state, which SWWAF_STATE_DIR_FILE names in a file
-# only the smallwebwaf user can read, as under "Settings given as files"
-# in docs/settings.md, and wait until it is healthy. smallwebwaf can write the
-# new volume only once the run script has given it to that user.
-start_moved_container() {
+# start_app: run the app's container, with rproxy in it, and wait until it is
+# healthy.
+start_app() {
+    docker run --detach --name "$APP" --network "$NETWORK" --publish 127.0.0.1::8080 \
+        --env SWRP_WAF_BACKEND="http://$SERVEWAF:8080" \
+        "$APP_IMAGE" >/dev/null
+    wait_for "the app's health check did not pass" healthy "$APP"
+    address="$(docker port "$APP" 8080/tcp)"
+}
+
+# start_moved_servewaf: run servewaf with the state files on a new volume at
+# /srv/state, which SWWAF_STATE_DIR_FILE names in a file only the smallwebwaf
+# user can read, as under "Settings given as files" in docs/settings.md, and
+# wait until it is healthy. servewaf can write the new volume only once the run
+# script has given it to that user.
+start_moved_servewaf() {
     docker volume create "$MOVED_VOLUME" >/dev/null
     docker volume create "$SETTINGS_VOLUME" >/dev/null
     docker run --rm --volume "$SETTINGS_VOLUME:/settings" \
         --entrypoint bash "$IMAGE" -c 'echo /srv/state > /settings/state-dir &&
             chown smallwebwaf:smallwebwaf /settings/state-dir &&
             chmod 0400 /settings/state-dir'
-    docker run --detach --name "$CONTAINER" \
+    docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
         --volume "$MOVED_VOLUME:/srv/state" \
         --volume "$SETTINGS_VOLUME:/etc/smallwebwaf/settings:ro" \
+        --env SMALLWEBWAF_COMMAND=servewaf \
         --env SWWAF_STATE_DIR_FILE=/etc/smallwebwaf/settings/state-dir \
         --env SWWAF_LOOKUP_SOURCE=off \
-        "$APP_IMAGE" >/dev/null
-    wait_for "the health check did not pass" healthy
+        "$IMAGE" >/dev/null
+    wait_for "servewaf's health check did not pass" healthy "$SERVEWAF"
+}
+
+# status_from <client> <path>: print the status a request for path from client,
+# as X-Forwarded-For names it, gets. rproxy believes the header from docker's
+# gateway, a private address.
+status_from() {
+    curl --silent --output /dev/null --write-out '%{http_code}' \
+        --max-time 10 --header "X-Forwarded-For: $1" "http://$address$2" || true
 }
 
 # refused: a request to the container gets 403, SWWAF_BAN_RESPONSE's
@@ -113,13 +148,14 @@ refused() {
     [ "$code" = 403 ]
 }
 
-# refused_from <client> <path>: a request for path from client, as
-# X-Forwarded-For names it, gets 403. smallwebwaf believes the header
-# from docker's gateway, a private address.
+# refused_from <client> <path>: a request for path from client gets 403.
 refused_from() {
-    code="$(curl --silent --output /dev/null --write-out '%{http_code}' \
-        --max-time 10 --header "X-Forwarded-For: $1" "http://$address$2")" || true
-    [ "$code" = 403 ]
+    [ "$(status_from "$1" "$2")" = 403 ]
+}
+
+# owner_of_state_dir <container>: print who owns /var/lib/smallwebwaf there.
+owner_of_state_dir() {
+    docker exec "$1" stat --format '%U' /var/lib/smallwebwaf
 }
 
 main() {
@@ -131,58 +167,88 @@ main() {
     docker build --no-cache --build-arg SMALLWEBWAF_IMAGE="$IMAGE" \
         -t "$APP_IMAGE" deploy/example-app
 
+    docker network create "$NETWORK" >/dev/null
     docker volume create "$VOLUME" >/dev/null
-    start_container
-    echo "example-app: the health check passes"
+    start_servewaf
+    start_app
+    echo "example-app: the health checks pass"
+
+    [ "$(owner_of_state_dir "$SERVEWAF")" = smallwebwaf ] ||
+        fail "servewaf's container did not give the state directory to smallwebwaf"
+    [ "$(owner_of_state_dir "$APP")" = root ] ||
+        fail "the app's container changed the state directory's owner"
+    echo "example-app: only servewaf's container changes the state directory's owner"
 
     page="$(curl --fail --silent --show-error --max-time 10 "http://$address/")" ||
         fail "no answer on port 8080"
     [ "$page" = "hello from the example app" ] || fail "port 8080 answered $page"
-    wait_for "smallwebwaf logged no request it forwarded" logged '"action":"forward"'
-    echo "example-app: smallwebwaf passes a request to the app and its answer back"
+    wait_for "servewaf logged no request it allowed" \
+        logged "$SERVEWAF" '"action":"forward"' '"servewaf_line":"report"'
+    echo "example-app: rproxy passes a request to the app and its answer back"
 
     refused || fail "a second request in a minute was not refused"
-    wait_for "smallwebwaf logged no ban" logged '"action":"rate_limited"'
+    wait_for "servewaf logged no ban" logged "$SERVEWAF" '"action":"rate_limited"'
     echo "example-app: a second request in a minute bans the client"
 
     refused_from 203.0.113.9 /.env || fail "a probe for /.env was not refused"
-    wait_for "smallwebwaf logged no ban for the probe" \
-        logged '"action":"banned"' '"rule_ids":["env-file"]'
+    wait_for "servewaf logged no ban for the probe" \
+        logged "$SERVEWAF" '"action":"banned"' '"rule_ids":["env-file"]'
     refused_from 203.0.113.9 / || fail "the client of the probe was let through"
     wait_for "the client's next request did not make its ban permanent" \
-        logged '"ban_expires":"permanent"'
+        logged "$SERVEWAF" '"ban_expires":"permanent"'
     echo "example-app: a probe for /.env bans the client, its next request for good"
 
-    docker exec "$CONTAINER" sv stop smallwebwaf >/dev/null ||
-        fail "sv stop smallwebwaf failed"
-    wait_for "smallwebwaf did not stop in order" logged '"msg":"stopped"'
-    echo "example-app: sv stop stops smallwebwaf in order"
+    docker exec "$SERVEWAF" sv stop smallwebwaf >/dev/null ||
+        fail "sv stop smallwebwaf failed in servewaf's container"
+    wait_for "servewaf did not stop in order" logged "$SERVEWAF" '"msg":"stopped"'
+    [ "$(status_from 203.0.113.9 /)" = 200 ] ||
+        fail "with servewaf stopped, the app did not answer through rproxy"
+    wait_for "rproxy logged no request SWRP_DEFAULT_DISPOSITION decided" \
+        logged "$APP" '"default_disposition":true'
+    docker exec "$APP" /usr/local/bin/smallwebwaf healthcheck ||
+        fail "with servewaf stopped, the app's health check failed"
+    echo "example-app: with servewaf stopped, the app answers through rproxy"
 
-    docker stop "$CONTAINER" >/dev/null
-    status="$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER")"
-    [ "$status" = 0 ] || fail "docker stop left exit status $status"
-    echo "example-app: docker stop stops the container in order"
+    docker exec "$SERVEWAF" sv start smallwebwaf >/dev/null ||
+        fail "sv start smallwebwaf failed in servewaf's container"
+    wait_for "once servewaf had started again, the ban did not apply" \
+        refused_from 203.0.113.9 /
+    echo "example-app: once servewaf has started again, bans apply again"
 
-    docker rm "$CONTAINER" >/dev/null
-    start_container
-    refused || fail "the new container let the banned client through"
-    wait_for "smallwebwaf logged no request refused under the ban" \
-        logged '"action":"banned"'
-    echo "example-app: a new container on the same volume keeps the ban"
+    docker exec "$APP" sv stop smallwebwaf >/dev/null ||
+        fail "sv stop smallwebwaf failed in the app's container"
+    wait_for "rproxy did not stop in order" logged "$APP" '"msg":"stopped"'
+    echo "example-app: sv stop stops rproxy in order"
 
-    docker rm --force "$CONTAINER" >/dev/null
-    start_moved_container
+    for container in "$APP" "$SERVEWAF"; do
+        docker stop "$container" >/dev/null
+        status="$(docker inspect --format '{{.State.ExitCode}}' "$container")"
+        [ "$status" = 0 ] || fail "docker stop left exit status $status"
+    done
+    echo "example-app: docker stop stops each container in order"
+
+    docker rm "$APP" "$SERVEWAF" >/dev/null
+    start_servewaf
+    start_app
+    refused || fail "the new servewaf container let the banned client through"
+    wait_for "servewaf logged no request refused under the ban" \
+        logged "$SERVEWAF" '"action":"banned"'
+    echo "example-app: a new servewaf container on the same volume keeps the ban"
+
+    docker rm --force "$SERVEWAF" >/dev/null
+    start_moved_servewaf
     echo "example-app: the run script gives the state files' directory named" \
         "by SWWAF_STATE_DIR_FILE to the smallwebwaf user"
 
-    docker rm --force "$CONTAINER" >/dev/null
-    docker run --detach --name "$CONTAINER" \
+    docker rm --force "$SERVEWAF" >/dev/null
+    docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
+        --env SMALLWEBWAF_COMMAND=servewaf \
         --env SWWAF_STATE_DIR=/srv/missing \
         --env SWWAF_LOOKUP_SOURCE=off \
-        "$APP_IMAGE" >/dev/null
-    wait_for "smallwebwaf did not report the missing state directory" \
-        logged "SWWAF_STATE_DIR cannot be written"
-    echo "example-app: smallwebwaf reports a state directory that does not exist"
+        "$IMAGE" >/dev/null
+    wait_for "servewaf did not report the missing state directory" \
+        logged "$SERVEWAF" "SWWAF_STATE_DIR cannot be written"
+    echo "example-app: servewaf reports a state directory that does not exist"
 }
 
 main "$@"