test: cover every verify-build failure mode from make check (closes #227)

script/verify-build is the build-integrity guard and nothing in make check tested it. Three separate reviews of it each found a fresh vacuous pass -- the grep exit-2 conflation, the discarded find status, the line-delimited walk -- and each was caught by someone building a tree by hand. script/test-verify-build makes that battery a target. It builds a fixture tree under mktemp -d holding script/verify-build as a symlink to the real script: verify-build takes its ROOT from dirname "$0"/.., so it runs against the fixture's dist/ and never touches the repo's build output, and the symlink rather than a copy is what makes a break in the real script fail here. Its own mktemp is pointed inside the work dir, so a run leaves nothing behind. Each of the 18 cases asserts the exit status AND a substring of the message, because a guard that fails for the wrong reason is itself a defect. The obstacle was that the CI image declares no USER, so CI runs as root, and root is not subject to file permissions: a chmod 000 case passes vacuously there while passing honestly on a developer machine. The harness therefore proves its runner before counting the three permission-dependent cases -- a mode-644 file and script/verify-build itself must be readable through it, a mode-000 file must not be -- and drops to uid 65534 via setpriv or runuser when running as root, both present in the pinned base image. If no candidate passes the probes the cases are skipped, named in a banner, and the final line says SKIPPED AND NOT PROVEN rather than passed. A green run cannot mean "the permission cases did not run" without saying so.
2026-08-12 09:03:21 +00:00
parent afe6ddaea0
commit 07780c3d8c
4 changed files with 462 additions and 3 deletions
--- a/README.md
+++ b/README.md
@@ -88,13 +88,21 @@ provide:
 - `script/lint` — run the linter
 - `script/fmt` — format all files (writes)
 - `script/fmt-check` — check formatting (read-only)
- `script/check` — run test, lint, and fmt-check
+- `script/check` — run test, test-verify-build, lint, and fmt-check
 - `script/verify-build` — assert the compiled `DEBUG` state of the bundles in
  `dist/`: every bundle containing `src/shared/constants.js` must have `DEBUG`
  off, or on when `AUTISTMASK_DEBUG=1`. Run automatically at the end of
  `make build` and `make build-debug`; fails loudly rather than passing if it
  cannot determine a bundle's state. Not part of `make check`, which does not
  depend on build artifacts existing.
+- `script/test-verify-build` — exercise every failure mode of
+  `script/verify-build` against a fixture tree in a temp dir, asserting the exit
+  status and the message of each. Part of `make check`; it reads no build
+  artifacts and writes nothing under `dist/`. The cases that depend on file
+  permissions cannot mean anything for a process that is not subject to them, so
+  the harness proves its runner against a mode-000 file before counting them,
+  dropping to an unprivileged user when run as root; if it cannot, it skips
+  those cases and says so in a banner rather than passing them.
 - `script/docker` — build the Docker image tagged via `script/projectname`
 - `script/cibuild` — CI entrypoint: plain `docker build .`
 - `script/precommit` — run by the git pre-commit hook; runs `script/check`
--- a/TODO.md
+++ b/TODO.md
@@ -44,6 +44,12 @@ undefined identifiers, which is how

 # Completed Steps

+- 2026-08-12: `script/verify-build`'s failure modes are now a committed target,
+  `script/test-verify-build`, run by `make check`. It asserts the exit status
+  and the message of every case against a fixture tree in a temp dir, and drops
+  privileges (proving the runner against a mode-000 file first) for the cases
+  that only mean something when file permissions are in force
+  ([#227](https://git.eeqj.de/sneak/AutistMask/issues/227)).
 - 2026-08-12: WaitTx lifecycle: a receipt and the 60-second timeout can no
  longer both render on one tick, no timer or in-flight lookup outlives its
  wait, a failed receipt lookup no longer counts as a timeout (but six in a row
--- a/script/check
+++ b/script/check
@@ -1,12 +1,13 @@
 #!/bin/sh
-# script/check: run all checks (test, lint, fmt-check). Our own
-# extension to scripts-to-rule-them-all. Must not modify any files.
+# script/check: run all checks (test, test-verify-build, lint, fmt-check).
+# Our own extension to scripts-to-rule-them-all. Must not modify any files.
 set -eu

 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"

 main() {
    "$SCRIPT_DIR/test"
+    "$SCRIPT_DIR/test-verify-build"
    "$SCRIPT_DIR/lint"
    "$SCRIPT_DIR/fmt-check"
 }
--- a/script/test-verify-build
+++ b/script/test-verify-build
@@ -0,0 +1,444 @@
+#!/bin/sh
+# script/test-verify-build: exercise every failure mode of
+# script/verify-build. Our own extension to scripts-to-rule-them-all, run
+# from script/check so make check covers it.
+#
+# Why this exists: verify-build is the build-integrity guard, and three
+# separate reviews of it each found a fresh vacuous pass — the grep exit-2
+# conflation, the discarded find status, the line-delimited walk. Every one
+# was caught by someone building a tree by hand, because nothing in make check
+# could catch it. This is that hand battery, committed and automated.
+#
+# Each case asserts the exit status AND a substring of the message. A guard
+# that fails for the wrong reason (right status, different fault) is itself a
+# defect, so matching the status alone would not be a test of anything.
+#
+# The fixture is a temp tree containing script/verify-build as a SYMLINK to
+# the real script: verify-build takes its ROOT from dirname "$0"/.., so it
+# operates on the fixture's dist/ and never reads or writes the repo's build
+# output. The symlink rather than a copy is what makes a deliberate break in
+# the real script fail here.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+VERIFY_BUILD="$ROOT/script/verify-build"
+
+MARKER_ON="autistmask-build-debug=on"
+MARKER_OFF="autistmask-build-debug=off"
+
+NEWLINE='
+'
+
+PASSED=0
+FAILED=0
+SKIPPED=0
+SKIPPED_NAMES=""
+
+# The command prefix that runs the permission-dependent cases as a user who
+# is actually subject to file permissions, and whether those cases can run at
+# all. Both are decided by probe_permission_runner, never assumed.
+UNPRIV=""
+PERM_ENABLED=no
+PERM_HOW=""
+
+WORK=""
+
+cleanup() {
+    [ -n "$WORK" ] || return 0
+    # The cases chmod 000 files and directories on purpose.
+    chmod -R u+rwX "$WORK" 2>/dev/null || true
+    rm -rf "$WORK"
+}
+trap cleanup EXIT INT TERM
+
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/autistmask-test-verify-build.XXXXXX")"
+FIXTURE="$WORK/fixture"
+
+# verify-build mktemps its dist/ listing under TMPDIR. Pointing that inside
+# our work dir keeps the run leaving no residue, and keeps it writable for the
+# unprivileged user the permission cases run as.
+TMPDIR="$WORK/tmp"
+export TMPDIR
+mkdir -p "$TMPDIR"
+chmod 1777 "$TMPDIR"
+chmod 755 "$WORK"
+
+# --- fixture ---------------------------------------------------------------
+
+# A stand-in for an emitted bundle: some text plus one marker literal, which
+# is all verify-build reads out of the real thing.
+write_bundle() {
+    printf 'var a=1;/* %s */\nvar b=2;\n' "$2" >"$1"
+}
+
+# A dist/ shaped like a real build: two listed bundles under different
+# browsers, an unlisted subtree to make unwalkable, and unlisted files that
+# carry no marker and must not be objected to.
+build_fixture() {
+    chmod -R u+rwX "$FIXTURE" 2>/dev/null || true
+    rm -rf "$FIXTURE"
+
+    mkdir -p "$FIXTURE/script"
+    ln -s "$VERIFY_BUILD" "$FIXTURE/script/verify-build"
+
+    mkdir -p "$FIXTURE/dist/chrome/src/popup" \
+        "$FIXTURE/dist/chrome/src/content" \
+        "$FIXTURE/dist/firefox/src/popup"
+
+    write_bundle "$FIXTURE/dist/chrome/src/popup/index.js" "$MARKER_OFF"
+    write_bundle "$FIXTURE/dist/firefox/src/popup/index.js" "$MARKER_OFF"
+    printf 'body{color:#000}\n' >"$FIXTURE/dist/styles.css"
+    printf 'var c=3;\n' >"$FIXTURE/dist/chrome/src/content/content.js"
+
+    {
+        echo "dist/chrome/src/popup/index.js"
+        echo "dist/firefox/src/popup/index.js"
+    } >"$FIXTURE/dist/constants-bundles.txt"
+
+    # Readable and traversable by the unprivileged user the permission cases
+    # run as, before those cases take that away again on purpose.
+    chmod -R a+rX "$FIXTURE"
+}
+
+# --- permission runner ------------------------------------------------------
+
+# Run a command through the current unprivileged runner. Unquoted on purpose:
+# UNPRIV is a command prefix that has to word-split.
+run_unpriv() {
+    # shellcheck disable=SC2086
+    $UNPRIV "$@"
+}
+
+# Decide whether the permission-dependent cases can run, and prove it rather
+# than assuming it.
+#
+# The problem: the CI image declares no USER, so CI runs as root, and root is
+# not subject to file permissions — chmod 000 stops neither find nor grep. A
+# permission case run as root passes vacuously, which is worse than no case at
+# all because it reads as coverage.
+#
+# So the runner is validated with two probes before any permission case is
+# counted:
+#
+#   - a mode-644 file MUST be readable through it. If not, the runner itself
+#     is broken (missing helper, no such user, sandbox), and every case run
+#     through it would fail for the wrong reason.
+#   - a mode-000 file MUST NOT be readable through it. If it is, permissions
+#     are not in force and the cases would pass without proving anything.
+#
+# Unprivileged: the runner is empty and both probes are about this process,
+# which is the honest answer. Root: setpriv and runuser are tried, both
+# present in the pinned CI base image. Only when no candidate passes both
+# probes are the cases skipped, and a skipped run says so unmistakably.
+probe_permission_runner() {
+    _probe="$WORK/probe"
+    mkdir -p "$_probe"
+    printf 'readable\n' >"$_probe/public"
+    printf 'secret\n' >"$_probe/private"
+    chmod 755 "$_probe"
+    chmod 644 "$_probe/public"
+    chmod 000 "$_probe/private"
+
+    if [ "$(id -u)" -eq 0 ]; then
+        _candidates="setpriv|setpriv --reuid=65534 --regid=65534 --clear-groups --
+runuser|runuser -u nobody --"
+    else
+        _candidates="direct|"
+    fi
+
+    _tried=""
+    _saved_ifs="$IFS"
+    IFS="$NEWLINE"
+    for _line in $_candidates; do
+        IFS="$_saved_ifs"
+        _label="${_line%%|*}"
+        _cmd="${_line#*|}"
+        _tried="${_tried:+$_tried, }$_label"
+
+        if [ -n "$_cmd" ]; then
+            _bin="${_cmd%% *}"
+            command -v "$_bin" >/dev/null 2>&1 || continue
+        fi
+
+        UNPRIV="$_cmd"
+        # Broken or unusable runner: the cases would fail for the wrong
+        # reason. Reaching the script under test is part of usable.
+        run_unpriv cat "$_probe/public" >/dev/null 2>&1 || continue
+        run_unpriv cat "$VERIFY_BUILD" >/dev/null 2>&1 || continue
+        # Permissions not in force through this runner: the cases would pass
+        # without testing anything.
+        if run_unpriv cat "$_probe/private" >/dev/null 2>&1; then
+            continue
+        fi
+
+        PERM_ENABLED=yes
+        PERM_HOW="$_label"
+        IFS="$_saved_ifs"
+        return 0
+    done
+    IFS="$_saved_ifs"
+
+    UNPRIV=""
+    PERM_ENABLED=no
+    PERM_HOW="$_tried"
+}
+
+# --- case runner ------------------------------------------------------------
+
+# check_case <name> <perm:yes|no> <mode:release|debug> <status> <text> <setup>
+#
+# Rebuilds the fixture, applies <setup> inside it, runs verify-build, and
+# requires both the exit status and the message. <perm> marks a case that only
+# means anything when file permissions are in force.
+check_case() {
+    _name="$1"
+    _perm="$2"
+    _mode="$3"
+    _want_status="$4"
+    _want_text="$5"
+    _setup="$6"
+
+    if [ "$_perm" = yes ] && [ "$PERM_ENABLED" != yes ]; then
+        SKIPPED=$((SKIPPED + 1))
+        SKIPPED_NAMES="$SKIPPED_NAMES##   - $_name$NEWLINE"
+        echo "  SKIP (permissions not in force): $_name"
+        return 0
+    fi
+
+    build_fixture
+    if ! (cd "$FIXTURE" && "$_setup") >/dev/null 2>&1; then
+        FAILED=$((FAILED + 1))
+        echo "  FAIL: $_name"
+        echo "    the case's own setup failed, so nothing was tested."
+        return 0
+    fi
+
+    if [ "$_mode" = debug ]; then
+        _debug=1
+    else
+        _debug=""
+    fi
+
+    # Exported rather than set as a command prefix: run_unpriv is a function,
+    # and an assignment prefixed to a function call is not portable.
+    AUTISTMASK_DEBUG="$_debug"
+    export AUTISTMASK_DEBUG
+
+    _status=0
+    if [ "$_perm" = yes ]; then
+        _out="$(run_unpriv "$FIXTURE/script/verify-build" 2>&1)" || _status=$?
+    else
+        _out="$("$FIXTURE/script/verify-build" 2>&1)" || _status=$?
+    fi
+
+    _ok=yes
+    _why=""
+
+    if [ "$_status" -ne "$_want_status" ]; then
+        _ok=no
+        _why="exit status $_status, wanted $_want_status"
+    fi
+
+    # Same discipline verify-build itself applies to grep: 0 and 1 are
+    # answers, anything else is not, and must not be read as "no match".
+    _g=0
+    printf '%s\n' "$_out" | grep -q -F -e "$_want_text" || _g=$?
+    case "$_g" in
+    0) ;;
+    1)
+        _ok=no
+        _why="${_why:+$_why; }message did not contain: $_want_text"
+        ;;
+    *)
+        _ok=no
+        _why="${_why:+$_why; }grep exited $_g matching the message, so the
+      message was never checked"
+        ;;
+    esac
+
+    if [ "$_ok" = yes ]; then
+        PASSED=$((PASSED + 1))
+        echo "  ok: $_name"
+        return 0
+    fi
+
+    FAILED=$((FAILED + 1))
+    echo "  FAIL: $_name"
+    echo "    $_why"
+    echo "    --- verify-build output ---"
+    printf '%s\n' "$_out" | sed 's/^/    /'
+    echo "    --- end output ---"
+}
+
+# --- cases ------------------------------------------------------------------
+#
+# Each runs with the fixture as its working directory.
+
+c_control() { :; }
+
+c_trailing_space() {
+    cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js "
+}
+
+c_embedded_newline() {
+    cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js$NEWLINE"
+}
+
+c_dist_symlink() {
+    mv dist dist.real
+    ln -s dist.real dist
+}
+
+c_unwalkable_subtree() { chmod 000 dist/chrome/src/content; }
+
+c_dangling_symlink() {
+    ln -s /nonexistent-target-for-test-verify-build dist/chrome/dangling.js
+}
+
+c_dir_symlink() { ln -s src dist/chrome/link-to-dir; }
+
+c_alias_symlink() { ln -s popup/index.js dist/chrome/src/aliased.js; }
+
+c_manifest_missing() { rm dist/constants-bundles.txt; }
+
+c_manifest_empty() { : >dist/constants-bundles.txt; }
+
+c_manifest_unreadable() { chmod 000 dist/constants-bundles.txt; }
+
+c_bundle_missing() { rm dist/chrome/src/popup/index.js; }
+
+c_bundle_empty() { : >dist/chrome/src/popup/index.js; }
+
+c_bundle_unreadable() { chmod 000 dist/chrome/src/popup/index.js; }
+
+c_unlisted_extension() {
+    cp dist/chrome/src/popup/index.js dist/chrome/src/popup/extra.mjs
+}
+
+c_no_marker() { printf 'var d=4;\n' >dist/chrome/src/popup/index.js; }
+
+c_both_markers() {
+    printf '/* %s */\n' "$MARKER_ON" >>dist/chrome/src/popup/index.js
+}
+
+run_cases() {
+    check_case "control: untouched dist passes" \
+        no release 0 "2 bundle(s) verified $MARKER_OFF" c_control
+
+    check_case "unlisted marker-carrying file, trailing space in name" \
+        no release 1 "carries a debug marker but is absent from" \
+        c_trailing_space
+
+    check_case "unlisted marker-carrying file, newline in name" \
+        no release 1 "carries a debug marker but is absent from" \
+        c_embedded_newline
+
+    check_case "dist/ replaced by a symlink" \
+        no release 1 "dist is a symlink, not a directory." c_dist_symlink
+
+    check_case "unwalkable subtree under dist/" \
+        yes release 1 "enumerating dist/, so part of the tree" \
+        c_unwalkable_subtree
+
+    check_case "dangling symlink under dist/" \
+        no release 1 \
+        "reading dist/chrome/dangling.js, so the file could not be" \
+        c_dangling_symlink
+
+    check_case "symlink to a directory under dist/" \
+        no release 1 \
+        "reading dist/chrome/link-to-dir, so the file could not be" \
+        c_dir_symlink
+
+    check_case "symlink to a listed bundle under an unlisted path" \
+        no release 1 \
+        "dist/chrome/src/aliased.js carries a debug marker but is absent" \
+        c_alias_symlink
+
+    check_case "manifest missing" \
+        no release 1 "dist/constants-bundles.txt is missing." \
+        c_manifest_missing
+
+    check_case "manifest empty" \
+        no release 1 "is empty, so no emitted bundle was found to contain" \
+        c_manifest_empty
+
+    check_case "manifest unreadable" \
+        yes release 1 "is not readable, so nothing was inspected." \
+        c_manifest_unreadable
+
+    check_case "listed bundle missing" \
+        no release 1 \
+        "lists dist/chrome/src/popup/index.js, which does not exist." \
+        c_bundle_missing
+
+    check_case "listed bundle empty" \
+        no release 1 "which is empty. An empty bundle" c_bundle_empty
+
+    check_case "listed bundle unreadable" \
+        yes release 1 \
+        "reading dist/chrome/src/popup/index.js, so the file could not be" \
+        c_bundle_unreadable
+
+    check_case "unlisted extension carrying a marker" \
+        no release 1 \
+        "dist/chrome/src/popup/extra.mjs carries a debug marker but is" \
+        c_unlisted_extension
+
+    check_case "listed bundle carries no marker" \
+        no release 1 "carries no debug marker, so its DEBUG state cannot be" \
+        c_no_marker
+
+    check_case "listed bundle carries both markers" \
+        no release 1 "carries both debug markers, so DEBUG was not resolved" \
+        c_both_markers
+
+    check_case "wrong marker for the requested mode" \
+        no debug 1 "is $MARKER_OFF but this build expects $MARKER_ON" \
+        c_control
+}
+
+# --- main --------------------------------------------------------------------
+
+main() {
+    cd "$ROOT"
+
+    [ -x "$VERIFY_BUILD" ] || {
+        echo "test-verify-build: $VERIFY_BUILD is missing or not executable" >&2
+        exit 1
+    }
+
+    echo "Testing script/verify-build failure modes..."
+    probe_permission_runner
+    if [ "$PERM_ENABLED" = yes ]; then
+        echo "  permission cases: enabled (runner: $PERM_HOW, proved against" \
+            "a mode-000 file)"
+    fi
+
+    run_cases
+
+    if [ "$FAILED" -ne 0 ]; then
+        echo "test-verify-build: $FAILED case(s) FAILED," \
+            "$PASSED passed, $SKIPPED skipped" >&2
+        exit 1
+    fi
+
+    if [ "$SKIPPED" -ne 0 ]; then
+        cat <<EOF
+################################################################################
+## WARNING: $SKIPPED PERMISSION CASE(S) DID NOT RUN, AND THIS RUN DOES NOT
+## PROVE THEM. This process is uid $(id -u), and no runner subject to file
+## permissions was available. Tried: $PERM_HOW.
+## Under root, chmod 000 stops neither find nor grep, so these cases would
+## have passed without testing anything. They were skipped, not counted:
+$SKIPPED_NAMES################################################################################
+EOF
+        echo "test-verify-build: $PASSED case(s) passed," \
+            "$SKIPPED SKIPPED AND NOT PROVEN (see the warning above)"
+        return 0
+    fi
+
+    echo "test-verify-build: $PASSED case(s) passed"
+}
+
+main "$@"