diff --git a/README.md b/README.md index 33949f9..b3f2dfe 100644 --- a/README.md +++ b/README.md @@ -88,13 +88,21 @@ provide: - `script/lint` — run the linter - `script/fmt` — format all files (writes) - `script/fmt-check` — check formatting (read-only) -- `script/check` — run test, lint, and fmt-check +- `script/check` — run test, test-verify-build, lint, and fmt-check - `script/verify-build` — assert the compiled `DEBUG` state of the bundles in `dist/`: every bundle containing `src/shared/constants.js` must have `DEBUG` off, or on when `AUTISTMASK_DEBUG=1`. Run automatically at the end of `make build` and `make build-debug`; fails loudly rather than passing if it cannot determine a bundle's state. Not part of `make check`, which does not depend on build artifacts existing. +- `script/test-verify-build` — exercise every failure mode of + `script/verify-build` against a fixture tree in a temp dir, asserting the exit + status and the message of each. Part of `make check`; it reads no build + artifacts and writes nothing under `dist/`. The cases that depend on file + permissions cannot mean anything for a process that is not subject to them, so + the harness proves its runner against a mode-000 file before counting them, + dropping to an unprivileged user when run as root; if it cannot, it skips + those cases and says so in a banner rather than passing them. - `script/docker` — build the Docker image tagged via `script/projectname` - `script/cibuild` — CI entrypoint: plain `docker build .` - `script/precommit` — run by the git pre-commit hook; runs `script/check` diff --git a/TODO.md b/TODO.md index dcc905f..a22708e 100644 --- a/TODO.md +++ b/TODO.md @@ -44,6 +44,12 @@ undefined identifiers, which is how # Completed Steps +- 2026-08-12: `script/verify-build`'s failure modes are now a committed target, + `script/test-verify-build`, run by `make check`. It asserts the exit status + and the message of every case against a fixture tree in a temp dir, and drops + privileges (proving the runner against a mode-000 file first) for the cases + that only mean something when file permissions are in force + ([#227](https://git.eeqj.de/sneak/AutistMask/issues/227)). - 2026-08-12: Bundled token list documentation no longer states a count. The four "top 250" claims in `README.md` and the "roughly 500" claim in `docs/README.md` are replaced with a description of how the list is actually diff --git a/script/check b/script/check index 3e1778c..ef90d77 100755 --- a/script/check +++ b/script/check @@ -1,12 +1,13 @@ #!/bin/sh -# script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. +# script/check: run all checks (test, test-verify-build, lint, fmt-check). +# Our own extension to scripts-to-rule-them-all. Must not modify any files. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" main() { "$SCRIPT_DIR/test" + "$SCRIPT_DIR/test-verify-build" "$SCRIPT_DIR/lint" "$SCRIPT_DIR/fmt-check" } diff --git a/script/test-verify-build b/script/test-verify-build new file mode 100755 index 0000000..a708afd --- /dev/null +++ b/script/test-verify-build @@ -0,0 +1,444 @@ +#!/bin/sh +# script/test-verify-build: exercise every failure mode of +# script/verify-build. Our own extension to scripts-to-rule-them-all, run +# from script/check so make check covers it. +# +# Why this exists: verify-build is the build-integrity guard, and three +# separate reviews of it each found a fresh vacuous pass — the grep exit-2 +# conflation, the discarded find status, the line-delimited walk. Every one +# was caught by someone building a tree by hand, because nothing in make check +# could catch it. This is that hand battery, committed and automated. +# +# Each case asserts the exit status AND a substring of the message. A guard +# that fails for the wrong reason (right status, different fault) is itself a +# defect, so matching the status alone would not be a test of anything. +# +# The fixture is a temp tree containing script/verify-build as a SYMLINK to +# the real script: verify-build takes its ROOT from dirname "$0"/.., so it +# operates on the fixture's dist/ and never reads or writes the repo's build +# output. The symlink rather than a copy is what makes a deliberate break in +# the real script fail here. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +VERIFY_BUILD="$ROOT/script/verify-build" + +MARKER_ON="autistmask-build-debug=on" +MARKER_OFF="autistmask-build-debug=off" + +NEWLINE=' +' + +PASSED=0 +FAILED=0 +SKIPPED=0 +SKIPPED_NAMES="" + +# The command prefix that runs the permission-dependent cases as a user who +# is actually subject to file permissions, and whether those cases can run at +# all. Both are decided by probe_permission_runner, never assumed. +UNPRIV="" +PERM_ENABLED=no +PERM_HOW="" + +WORK="" + +cleanup() { + [ -n "$WORK" ] || return 0 + # The cases chmod 000 files and directories on purpose. + chmod -R u+rwX "$WORK" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT INT TERM + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/autistmask-test-verify-build.XXXXXX")" +FIXTURE="$WORK/fixture" + +# verify-build mktemps its dist/ listing under TMPDIR. Pointing that inside +# our work dir keeps the run leaving no residue, and keeps it writable for the +# unprivileged user the permission cases run as. +TMPDIR="$WORK/tmp" +export TMPDIR +mkdir -p "$TMPDIR" +chmod 1777 "$TMPDIR" +chmod 755 "$WORK" + +# --- fixture --------------------------------------------------------------- + +# A stand-in for an emitted bundle: some text plus one marker literal, which +# is all verify-build reads out of the real thing. +write_bundle() { + printf 'var a=1;/* %s */\nvar b=2;\n' "$2" >"$1" +} + +# A dist/ shaped like a real build: two listed bundles under different +# browsers, an unlisted subtree to make unwalkable, and unlisted files that +# carry no marker and must not be objected to. +build_fixture() { + chmod -R u+rwX "$FIXTURE" 2>/dev/null || true + rm -rf "$FIXTURE" + + mkdir -p "$FIXTURE/script" + ln -s "$VERIFY_BUILD" "$FIXTURE/script/verify-build" + + mkdir -p "$FIXTURE/dist/chrome/src/popup" \ + "$FIXTURE/dist/chrome/src/content" \ + "$FIXTURE/dist/firefox/src/popup" + + write_bundle "$FIXTURE/dist/chrome/src/popup/index.js" "$MARKER_OFF" + write_bundle "$FIXTURE/dist/firefox/src/popup/index.js" "$MARKER_OFF" + printf 'body{color:#000}\n' >"$FIXTURE/dist/styles.css" + printf 'var c=3;\n' >"$FIXTURE/dist/chrome/src/content/content.js" + + { + echo "dist/chrome/src/popup/index.js" + echo "dist/firefox/src/popup/index.js" + } >"$FIXTURE/dist/constants-bundles.txt" + + # Readable and traversable by the unprivileged user the permission cases + # run as, before those cases take that away again on purpose. + chmod -R a+rX "$FIXTURE" +} + +# --- permission runner ------------------------------------------------------ + +# Run a command through the current unprivileged runner. Unquoted on purpose: +# UNPRIV is a command prefix that has to word-split. +run_unpriv() { + # shellcheck disable=SC2086 + $UNPRIV "$@" +} + +# Decide whether the permission-dependent cases can run, and prove it rather +# than assuming it. +# +# The problem: the CI image declares no USER, so CI runs as root, and root is +# not subject to file permissions — chmod 000 stops neither find nor grep. A +# permission case run as root passes vacuously, which is worse than no case at +# all because it reads as coverage. +# +# So the runner is validated with two probes before any permission case is +# counted: +# +# - a mode-644 file MUST be readable through it. If not, the runner itself +# is broken (missing helper, no such user, sandbox), and every case run +# through it would fail for the wrong reason. +# - a mode-000 file MUST NOT be readable through it. If it is, permissions +# are not in force and the cases would pass without proving anything. +# +# Unprivileged: the runner is empty and both probes are about this process, +# which is the honest answer. Root: setpriv and runuser are tried, both +# present in the pinned CI base image. Only when no candidate passes both +# probes are the cases skipped, and a skipped run says so unmistakably. +probe_permission_runner() { + _probe="$WORK/probe" + mkdir -p "$_probe" + printf 'readable\n' >"$_probe/public" + printf 'secret\n' >"$_probe/private" + chmod 755 "$_probe" + chmod 644 "$_probe/public" + chmod 000 "$_probe/private" + + if [ "$(id -u)" -eq 0 ]; then + _candidates="setpriv|setpriv --reuid=65534 --regid=65534 --clear-groups -- +runuser|runuser -u nobody --" + else + _candidates="direct|" + fi + + _tried="" + _saved_ifs="$IFS" + IFS="$NEWLINE" + for _line in $_candidates; do + IFS="$_saved_ifs" + _label="${_line%%|*}" + _cmd="${_line#*|}" + _tried="${_tried:+$_tried, }$_label" + + if [ -n "$_cmd" ]; then + _bin="${_cmd%% *}" + command -v "$_bin" >/dev/null 2>&1 || continue + fi + + UNPRIV="$_cmd" + # Broken or unusable runner: the cases would fail for the wrong + # reason. Reaching the script under test is part of usable. + run_unpriv cat "$_probe/public" >/dev/null 2>&1 || continue + run_unpriv cat "$VERIFY_BUILD" >/dev/null 2>&1 || continue + # Permissions not in force through this runner: the cases would pass + # without testing anything. + if run_unpriv cat "$_probe/private" >/dev/null 2>&1; then + continue + fi + + PERM_ENABLED=yes + PERM_HOW="$_label" + IFS="$_saved_ifs" + return 0 + done + IFS="$_saved_ifs" + + UNPRIV="" + PERM_ENABLED=no + PERM_HOW="$_tried" +} + +# --- case runner ------------------------------------------------------------ + +# check_case +# +# Rebuilds the fixture, applies inside it, runs verify-build, and +# requires both the exit status and the message. marks a case that only +# means anything when file permissions are in force. +check_case() { + _name="$1" + _perm="$2" + _mode="$3" + _want_status="$4" + _want_text="$5" + _setup="$6" + + if [ "$_perm" = yes ] && [ "$PERM_ENABLED" != yes ]; then + SKIPPED=$((SKIPPED + 1)) + SKIPPED_NAMES="$SKIPPED_NAMES## - $_name$NEWLINE" + echo " SKIP (permissions not in force): $_name" + return 0 + fi + + build_fixture + if ! (cd "$FIXTURE" && "$_setup") >/dev/null 2>&1; then + FAILED=$((FAILED + 1)) + echo " FAIL: $_name" + echo " the case's own setup failed, so nothing was tested." + return 0 + fi + + if [ "$_mode" = debug ]; then + _debug=1 + else + _debug="" + fi + + # Exported rather than set as a command prefix: run_unpriv is a function, + # and an assignment prefixed to a function call is not portable. + AUTISTMASK_DEBUG="$_debug" + export AUTISTMASK_DEBUG + + _status=0 + if [ "$_perm" = yes ]; then + _out="$(run_unpriv "$FIXTURE/script/verify-build" 2>&1)" || _status=$? + else + _out="$("$FIXTURE/script/verify-build" 2>&1)" || _status=$? + fi + + _ok=yes + _why="" + + if [ "$_status" -ne "$_want_status" ]; then + _ok=no + _why="exit status $_status, wanted $_want_status" + fi + + # Same discipline verify-build itself applies to grep: 0 and 1 are + # answers, anything else is not, and must not be read as "no match". + _g=0 + printf '%s\n' "$_out" | grep -q -F -e "$_want_text" || _g=$? + case "$_g" in + 0) ;; + 1) + _ok=no + _why="${_why:+$_why; }message did not contain: $_want_text" + ;; + *) + _ok=no + _why="${_why:+$_why; }grep exited $_g matching the message, so the + message was never checked" + ;; + esac + + if [ "$_ok" = yes ]; then + PASSED=$((PASSED + 1)) + echo " ok: $_name" + return 0 + fi + + FAILED=$((FAILED + 1)) + echo " FAIL: $_name" + echo " $_why" + echo " --- verify-build output ---" + printf '%s\n' "$_out" | sed 's/^/ /' + echo " --- end output ---" +} + +# --- cases ------------------------------------------------------------------ +# +# Each runs with the fixture as its working directory. + +c_control() { :; } + +c_trailing_space() { + cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js " +} + +c_embedded_newline() { + cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js$NEWLINE" +} + +c_dist_symlink() { + mv dist dist.real + ln -s dist.real dist +} + +c_unwalkable_subtree() { chmod 000 dist/chrome/src/content; } + +c_dangling_symlink() { + ln -s /nonexistent-target-for-test-verify-build dist/chrome/dangling.js +} + +c_dir_symlink() { ln -s src dist/chrome/link-to-dir; } + +c_alias_symlink() { ln -s popup/index.js dist/chrome/src/aliased.js; } + +c_manifest_missing() { rm dist/constants-bundles.txt; } + +c_manifest_empty() { : >dist/constants-bundles.txt; } + +c_manifest_unreadable() { chmod 000 dist/constants-bundles.txt; } + +c_bundle_missing() { rm dist/chrome/src/popup/index.js; } + +c_bundle_empty() { : >dist/chrome/src/popup/index.js; } + +c_bundle_unreadable() { chmod 000 dist/chrome/src/popup/index.js; } + +c_unlisted_extension() { + cp dist/chrome/src/popup/index.js dist/chrome/src/popup/extra.mjs +} + +c_no_marker() { printf 'var d=4;\n' >dist/chrome/src/popup/index.js; } + +c_both_markers() { + printf '/* %s */\n' "$MARKER_ON" >>dist/chrome/src/popup/index.js +} + +run_cases() { + check_case "control: untouched dist passes" \ + no release 0 "2 bundle(s) verified $MARKER_OFF" c_control + + check_case "unlisted marker-carrying file, trailing space in name" \ + no release 1 "carries a debug marker but is absent from" \ + c_trailing_space + + check_case "unlisted marker-carrying file, newline in name" \ + no release 1 "carries a debug marker but is absent from" \ + c_embedded_newline + + check_case "dist/ replaced by a symlink" \ + no release 1 "dist is a symlink, not a directory." c_dist_symlink + + check_case "unwalkable subtree under dist/" \ + yes release 1 "enumerating dist/, so part of the tree" \ + c_unwalkable_subtree + + check_case "dangling symlink under dist/" \ + no release 1 \ + "reading dist/chrome/dangling.js, so the file could not be" \ + c_dangling_symlink + + check_case "symlink to a directory under dist/" \ + no release 1 \ + "reading dist/chrome/link-to-dir, so the file could not be" \ + c_dir_symlink + + check_case "symlink to a listed bundle under an unlisted path" \ + no release 1 \ + "dist/chrome/src/aliased.js carries a debug marker but is absent" \ + c_alias_symlink + + check_case "manifest missing" \ + no release 1 "dist/constants-bundles.txt is missing." \ + c_manifest_missing + + check_case "manifest empty" \ + no release 1 "is empty, so no emitted bundle was found to contain" \ + c_manifest_empty + + check_case "manifest unreadable" \ + yes release 1 "is not readable, so nothing was inspected." \ + c_manifest_unreadable + + check_case "listed bundle missing" \ + no release 1 \ + "lists dist/chrome/src/popup/index.js, which does not exist." \ + c_bundle_missing + + check_case "listed bundle empty" \ + no release 1 "which is empty. An empty bundle" c_bundle_empty + + check_case "listed bundle unreadable" \ + yes release 1 \ + "reading dist/chrome/src/popup/index.js, so the file could not be" \ + c_bundle_unreadable + + check_case "unlisted extension carrying a marker" \ + no release 1 \ + "dist/chrome/src/popup/extra.mjs carries a debug marker but is" \ + c_unlisted_extension + + check_case "listed bundle carries no marker" \ + no release 1 "carries no debug marker, so its DEBUG state cannot be" \ + c_no_marker + + check_case "listed bundle carries both markers" \ + no release 1 "carries both debug markers, so DEBUG was not resolved" \ + c_both_markers + + check_case "wrong marker for the requested mode" \ + no debug 1 "is $MARKER_OFF but this build expects $MARKER_ON" \ + c_control +} + +# --- main -------------------------------------------------------------------- + +main() { + cd "$ROOT" + + [ -x "$VERIFY_BUILD" ] || { + echo "test-verify-build: $VERIFY_BUILD is missing or not executable" >&2 + exit 1 + } + + echo "Testing script/verify-build failure modes..." + probe_permission_runner + if [ "$PERM_ENABLED" = yes ]; then + echo " permission cases: enabled (runner: $PERM_HOW, proved against" \ + "a mode-000 file)" + fi + + run_cases + + if [ "$FAILED" -ne 0 ]; then + echo "test-verify-build: $FAILED case(s) FAILED," \ + "$PASSED passed, $SKIPPED skipped" >&2 + exit 1 + fi + + if [ "$SKIPPED" -ne 0 ]; then + cat <