AutistMask/script/test-verify-build

#!/bin/sh
# script/test-verify-build: exercise every failure mode of
# script/verify-build. Our own extension to scripts-to-rule-them-all, run
# from script/check so make check covers it.
#
# Why this exists: verify-build is the build-integrity guard, and three
# separate reviews of it each found a fresh vacuous pass — the grep exit-2
# conflation, the discarded find status, the line-delimited walk. Every one
# was caught by someone building a tree by hand, because nothing in make check
# could catch it. This is that hand battery, committed and automated.
#
# Each case asserts the exit status AND a substring of the message. A guard
# that fails for the wrong reason (right status, different fault) is itself a
# defect, so matching the status alone would not be a test of anything.
#
# The fixture is a temp tree containing script/verify-build as a SYMLINK to
# the real script: verify-build takes its ROOT from dirname "$0"/.., so it
# operates on the fixture's dist/ and never reads or writes the repo's build
# output. The symlink rather than a copy is what makes a deliberate break in
# the real script fail here.
set -eu

ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
VERIFY_BUILD="$ROOT/script/verify-build"

MARKER_ON="autistmask-build-debug=on"
MARKER_OFF="autistmask-build-debug=off"

NEWLINE='
'

PASSED=0
FAILED=0
SKIPPED=0
SKIPPED_NAMES=""

# The command prefix that runs the permission-dependent cases as a user who
# is actually subject to file permissions, and whether those cases can run at
# all. Both are decided by probe_permission_runner, never assumed.
UNPRIV=""
PERM_ENABLED=no
PERM_HOW=""

WORK=""

cleanup() {
    [ -n "$WORK" ] || return 0
    # The cases chmod 000 files and directories on purpose.
    chmod -R u+rwX "$WORK" 2>/dev/null || true
    rm -rf "$WORK"
}
trap cleanup EXIT INT TERM

WORK="$(mktemp -d "${TMPDIR:-/tmp}/autistmask-test-verify-build.XXXXXX")"
FIXTURE="$WORK/fixture"

# verify-build mktemps its dist/ listing under TMPDIR. Pointing that inside
# our work dir keeps the run leaving no residue, and keeps it writable for the
# unprivileged user the permission cases run as.
TMPDIR="$WORK/tmp"
export TMPDIR
mkdir -p "$TMPDIR"
chmod 1777 "$TMPDIR"
chmod 755 "$WORK"

# --- fixture ---------------------------------------------------------------

# A stand-in for an emitted bundle: some text plus one marker literal, which
# is all verify-build reads out of the real thing.
write_bundle() {
    printf 'var a=1;/* %s */\nvar b=2;\n' "$2" >"$1"
}

# A dist/ shaped like a real build: two listed bundles under different
# browsers, an unlisted subtree to make unwalkable, and unlisted files that
# carry no marker and must not be objected to.
build_fixture() {
    chmod -R u+rwX "$FIXTURE" 2>/dev/null || true
    rm -rf "$FIXTURE"

    mkdir -p "$FIXTURE/script"
    ln -s "$VERIFY_BUILD" "$FIXTURE/script/verify-build"

    mkdir -p "$FIXTURE/dist/chrome/src/popup" \
        "$FIXTURE/dist/chrome/src/content" \
        "$FIXTURE/dist/firefox/src/popup"

    write_bundle "$FIXTURE/dist/chrome/src/popup/index.js" "$MARKER_OFF"
    write_bundle "$FIXTURE/dist/firefox/src/popup/index.js" "$MARKER_OFF"
    printf 'body{color:#000}\n' >"$FIXTURE/dist/styles.css"
    printf 'var c=3;\n' >"$FIXTURE/dist/chrome/src/content/content.js"

    {
        echo "dist/chrome/src/popup/index.js"
        echo "dist/firefox/src/popup/index.js"
    } >"$FIXTURE/dist/constants-bundles.txt"

    # Readable and traversable by the unprivileged user the permission cases
    # run as, before those cases take that away again on purpose.
    chmod -R a+rX "$FIXTURE"
}

# --- permission runner ------------------------------------------------------

# Run a command through the current unprivileged runner. Unquoted on purpose:
# UNPRIV is a command prefix that has to word-split.
run_unpriv() {
    # shellcheck disable=SC2086
    $UNPRIV "$@"
}

# Decide whether the permission-dependent cases can run, and prove it rather
# than assuming it.
#
# The problem: the CI image declares no USER, so CI runs as root, and root is
# not subject to file permissions — chmod 000 stops neither find nor grep. A
# permission case run as root passes vacuously, which is worse than no case at
# all because it reads as coverage.
#
# So the runner is validated with two probes before any permission case is
# counted:
#
#   - a mode-644 file MUST be readable through it. If not, the runner itself
#     is broken (missing helper, no such user, sandbox), and every case run
#     through it would fail for the wrong reason.
#   - a mode-000 file MUST NOT be readable through it. If it is, permissions
#     are not in force and the cases would pass without proving anything.
#
# Unprivileged: the runner is empty and both probes are about this process,
# which is the honest answer. Root: setpriv and runuser are tried, both
# present in the pinned CI base image. Only when no candidate passes both
# probes are the cases skipped, and a skipped run says so unmistakably.
probe_permission_runner() {
    _probe="$WORK/probe"
    mkdir -p "$_probe"
    printf 'readable\n' >"$_probe/public"
    printf 'secret\n' >"$_probe/private"
    chmod 755 "$_probe"
    chmod 644 "$_probe/public"
    chmod 000 "$_probe/private"

    if [ "$(id -u)" -eq 0 ]; then
        _candidates="setpriv|setpriv --reuid=65534 --regid=65534 --clear-groups --
runuser|runuser -u nobody --"
    else
        _candidates="direct|"
    fi

    _tried=""
    _saved_ifs="$IFS"
    IFS="$NEWLINE"
    for _line in $_candidates; do
        IFS="$_saved_ifs"
        _label="${_line%%|*}"
        _cmd="${_line#*|}"
        _tried="${_tried:+$_tried, }$_label"

        if [ -n "$_cmd" ]; then
            _bin="${_cmd%% *}"
            command -v "$_bin" >/dev/null 2>&1 || continue
        fi

        UNPRIV="$_cmd"
        # Broken or unusable runner: the cases would fail for the wrong
        # reason. Reaching the script under test is part of usable.
        run_unpriv cat "$_probe/public" >/dev/null 2>&1 || continue
        run_unpriv cat "$VERIFY_BUILD" >/dev/null 2>&1 || continue
        # Permissions not in force through this runner: the cases would pass
        # without testing anything.
        if run_unpriv cat "$_probe/private" >/dev/null 2>&1; then
            continue
        fi

        PERM_ENABLED=yes
        PERM_HOW="$_label"
        IFS="$_saved_ifs"
        return 0
    done
    IFS="$_saved_ifs"

    UNPRIV=""
    PERM_ENABLED=no
    PERM_HOW="$_tried"
}

# --- case runner ------------------------------------------------------------

# check_case <name> <perm:yes|no> <mode:release|debug> <status> <text> <setup>
#
# Rebuilds the fixture, applies <setup> inside it, runs verify-build, and
# requires both the exit status and the message. <perm> marks a case that only
# means anything when file permissions are in force.
check_case() {
    _name="$1"
    _perm="$2"
    _mode="$3"
    _want_status="$4"
    _want_text="$5"
    _setup="$6"

    if [ "$_perm" = yes ] && [ "$PERM_ENABLED" != yes ]; then
        SKIPPED=$((SKIPPED + 1))
        SKIPPED_NAMES="$SKIPPED_NAMES##   - $_name$NEWLINE"
        echo "  SKIP (permissions not in force): $_name"
        return 0
    fi

    build_fixture
    if ! (cd "$FIXTURE" && "$_setup") >/dev/null 2>&1; then
        FAILED=$((FAILED + 1))
        echo "  FAIL: $_name"
        echo "    the case's own setup failed, so nothing was tested."
        return 0
    fi

    if [ "$_mode" = debug ]; then
        _debug=1
    else
        _debug=""
    fi

    # Exported rather than set as a command prefix: run_unpriv is a function,
    # and an assignment prefixed to a function call is not portable.
    AUTISTMASK_DEBUG="$_debug"
    export AUTISTMASK_DEBUG

    _status=0
    if [ "$_perm" = yes ]; then
        _out="$(run_unpriv "$FIXTURE/script/verify-build" 2>&1)" || _status=$?
    else
        _out="$("$FIXTURE/script/verify-build" 2>&1)" || _status=$?
    fi

    _ok=yes
    _why=""

    if [ "$_status" -ne "$_want_status" ]; then
        _ok=no
        _why="exit status $_status, wanted $_want_status"
    fi

    # Same discipline verify-build itself applies to grep: 0 and 1 are
    # answers, anything else is not, and must not be read as "no match".
    _g=0
    printf '%s\n' "$_out" | grep -q -F -e "$_want_text" || _g=$?
    case "$_g" in
    0) ;;
    1)
        _ok=no
        _why="${_why:+$_why; }message did not contain: $_want_text"
        ;;
    *)
        _ok=no
        _why="${_why:+$_why; }grep exited $_g matching the message, so the
      message was never checked"
        ;;
    esac

    if [ "$_ok" = yes ]; then
        PASSED=$((PASSED + 1))
        echo "  ok: $_name"
        return 0
    fi

    FAILED=$((FAILED + 1))
    echo "  FAIL: $_name"
    echo "    $_why"
    echo "    --- verify-build output ---"
    printf '%s\n' "$_out" | sed 's/^/    /'
    echo "    --- end output ---"
}

# --- cases ------------------------------------------------------------------
#
# Each runs with the fixture as its working directory.

c_control() { :; }

c_trailing_space() {
    cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js "
}

c_embedded_newline() {
    cp dist/chrome/src/popup/index.js "dist/chrome/src/popup/index.js$NEWLINE"
}

c_dist_symlink() {
    mv dist dist.real
    ln -s dist.real dist
}

c_unwalkable_subtree() { chmod 000 dist/chrome/src/content; }

c_dangling_symlink() {
    ln -s /nonexistent-target-for-test-verify-build dist/chrome/dangling.js
}

c_dir_symlink() { ln -s src dist/chrome/link-to-dir; }

c_alias_symlink() { ln -s popup/index.js dist/chrome/src/aliased.js; }

c_manifest_missing() { rm dist/constants-bundles.txt; }

c_manifest_empty() { : >dist/constants-bundles.txt; }

c_manifest_unreadable() { chmod 000 dist/constants-bundles.txt; }

c_bundle_missing() { rm dist/chrome/src/popup/index.js; }

c_bundle_empty() { : >dist/chrome/src/popup/index.js; }

c_bundle_unreadable() { chmod 000 dist/chrome/src/popup/index.js; }

c_unlisted_extension() {
    cp dist/chrome/src/popup/index.js dist/chrome/src/popup/extra.mjs
}

c_no_marker() { printf 'var d=4;\n' >dist/chrome/src/popup/index.js; }

c_both_markers() {
    printf '/* %s */\n' "$MARKER_ON" >>dist/chrome/src/popup/index.js
}

run_cases() {
    check_case "control: untouched dist passes" \
        no release 0 "2 bundle(s) verified $MARKER_OFF" c_control

    check_case "unlisted marker-carrying file, trailing space in name" \
        no release 1 "carries a debug marker but is absent from" \
        c_trailing_space

    check_case "unlisted marker-carrying file, newline in name" \
        no release 1 "carries a debug marker but is absent from" \
        c_embedded_newline

    check_case "dist/ replaced by a symlink" \
        no release 1 "dist is a symlink, not a directory." c_dist_symlink

    check_case "unwalkable subtree under dist/" \
        yes release 1 "enumerating dist/, so part of the tree" \
        c_unwalkable_subtree

    check_case "dangling symlink under dist/" \
        no release 1 \
        "reading dist/chrome/dangling.js, so the file could not be" \
        c_dangling_symlink

    check_case "symlink to a directory under dist/" \
        no release 1 \
        "reading dist/chrome/link-to-dir, so the file could not be" \
        c_dir_symlink

    check_case "symlink to a listed bundle under an unlisted path" \
        no release 1 \
        "dist/chrome/src/aliased.js carries a debug marker but is absent" \
        c_alias_symlink

    check_case "manifest missing" \
        no release 1 "dist/constants-bundles.txt is missing." \
        c_manifest_missing

    check_case "manifest empty" \
        no release 1 "is empty, so no emitted bundle was found to contain" \
        c_manifest_empty

    check_case "manifest unreadable" \
        yes release 1 "is not readable, so nothing was inspected." \
        c_manifest_unreadable

    check_case "listed bundle missing" \
        no release 1 \
        "lists dist/chrome/src/popup/index.js, which does not exist." \
        c_bundle_missing

    check_case "listed bundle empty" \
        no release 1 "which is empty. An empty bundle" c_bundle_empty

    check_case "listed bundle unreadable" \
        yes release 1 \
        "reading dist/chrome/src/popup/index.js, so the file could not be" \
        c_bundle_unreadable

    check_case "unlisted extension carrying a marker" \
        no release 1 \
        "dist/chrome/src/popup/extra.mjs carries a debug marker but is" \
        c_unlisted_extension

    check_case "listed bundle carries no marker" \
        no release 1 "carries no debug marker, so its DEBUG state cannot be" \
        c_no_marker

    check_case "listed bundle carries both markers" \
        no release 1 "carries both debug markers, so DEBUG was not resolved" \
        c_both_markers

    check_case "wrong marker for the requested mode" \
        no debug 1 "is $MARKER_OFF but this build expects $MARKER_ON" \
        c_control
}

# --- main --------------------------------------------------------------------

main() {
    cd "$ROOT"

    [ -x "$VERIFY_BUILD" ] || {
        echo "test-verify-build: $VERIFY_BUILD is missing or not executable" >&2
        exit 1
    }

    echo "Testing script/verify-build failure modes..."
    probe_permission_runner
    if [ "$PERM_ENABLED" = yes ]; then
        echo "  permission cases: enabled (runner: $PERM_HOW, proved against" \
            "a mode-000 file)"
    fi

    run_cases

    if [ "$FAILED" -ne 0 ]; then
        echo "test-verify-build: $FAILED case(s) FAILED," \
            "$PASSED passed, $SKIPPED skipped" >&2
        exit 1
    fi

    if [ "$SKIPPED" -ne 0 ]; then
        cat <<EOF
################################################################################
## WARNING: $SKIPPED PERMISSION CASE(S) DID NOT RUN, AND THIS RUN DOES NOT
## PROVE THEM. This process is uid $(id -u), and no runner subject to file
## permissions was available. Tried: $PERM_HOW.
## Under root, chmod 000 stops neither find nor grep, so these cases would
## have passed without testing anything. They were skipped, not counted:
$SKIPPED_NAMES################################################################################
EOF
        echo "test-verify-build: $PASSED case(s) passed," \
            "$SKIPPED SKIPPED AND NOT PROVEN (see the warning above)"
        return 0
    fi

    echo "test-verify-build: $PASSED case(s) passed"
}

main "$@"