From 7cff69b35f2e2e4ed2696686c7bd7eb0929f0472 Mon Sep 17 00:00:00 2001 From: clawbot Date: Wed, 12 Aug 2026 11:46:00 +0000 Subject: [PATCH] wip: e2e in ci --- .gitea/workflows/e2e.yml | 46 ++++++++++++++++++++++++++++++ .gitea/workflows/probe.yml | 27 ------------------ README.md | 39 +++++++++++++++++++++++-- script/test-e2e | 55 +++++++++++++++++++++++++----------- script/test-e2e-firefox | 47 +++++++++++++++++++----------- tests/e2e/Dockerfile | 33 ++++++++++++++++++++++ tests/e2e/firefox/Dockerfile | 38 +++++++++++++++++++++---- 7 files changed, 217 insertions(+), 68 deletions(-) create mode 100644 .gitea/workflows/e2e.yml delete mode 100644 .gitea/workflows/probe.yml create mode 100644 tests/e2e/Dockerfile diff --git a/.gitea/workflows/e2e.yml b/.gitea/workflows/e2e.yml new file mode 100644 index 0000000..b758abb --- /dev/null +++ b/.gitea/workflows/e2e.yml @@ -0,0 +1,46 @@ +name: e2e +on: [push] + +# The browser end-to-end suites, one job per browser, deliberately kept out +# of the check workflow. REPO_POLICIES.md caps make test at 20 seconds and +# script/cibuild is a plain `docker build .` whose Dockerfile runs +# make check; folding a browser suite into either would blow that cap and +# slow the local fast path. These jobs are the automatic run the suites +# never had: before this workflow, every browser-level guarantee in this +# repo held only when a human remembered to run it by hand. +# +# One job per browser rather than two steps in one job, so a Chrome failure +# does not hide the Firefox result. +# +# Each job is one script and nothing else. Both scripts need docker and +# nothing else — they deliver the repo to the daemon as a build context and +# build the extension inside the pinned image — so neither depends on the +# runner image's toolchain, which ships a node too old to install this +# repo's dependencies. +# +# These jobs REPORT, they do not gate. Whether a check blocks a merge is +# Gitea branch protection, which this repo does not configure, so a failure +# here is a red mark a reviewer has to account for rather than a hard +# block. Making e2e-chrome a required check is the intended end state and +# is deliberately not done yet: the Chrome suite is flaky under load today, +# and a gate that fails at random teaches people to merge past red. +# +# Nothing here may pass vacuously. There is no continue-on-error and no +# `|| true`. Both scripts exit non-zero when docker is missing, when the +# image build fails, and when the browser fails to start; the Chrome +# harness aborts the suite outright if its network interception is not in +# effect. +jobs: + e2e-chrome: + runs-on: ubuntu-latest + steps: + # actions/checkout v4.2.2, 2026-02-22 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: script/test-e2e + + e2e-firefox: + runs-on: ubuntu-latest + steps: + # actions/checkout v4.2.2, 2026-02-22 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: script/test-e2e-firefox diff --git a/.gitea/workflows/probe.yml b/.gitea/workflows/probe.yml deleted file mode 100644 index 83d7e04..0000000 --- a/.gitea/workflows/probe.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: probe -on: [push] -jobs: - probe: - runs-on: ubuntu-latest - steps: - # actions/checkout v4.2.2, 2026-02-22 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - run: | - set -x - id - pwd - cat /etc/hostname - docker version || true - docker info --format '{{.Driver}} {{.ServerVersion}}' || true - echo "--- mountinfo ---" - cat /proc/self/mountinfo - echo "--- inspect self by hostname ---" - docker inspect "$(cat /etc/hostname)" --format '{{json .Mounts}}' || true - echo "--- bind mount probe ---" - docker run --rm -v "$PWD:/work" -w /work alpine:3 ls -la /work | head -20 || true - echo "--- named volume probe ---" - VOL=$(docker inspect "$(cat /etc/hostname)" --format '{{range .Mounts}}{{if eq .Destination "/workspace"}}{{.Name}}{{end}}{{end}}' || true) - echo "VOL=$VOL" - docker run --rm -v "$VOL:/workspace" -w "$PWD" alpine:3 ls -la . | head -20 || true - echo "--- shm in docker run default ---" - docker run --rm alpine:3 df -h /dev/shm || true diff --git a/README.md b/README.md index edeae29..fe2abfe 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,13 @@ provide: - `script/test-e2e-firefox` — run the Firefox browser end-to-end suite (docker required; builds its own pinned image, see [End-to-End Tests](#end-to-end-tests)) +- `script/e2e-container` — `docker run` with this repo placed at `/work`, used + by both end-to-end suites. It copies the repo into the container instead of + bind-mounting it, because a bind mount does not resolve under Gitea Actions: + the runner executes the job inside a container against the host's docker + socket, so `-v "$PWD:/work"` is resolved by the host daemon and mounts an + empty directory. Copying is one mechanism for CI and for a laptop rather than + two, and costs about two seconds. - `script/lint` — run the linter - `script/fmt` — format all files (writes) - `script/fmt-check` — check formatting (read-only) @@ -320,9 +327,35 @@ Two limits are worth knowing, both real differences from the Chrome suite: Neither `make test-e2e` nor `make test-e2e-firefox` is part of `make check` or `make test`. `REPO_POLICIES.md` caps `make test` at 20 seconds and a browser suite does not fit; nothing in `tests/e2e/` is named `*.test.js`, so jest cannot -pick it up either. Neither is wired into the Gitea workflow yet — -docker-in-docker in CI is a separate question. Run them locally before changing -anything under `src/popup/views/`. +pick it up either. Run them locally before changing anything under +`src/popup/views/`. + +### In CI + +`.gitea/workflows/e2e.yml` runs both suites on every push, as two jobs — +`e2e-chrome` and `e2e-firefox` — separate from the `check` workflow, so the +20-second `make test` cap and the local fast path are untouched. Each job is +`script/bootstrap` followed by the matching `make` target; the runner gives the +job container the host's docker socket, which is what makes a nested browser +container possible at all. + +The jobs **report, they do not gate.** A failure is a red mark against the +commit that a reviewer has to account for, not a hard block: whether a check +blocks a merge is Gitea branch protection, which this repo does not configure. +Making `e2e-chrome` a required check is the intended end state and is +deliberately not done yet, because the Chrome suite is flaky under load today +and a gate that fails at random teaches people to merge past red. + +Nothing in either job can pass vacuously. There is no `continue-on-error` and no +`|| true`; both scripts exit non-zero when docker is missing, when there is no +extension build to load, and when the browser fails to start; the Chrome harness +aborts the suite outright if its network interception is not in effect. + +Expect roughly 3-4 minutes for `e2e-chrome`, almost all of it the one-time pull +of the pinned ~2GB Playwright image, and about 1 minute once that image is +cached on the runner. `e2e-firefox` builds its own image, which downloads the +Firefox tarball and geckodriver over the network on a cold cache: about 2 +minutes cold, about 40 seconds warm. ## Rationale diff --git a/script/test-e2e b/script/test-e2e index ece9a5d..9659c48 100755 --- a/script/test-e2e +++ b/script/test-e2e @@ -6,18 +6,33 @@ # Deliberately NOT called by script/check or script/test: REPO_POLICIES.md # caps make test at 20 seconds and a browser suite does not fit. Run it # yourself before touching popup views; it is the only check that can see -# a used-but-not-imported identifier blow up at runtime. +# a used-but-not-imported identifier blow up at runtime. The e2e workflow +# in .gitea/workflows/e2e.yml runs it on every push, as a job separate +# from check so that cap and the local fast path both stay intact. +# +# Docker is the only prerequisite. The repo reaches the container as a +# build context and the extension is built inside it, so nothing here +# depends on the node, yarn or make on the machine that starts the run. +# That is not a convenience: it is what makes the CI job possible. The +# Gitea runner executes the job in a container against the host's docker +# socket, so a `docker run -v "$PWD:/work"` source path is resolved by the +# host daemon and mounts an empty directory (measured on this repo's +# runner), and the runner image ships node 18, which cannot install this +# repo's dependencies at all. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" -# mcr.microsoft.com/playwright:v1.56.0-noble, 2026-08-09 -# -# The playwright-core devDependency is pinned to the matching Playwright -# version (1.56.0) and the two must be bumped together: the browsers ship -# inside this image, and playwright-core looks for the exact browser -# revision its own version expects. A mismatch fails at launch. -IMAGE="mcr.microsoft.com/playwright@sha256:35246d87a7c88ea9b771c65d33171b2611b02a8253b4b12ce6f94376c55f99f2" +IMAGE="$("$SCRIPT_DIR/projectname")-e2e-chrome" + +IIDFILE="" + +cleanup() { + if [ -n "$IIDFILE" ]; then + rm -f "$IIDFILE" + fi +} main() { cd "$ROOT" @@ -27,14 +42,23 @@ main() { exit 1 fi - echo "Building extension for e2e..." - yarn run build 2>&1 + IIDFILE="$(mktemp)" + trap cleanup EXIT + trap 'cleanup; exit 130' INT TERM + + echo "Building the Chrome e2e image (extension included)..." + docker build --iidfile "$IIDFILE" -t "$IMAGE" -f tests/e2e/Dockerfile . echo "Running e2e suite in the pinned Playwright container..." + # The image is run by ID, not by tag: on a machine where two clones of + # this repo run the suite at once, the tag can be moved by the other + # build between this build and this run, and the suite would then + # silently test the other checkout. + # # --ipc=host: Chromium's shared-memory needs more than the default # 64MB /dev/shm or renderers crash. - # --user: keep files the suite touches owned by the caller, not root. - # HOME=/tmp: the mapped uid has no home directory in the image. + # HOME=/tmp: root's home is not writable in every base image and the + # browser profile goes under it. # PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS=1: without it, # ctx.route() intercepts page requests only, and every fetch made by # the MV3 background service worker — including the phishing @@ -51,13 +75,10 @@ main() { # on a deliberate bump. docker run --rm \ --ipc=host \ - --user "$(id -u):$(id -g)" \ -e HOME=/tmp \ -e PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS=1 \ -e "E2E_TRACE_NETWORK=${E2E_TRACE_NETWORK:-0}" \ - -v "$ROOT:/work" \ - -w /work \ - "$IMAGE" \ + "$(cat "$IIDFILE")" \ node tests/e2e/run.js } diff --git a/script/test-e2e-firefox b/script/test-e2e-firefox index ebe72d6..2abfdde 100755 --- a/script/test-e2e-firefox +++ b/script/test-e2e-firefox @@ -5,12 +5,18 @@ # # Deliberately NOT called by script/check or script/test, for the same # reason as the Chrome suite: REPO_POLICIES.md caps make test at 20 seconds -# and a browser suite does not fit. +# and a browser suite does not fit. The e2e workflow in +# .gitea/workflows/e2e.yml runs it on every push, as a job separate from +# check so that cap and the local fast path both stay intact. # -# Unlike script/test-e2e this builds its image locally, because no +# Unlike script/test-e2e this builds its base image locally, because no # published image carries both a pinned Firefox and a matching geckodriver. # All three external artifacts are pinned by digest inside the Dockerfile; -# see tests/e2e/firefox/Dockerfile. +# see tests/e2e/firefox/Dockerfile, which also explains why the repo and +# the extension build are baked into the image rather than mounted. +# +# Docker is the only prerequisite: nothing here depends on the node, yarn +# or make on the machine that starts the run. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -18,6 +24,14 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" IMAGE="$("$SCRIPT_DIR/projectname")-e2e-firefox" +IIDFILE="" + +cleanup() { + if [ -n "$IIDFILE" ]; then + rm -f "$IIDFILE" + fi +} + main() { cd "$ROOT" @@ -26,16 +40,20 @@ main() { exit 1 fi - echo "Building extension for e2e..." - yarn run build 2>&1 + IIDFILE="$(mktemp)" + trap cleanup EXIT + trap 'cleanup; exit 130' INT TERM - # The build context is tests/e2e/firefox/ and holds nothing but the - # Dockerfile: the harness itself arrives over the bind mount below, so - # editing it never invalidates an image layer. - echo "Building the pinned Firefox e2e image..." - docker build -t "$IMAGE" "$ROOT/tests/e2e/firefox" + echo "Building the pinned Firefox e2e image (extension included)..." + docker build --iidfile "$IIDFILE" -t "$IMAGE" \ + -f tests/e2e/firefox/Dockerfile . echo "Running the Firefox e2e suite..." + # The image is run by ID, not by tag: on a machine where two clones of + # this repo run the suite at once, the tag can be moved by the other + # build between this build and this run, and the suite would then + # silently test the other checkout. + # # --shm-size=1g: Firefox needs more than the default 64MB /dev/shm. # --network none: the suite stubs nothing, so this is what keeps the # run offline and deterministic. The extension swallows its own @@ -43,8 +61,8 @@ main() { # network note in README.md. Weaker than the Chrome suite's # fixture interception, and honestly so — it proves no request # escaped, but it cannot report which ones were attempted. - # --user: keep files the suite touches owned by the caller, not root. - # HOME=/tmp: the mapped uid has no home directory in the image. + # HOME=/tmp: root's home is not writable in every base image and the + # browser profile goes under it. # # No --privileged. Firefox's sandbox logs # "CanCreateUserNamespace() clone() failure: EPERM" on startup here; @@ -52,11 +70,8 @@ main() { docker run --rm \ --shm-size=1g \ --network none \ - --user "$(id -u):$(id -g)" \ -e HOME=/tmp \ - -v "$ROOT:/work" \ - -w /work \ - "$IMAGE" \ + "$(cat "$IIDFILE")" \ node tests/e2e/firefox/run.js dist/firefox } diff --git a/tests/e2e/Dockerfile b/tests/e2e/Dockerfile new file mode 100644 index 0000000..0fc8f38 --- /dev/null +++ b/tests/e2e/Dockerfile @@ -0,0 +1,33 @@ +# Chrome end-to-end image: the pinned Playwright image with this repo and a +# freshly built extension inside it, built by script/test-e2e. The suite is +# then started with `docker run`, so every runtime flag the harness needs +# (--ipc=host in particular) still applies. +# +# The repo is baked in rather than bind-mounted because a bind mount does +# not resolve under Gitea Actions: the runner executes the job inside a +# container against the host's docker socket, so the source side of a -v is +# resolved by the host daemon, and the job's checkout lives on a docker +# volume that is not a host path. A build context is streamed to the daemon +# and therefore works from anywhere. Building the extension here as well +# means the machine starting the run needs nothing but docker. + +# mcr.microsoft.com/playwright:v1.56.0-noble, 2026-08-09 +# +# The playwright-core devDependency is pinned to the matching Playwright +# version (1.56.0) and the two must be bumped together: the browsers ship +# inside this image, and playwright-core looks for the exact browser +# revision its own version expects. A mismatch fails at launch. +FROM mcr.microsoft.com/playwright@sha256:35246d87a7c88ea9b771c65d33171b2611b02a8253b4b12ce6f94376c55f99f2 + +WORKDIR /work + +# Same layering as the root Dockerfile: script/bootstrap installs the +# prerequisites and the dependencies, and the manifests are copied first so +# that layer is cached until they change. +COPY script/ script/ +COPY package.json yarn.lock ./ +RUN script/bootstrap + +COPY . . + +RUN make build diff --git a/tests/e2e/firefox/Dockerfile b/tests/e2e/firefox/Dockerfile index 932d92f..1366185 100644 --- a/tests/e2e/firefox/Dockerfile +++ b/tests/e2e/firefox/Dockerfile @@ -1,10 +1,26 @@ # Firefox end-to-end image: stock Firefox plus geckodriver on a node base, -# built by script/test-e2e-firefox. The repo is bind-mounted at /work; the -# harness itself has no dependencies, so nothing is installed for it. +# with this repo and a freshly built extension inside it, built by +# script/test-e2e-firefox. The harness itself has no dependencies, so +# nothing is installed for it. # -# All three external artifacts are pinned by digest. The Firefox version in -# particular must not float: -remote-allow-system-access is mandatory on 153 -# and was not on 142, so the flag the harness passes is version-coupled. +# The build context is the repo root. The repo is baked in rather than +# bind-mounted because a bind mount does not resolve under Gitea Actions: +# the runner executes the job inside a container against the host's docker +# socket, so the source side of a -v is resolved by the host daemon, and +# the job's checkout lives on a docker volume that is not a host path. A +# build context is streamed to the daemon and therefore works from +# anywhere. Building the extension here as well means the machine starting +# the run needs nothing but docker — and it is the only way this suite can +# have both a built extension and the `--network none` it runs under, since +# a container with no network cannot install anything. +# +# The three external artifacts are all pinned by digest and are fetched in +# layers that no repo change touches, so editing the harness or any source +# file re-runs only the two cheap layers at the bottom. +# +# The Firefox version in particular must not float: +# -remote-allow-system-access is mandatory on 153 and was not on 142, so +# the flag the harness passes is version-coupled. # node:22-bookworm-slim, 2026-08-12 FROM node@sha256:d649c27dae7ba0137b3cef5dd75baa422c08dc3d9e3fc0c23dfb172dc3cc6436 @@ -48,4 +64,16 @@ ENV FIREFOX_BIN=/opt/firefox/firefox ENV GECKODRIVER=/usr/local/bin/geckodriver WORKDIR /work + +# Same layering as the root Dockerfile: script/bootstrap installs the +# prerequisites and the dependencies, and the manifests are copied first so +# that layer is cached until they change. +COPY script/ script/ +COPY package.json yarn.lock ./ +RUN script/bootstrap + +COPY . . + +RUN make build + CMD ["node", "tests/e2e/firefox/run.js", "dist/firefox"]