diff --git a/Dockerfile b/Dockerfile index de844aa..332a4d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,11 +10,22 @@ # phase covers the frontend as well, through a copy from a node stage. # Inside them each tool is invoked directly, never through make or # script/, whose lint and test are themselves docker builds. +# +# In every stage ARG CACHEBUST comes right after the steps that install +# system packages and download Go's modules, or right after FROM in a +# stage with neither, and every step that lints, tests or compiles +# comes after it, as does yarn install in the node stages. script/ +# passes a new value on every build, which reruns every RUN step after +# it. Docker keeps the layers above it until the base image, their own +# lines or go.mod and go.sum change. The steps that compile mount Go's +# build cache, and golangci-lint run also mounts golangci-lint's; both +# survive from build to build. # Frontend lint stage: eslint with the rules in eslint.config.js. The # lint phase below runs it. # node:22-alpine as of 2026-02-22 FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend-lint +ARG CACHEBUST WORKDIR /app COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile @@ -29,8 +40,11 @@ FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a924 WORKDIR /src COPY go.mod go.sum ./ RUN go mod download +ARG CACHEBUST COPY . . -RUN golangci-lint run --config .golangci.yml ./... +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run --config .golangci.yml ./... # Nothing is wanted from frontend-lint; the copy is what makes this # phase run it. COPY --from=frontend-lint /app/yarn.lock /dev/null @@ -43,6 +57,7 @@ COPY --from=frontend-lint /app/yarn.lock /dev/null # node would take a reporter option for one more file. # node:22-alpine as of 2026-02-22 FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend-test +ARG CACHEBUST WORKDIR /app COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile @@ -58,19 +73,19 @@ RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \ # ships and the alpine one does not. -timeout 90s is a backstop above # the 60-second cap on the suite. The rerun with -v only shows details: # the step fails however it ends, because the first run already failed. -# Go's module and build caches are in memory (tmpfs) for the go test -# step alone, so the image make test tags does not carry them and the -# build spends no time writing them into it. They start empty on every -# build, so no stored test result can stand in for a run. +# -count=1 because the mounted cache also holds test results, which go +# test would otherwise replay. # golang:1.25.7-trixie, 2026-10-07 FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +ARG CACHEBUST COPY . . -RUN --mount=type=tmpfs,target=/go/pkg/mod \ - --mount=type=tmpfs,target=/root/.cache/go-build \ - go test -timeout 90s -race -cover ./... || \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 90s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } # Nothing is wanted from the frontend-test stage; the copy is what makes # this phase run its tests. COPY --from=frontend-test /app/yarn.lock /dev/null @@ -85,21 +100,23 @@ COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null # vite.config.js reads the commit for the page's footer with git. RUN apk add --no-cache git +ARG CACHEBUST RUN yarn build # Backend build stage. Nothing is wanted from the two phases; the copies -# are what make BuildKit build them first, so this stage cannot run -# unless lint and test passed. +# are what make BuildKit build them first, so nothing after them can +# run unless lint and test passed. # golang:1.25-alpine (2026-02-27) FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder -COPY --from=lint /src/go.sum /dev/null -COPY --from=test /src/go.sum /dev/null RUN apk add --no-cache git -# A tar-stream context keeps the sender's file owners, which git refuses. -RUN git config --system --add safe.directory /src WORKDIR /src COPY go.mod go.sum ./ RUN go mod download +ARG CACHEBUST +COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src COPY . . # The same go build as in script/build, which builds the binary on the @@ -112,7 +129,8 @@ COPY . . # still empty, dev or unknown fails the build: git is missing or could # not read the checkout. ARG VERSION -RUN version="${VERSION:-$(git describe --tags --always)}"; \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + version="${VERSION:-$(git describe --tags --always)}"; \ if [ -e .git ]; then \ case "$version" in ""|dev|unknown) \ echo "version is '$version' although .git is present" >&2; \ @@ -127,6 +145,7 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \ # and the stages it copies from, the two phases included. # nginx:stable-alpine as of 2026-02-22 FROM nginx@sha256:15e96e59aa3b0aada3a121296e3bce117721f42d88f5f64217ef4b18f458c6ab +ARG CACHEBUST # netwatch-server runs as this user, which owns the report directory. # nginx keeps the image's own arrangement: its main process runs as diff --git a/README.md b/README.md index 1574245..fdd62ce 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,11 @@ either one is broken. We provide: defaults (see [Settings](#settings)) - `script/clean` — remove what `script/build` writes - `script/projectname` — print the project name (used for the Docker image tag) -- `script/test` — build the `test` phase of `Dockerfile` without the cache: the +- `script/test` — build the `test` phase of `Dockerfile`, writing no image: the frontend's unit tests in its `frontend-test` stage, and the backend's Go tests with the race detector and coverage. The frontend's production build is left to the image build, in the `frontend-build` stage -- `script/lint` — build the `lint` phase of `Dockerfile` without the cache: +- `script/lint` — build the `lint` phase of `Dockerfile`, writing no image: eslint in its `frontend-lint` stage, and golangci-lint over the Go packages - `script/fmt` — format all files (writes): prettier over the JavaScript, CSS, HTML and Markdown, then gofmt over `cmd/` and `internal/`. It runs on the @@ -90,13 +90,22 @@ either one is broken. We provide: starts, it checks that the page, and a path that falls back to it, are served with `Cache-Control: no-cache` and `Content-Security-Policy`. Not part of `script/check`: it takes minutes. -- `script/docker` — build the image from `Dockerfile` without the build cache, - tagged `netwatch` via `script/projectname` +- `script/docker` — build the image from `Dockerfile`, tagged `netwatch` via + `script/projectname` - `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`, - then builds the image as `script/docker` does, without the build cache + then builds the image as `script/docker` does - `script/precommit` — run by the git pre-commit hook; runs `script/check` - `script/install-precommit` — install the git pre-commit hook +`script/test`, `script/lint`, `script/docker` and `script/cibuild` each pass a +new `CACHEBUST` value, so every step in `Dockerfile` that lints, tests or +compiles runs again even when nothing has changed. Docker keeps the system +packages and the Go modules from earlier builds until the base image, the step +that installs them, or `go.mod` and `go.sum` change. Go's build cache and +golangci-lint's are kept as well, and each reuses a result only when everything +it was made from is unchanged. Both `go test` runs carry `-count=1`, so no test +result is reused. The yarn packages are installed again on every build. + ## Responsive layout The narrow-viewport layout lives in the `max-width: 768px` media block in diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 20382d1..1067698 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-10-04 +last_modified: 2026-10-18 --- This document covers repository structure, tooling, and workflow standards. Code @@ -54,34 +54,39 @@ style conventions are in separate documents: `cibuild`. `script/bootstrap` installs all dependencies idempotently and assumes nothing is present: base tools come from nix, apt, brew, or apk (detected in that order; apt runs noninteractive). For node it uses the - installed node if present; otherwise it installs a PINNED node version via - nvm, first installing nvm itself if missing — from a hash-verified GitHub - release archive (never `curl | sh`), with bash installed as an explicit - prerequisite since nvm requires bash. yarn is then pinned via - `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; - always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root, runs `script/bootstrap`, runs `script/check`, and builds the image - with the version; the Gitea workflow calls it. **`script/cibuild` runs + installed node only when its major version is the pinned one; otherwise (node + missing, or another major, such as the node 24 the CI runner image ships) it + installs the PINNED node version via nvm and installs yarn and the packages + under it. nvm itself is installed first if missing, from a hash-verified + GitHub release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. Only the major version is compared + because the `Dockerfile` stages start from a node image whose exact version is + not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned + via `corepack prepare yarn@ --activate`. Never install "latest" or + "lts"; always exact versions. `script/cibuild` runs the CI build: it changes + to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the + image with the version; the Gitea workflow calls it. **`script/cibuild` runs `script/bootstrap` first**, because the workflow checks out the repo and runs nothing else, while `script/fmt-check` runs the formatter on the host: on a pristine checkout with nothing installed the run dies there, after the - containerised gates have passed. **The bootstrap alone is not enough**: - `script/bootstrap` installs node and yarn under nvm and leaves neither on the - `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host - entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore - source nvm for the pinned node version before invoking it, exactly as - `script/bootstrap`'s own install step does. A runner carrying nothing but - docker and git then gets through `script/check`. Four further scripts are our - own extensions to the standard: `script/check` runs `script/test`, - `script/lint` and `script/fmt-check`; `script/precommit` is what the git - pre-commit hook runs, and it calls `script/check`; `script/install-precommit` - installs the git pre-commit hook (the `make hooks` target shims to it); and - `script/projectname` (literally that filename) simply outputs the project's - name. Scripts that need the name call `script/projectname` — e.g. - `script/docker` assembles its image tag from it — so those scripts stay - byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. - `go mod tidy` verification in Go repos) belong in `script/precommit`, not in - the hook itself. Model scripts are at + containerised gates have passed. **The bootstrap alone is not enough**: when + `script/bootstrap` installs node and yarn under nvm it leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs + under another node. The host entrypoints that need yarn — `script/fmt` and + `script/fmt-check` — therefore use the same test as `script/bootstrap`: when + the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, + and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the + pinned node. A runner carrying nothing but docker and git then gets through + `script/check`. Four further scripts are our own extensions to the standard: + `script/check` runs `script/test`, `script/lint` and `script/fmt-check`; + `script/precommit` is what the git pre-commit hook runs, and it calls + `script/check`; `script/install-precommit` installs the git pre-commit hook + (the `make hooks` target shims to it); and `script/projectname` (literally + that filename) simply outputs the project's name. Scripts that need the name + call `script/projectname` — e.g. `script/docker` assembles its image tag from + it — so those scripts stay byte-identical across all repos. Repo-type-specific + pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in + `script/precommit`, not in the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -118,8 +123,10 @@ style conventions are in separate documents: and nothing else: ```sh - docker build --no-cache --target lint -t "$(script/projectname)-lint" . - docker build --no-cache --target test -t "$(script/projectname)-test" . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint --output type=cacheonly . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test --output type=cacheonly . ``` **A stage that is not the last one in the file is built only when the final @@ -129,10 +136,15 @@ style conventions are in separate documents: plain `docker build .` builds the last stage alone and exits 0 having linted and tested nothing. - **Every `docker build` in `script/` is tagged**, here and in - `script/cibuild` and `script/docker`. An untagged build leaves a dangling - image behind on every invocation, on every developer host and every CI - runner; a tagged one replaces the previous image. + **The gate builds write no image.** With `--output type=cacheonly` the phase + runs and a failing step fails the build, but the result is not exported. + Nothing uses those images, and writing one out is slow: a Go test phase's + image holds the toolchain. A build given neither `--output` nor `-t` writes + an untagged image and leaves it dangling, on every developer host and every + CI runner. `script/cibuild` and `script/docker` build the image that ships + and tag it, so each build replaces the previous image; each assigns the tag + on its own line before the build, so `set -e` stops it where + `script/projectname` fails. Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, `eslint`, `prettier` — never through `make lint` or `script/test`, which are @@ -147,22 +159,42 @@ style conventions are in separate documents: one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs exit non-zero with `parallel golangci-lint is running` — a status a caller cannot tell from real findings. Both have produced wrong verdicts in this - org, in both directions. A container has its own cache, its own `TMPDIR` and - a digest-pinned binary, so neither is reachable. + org, in both directions. A container has its own `TMPDIR`, so the lock is + out of reach. The lint phase does keep golangci-lint's cache between check + builds (below), but it lints every checkout at `/src`, and a finding is + taken from that cache only for the same files, settings and golangci-lint + version. -- **Any build that runs checks is built with `--no-cache`.** Docker invalidates - a `COPY` layer only when the copied content changes, so on an unchanged tree - the check `RUN` is served from cache, nothing executes, and the build still - exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: - `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the - four, and there is no fifth — `script/check` runs the two gate phases and - `script/fmt-check`, and builds no image of its own. A bare `docker build .` is - not evidence that anything ran: a sub-second build reporting success is a - cache hit, not a result. Never invalidate by pruning — `docker builder prune` - and friends destroy a build cache shared with every other build on the host. - When a check is added or changed, prove it works by planting a defect it must - catch and watching the run fail on it, then revert the defect. A green run - alone shows neither that the check ran nor that it covers what it should. +- **Any build that runs checks is given a new `CACHEBUST` value.** Docker + invalidates a `COPY` layer only when the copied content changes, so on an + unchanged tree the check `RUN` is served from cache, nothing executes, and the + build still exits 0. Every `docker build` in `script/` therefore passes + `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process + id, so no two builds share a value, not even the three in one + `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and + `script/docker` are the four, and there is no fifth — `script/check` runs the + two gate phases and `script/fmt-check`, and builds no image of its own. In + every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps + that install system packages and download Go's modules, or right after `FROM` + in a stage with neither, and every step that lints, tests or compiles comes + after it. A step that installs anything else as well, such as a + `script/bootstrap` that also installs node packages, comes after it too. A new + value reruns every `RUN` step after that line; `COPY` steps are compared by + content and need nothing. Docker keeps the layers above the line until the + base image, their own lines or the files they copy change. A stage without the + `ARG` line is served from cache on an unchanged tree, so a repository + re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do + not pass `--no-cache`: it reruns every step too, but it also builds the layers + above the line again and empties every cache mount, so Go's build cache and + golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they + survive between check builds, would start empty every time. A bare + `docker build .` is not evidence that anything ran: a sub-second build + reporting success is a cache hit, not a result. Never invalidate by pruning — + `docker builder prune` and friends destroy a build cache shared with every + other build on the host. When a check is added or changed, prove it works by + planting a defect it must catch and watching the run fail on it, then revert + the defect. A green run alone shows neither that the check ran nor that it + covers what it should. - **The gate phases are separate stages, and the build stage depends on both.** The lint phase is based on the `golangci/golangci-lint` image (pinned by @@ -171,40 +203,56 @@ style conventions are in separate documents: `Dockerfile`: ```dockerfile + # In every stage ARG CACHEBUST comes right after the steps that install + # system packages and download Go's modules, or right after FROM in a + # stage with neither, and every step that lints, tests or compiles + # comes after it. script/ passes a new value on every build, which + # reruns every RUN step after it. Docker keeps the layers above it + # until the base image, their own lines or go.mod and go.sum change. + # The steps that compile mount Go's build cache, and golangci-lint run + # also mounts golangci-lint's; both survive from build to build. + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . - RUN golangci-lint run --config .golangci.yml ./... + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + golangci-lint run --config .golangci.yml ./... # Test phase. -race needs cgo and so a C compiler, which the Debian Go - # image ships and the alpine one does not. + # image ships and the alpine one does not. -count=1 because the mounted + # cache also holds test results, which go test would otherwise replay. # golang:1.x, YYYY-MM-DD FROM golang@sha256:... AS test WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . - RUN go test -timeout 90s -race -cover ./... || \ + RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 90s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } # Build stage. Nothing is wanted from either phase above; the copies - # are what make BuildKit build them first, so this stage cannot run - # unless lint and test passed. + # are what make BuildKit build them first, so nothing after them can + # run unless lint and test passed. # golang:1.x-alpine, YYYY-MM-DD FROM golang@sha256:... AS builder - COPY --from=lint /src/go.sum /dev/null - COPY --from=test /src/go.sum /dev/null RUN apk add --no-cache git - # A tar-stream context keeps the sender's file owners, which git refuses. - RUN git config --system --add safe.directory /src WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src COPY . . # The VERSION build arg when one is given, otherwise @@ -212,7 +260,8 @@ style conventions are in separate documents: # .git present, a version that is still empty, dev or unknown fails the # build: git is missing or could not read the checkout. ARG VERSION - RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ if [ -e .git ]; then \ case "$VERSION" in ""|dev|unknown) \ echo "version is '$VERSION' although .git is present" >&2; \ @@ -225,6 +274,7 @@ style conventions are in separate documents: # Runtime stage, and the last one FROM alpine@sha256:... + ARG CACHEBUST COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` @@ -239,6 +289,26 @@ style conventions are in separate documents: - Keep the runtime stage last, and if you add a stage after it, give it the same two copies. A plain `docker build .` builds the last stage's chain and nothing else. + - Check builds keep these, and none of them can hand back a stale result: + - The layers above `ARG CACHEBUST`: the system packages and Go's + modules. Docker installs the packages again when the base image or the + install line changes, and downloads the modules again when `go.mod` or + `go.sum` changes. Go checks every module against `go.sum` as it + downloads it, and no later build can change a kept layer. The modules + are kept in a layer rather than a cache mount because a cache mount is + shared by every build on the host, and Go builds from whatever files + it holds without checking them again (only `go mod verify` does), so a + file changed there would go unnoticed. + - Go's build cache (`/root/.cache/go-build`, where Go keeps it for + root), mounted on the steps that compile: `golangci-lint run`, both + `go test` runs and `go build`. Go reuses a compiled package only when + all its inputs are unchanged. The cache also holds test results, which + is why both `go test` runs carry `-count=1`. + - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned + image keeps it), mounted on `golangci-lint run`. golangci-lint reuses + a package's findings only when the package's files and those of every + package it imports, `go.mod`, the linters and their settings in + `.golangci.yml`, and the golangci-lint version are all unchanged. - If the project uses `//go:embed` directives that reference build artifacts (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: @@ -247,12 +317,20 @@ style conventions are in separate documents: in the lint phase. The `golangci/golangci-lint` image is Debian-based and has no `apk`, so install with `apt-get` under the Debian package name (`libvips-dev`, where alpine says `vips-dev`), and delete the package - lists in the same `RUN`, so the layer does not keep them: + lists in the same `RUN`, so the layer does not keep them. The test phase, + on the Debian Go image, installs the libraries its tests need the same + way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so + Docker keeps it until the base image or the install line changes: ```dockerfile + FROM golangci/golangci-lint@sha256:... AS lint RUN apt-get update \ && apt-get install -y --no-install-recommends libvips-dev \ && rm -rf /var/lib/apt/lists/* + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + ARG CACHEBUST ``` - `.dockerignore` lets `.git` into the build context. It keeps out every git @@ -278,20 +356,39 @@ style conventions are in separate documents: `.git` and the version still comes out empty, `dev` or `unknown`. A plain `docker build .` with no build arguments must succeed; a Dockerfile that refuses an empty build argument drops that refusal and keeps the argument. + A checkout whose `.git` is a file (a linked worktree, or a repository + checked out as a submodule) is the exception: that file points to a git + directory outside the build context, so the build cannot read the version + and a plain `docker build .` fails; pass the version with + `--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already + do. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` on push, and checks out the repo as its only other step. - That script bootstraps, runs the gate phases, and then builds the image, so a - successful run means every check passed; a bare `docker build .` does not + runs `script/cibuild` on push, and checks out the repo as its only other step, + with `persist-credentials: false`: `script/cibuild` needs no token, and + without it the checkout leaves the job's token in `.git/config` for every + later step. The checkout step also sets `fetch-depth: 0`, which fetches the + tags `git describe` needs: by default it clones shallow with no tags, and a + tagged repository's CI build would stamp a bare short commit id. The + workflow's `concurrency` block groups runs by workflow and branch + (`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`, + so a new push cancels the older run on the same branch, queued or running, and + no other: runs for replaced commits do not hold up the shared runner. + `script/cibuild` bootstraps, runs the gate phases, and then builds the image, + so a successful run means every check passed; a bare `docker build .` does not carry the same guarantee, because its gate phases may come from the cache. The - image build is uncached and so runs the gate phases a second time. That is the - price of the rule above, and it is worth paying: the image that ships is built - from a run of its own gates rather than from a cache entry. A separate - workflow limited to `main` by a `branches` list under `on: push` cannot be - checked by review: to try a change to it, add the feature branch to that list - and push, then remove the branch from the list again before merging. Keep any - job in it that publishes behind `if: github.ref_name == 'main'`, so the run - from the feature branch publishes nothing. + image build gets a `CACHEBUST` value of its own and so runs the gate phases a + second time. That is the price of the rule above, and it is worth paying: the + image that ships is built from a run of its own gates rather than from a cache + entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the + shared runner after 20 minutes. That allows for the three Docker builds + described above (the test phase, the lint phase, then the image), each held to + the 5-minute Docker build limit below, plus the bootstrap. A separate workflow + limited to `main` by a `branches` list under `on: push` cannot be checked by + review: to try a change to it, add the feature branch to that list and push, + then remove the branch from the list again before merging. Keep any job in it + that publishes behind `if: github.ref_name == 'main'`, so the run from the + feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -351,12 +448,11 @@ style conventions are in separate documents: That cache is Go's own, separate from Docker's layer cache. Go stores a passing result in its cache directory (`GOCACHE`), and when the same tests run again on unchanged code it prints that result, marked `(cached)`, - without running them. That matters on a developer's machine, where this - target runs and the directory lasts from one run to the next. The `test` - phase of the `Dockerfile` needs no `-count=1`: its base image holds no - result for this repo's tests and nothing before its `go test` step runs a - test, so there is nothing to replay. `--no-cache` (above) is what makes that - step run on an unchanged tree. + without running them. The directory lasts from one run to the next on a + developer's machine, where this target runs, and in the `test` phase of the + `Dockerfile`, which mounts it to keep compiled packages between check + builds, so the phase's `go test` runs carry `-count=1` too. A new + `CACHEBUST` value (above) is what makes that step run on an unchanged tree. Python example: @@ -384,9 +480,12 @@ style conventions are in separate documents: - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), - language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` - from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when - setting up a new repo. These patterns are written to `.gitignore`'s own + `node_modules/`, and the repo's own build outputs. Fetch the standard + `.gitignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up + a new repo. A repo's `.gitignore` is the standard file followed by the repo's + own entries, such as its binaries; a re-vendor replaces the standard part and + keeps those entries. These patterns are written to `.gitignore`'s own semantics, in which an unanchored pattern already matches at every depth; they are not a `.dockerignore` and must not be transplanted into one unmodified. @@ -434,13 +533,15 @@ style conventions are in separate documents: byte-identically across repos: ```sh - # Own line: a failing command substitution inside an argument does not - # trip `set -e`, so the inline form degrades to an empty constant. + # The version and the tag each get their own line: a failing command + # substitution inside an argument does not trip `set -e`, so the inline + # form degrades to an empty constant. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$(script/projectname)" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$(script/projectname)" . + -t "$tag" . ``` `--always` makes an untagged repo yield an abbreviated commit hash rather @@ -452,8 +553,8 @@ style conventions are in separate documents: there because `ARG` is stage-scoped; passing `VERSION` to a repo whose Dockerfile declares no such `ARG` is ignored and costs nothing, which is why the scripts stay byte-identical. One consequence for CI: the standard - checkout action clones shallow and fetches no tags, so a repo that embeds a - tag-derived version must set `fetch-depth: 0` on its checkout step. + checkout action clones shallow and fetches no tags, so the canonical + `.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step. - **Verify `.dockerignore` by enumerating the image, not by reading the patterns.** Plant files at the root _and_ at least two directories deep, build @@ -520,6 +621,11 @@ style conventions are in separate documents: function defined and never invoked has the same exit status and the same empty output as one that worked. + Node is the exception to the whole-token comparison: the canonical + `script/bootstrap` compares only its major version, because the node 22 + alpine image the `Dockerfile` stages start from is not the exact pin, and + nvm cannot install a prebuilt node on alpine. + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. A Go tool a repo needs on the host is installed with `go install` pinned to @@ -636,7 +742,11 @@ style conventions are in separate documents: Never edit existing migrations after release. - All repos should have an `.editorconfig` enforcing the project's indentation - settings. + settings: the standard file from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets + tabs for `Makefile` and Go files, followed by the repo's own sections, such as + one for another language it uses. A re-vendor replaces the standard part and + keeps those sections. - Avoid putting files in the repo root unless necessary. Root should contain only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, diff --git a/TODO.md b/TODO.md index 539dfe2..bc9859c 100644 --- a/TODO.md +++ b/TODO.md @@ -21,6 +21,19 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of # Completed Steps +- 2026-10-19: check builds keep Go's modules and compiled packages between runs + ([#118](https://git.eeqj.de/sneak/netwatch/issues/118)), on sneak's rulings in + [prompts#113](https://git.eeqj.de/sneak/prompts/issues/113) and + [prompts#130](https://git.eeqj.de/sneak/prompts/issues/130). + `REPO_POLICIES.md`, `script/test`, `script/lint`, `script/docker` and + `script/cibuild` are the sneak/prompts copies at `0653579`: the scripts pass a + new `CACHEBUST` value where they passed `--no-cache`, and the lint and test + builds write no image. In each stage of `Dockerfile`, `ARG CACHEBUST` comes + after the system packages and the Go module download, and before every step + that lints, tests or compiles and the node stages' `yarn install`. Go's build + cache and golangci-lint's are cache mounts, and both `go test` runs carry + `-count=1`. The test phase's Go caches used to be in memory, empty on every + build - 2026-10-18: when the backend runs on its own, a `DATA_DIR` that cannot be created or read, or where no file can be created, stops the start with an error naming `DATA_DIR` and its value, as the other settings' errors do @@ -614,9 +627,3 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of - A configurable host list (an environment variable or a config file) - Export of the latency history (CSV or JSON) - A notification when the health status changes to DEGRADED -- `make test` within the 20-second target - ([#118](https://git.eeqj.de/sneak/netwatch/issues/118)): every run compiles - the Go standard library and every dependency with the race detector from - nothing. Whether test builds may keep Go's compiled packages between runs - waits on sneak's ruling in - [prompts#113](https://git.eeqj.de/sneak/prompts/issues/113) diff --git a/script/cibuild b/script/cibuild index d8d3200..2656771 100755 --- a/script/cibuild +++ b/script/cibuild @@ -2,9 +2,12 @@ # script/cibuild: run the CI build. It bootstraps first: a CI runner # checks out and runs this and nothing else, and script/fmt-check runs # the formatter on the host, which a pristine checkout cannot do. -# --no-cache for the same reason as script/docker: the gate phases the -# final stage depends on are RUN steps, and a cached one is a check that -# did not run. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -14,15 +17,17 @@ main() { cd "$ROOT" "$SCRIPT_DIR/bootstrap" "$SCRIPT_DIR/check" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. The VERSION build argument takes precedence over - # the version a build stage derives from the .git in the context. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/docker b/script/docker index 07b626c..cc00d9c 100755 --- a/script/docker +++ b/script/docker @@ -1,8 +1,12 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# --no-cache because the gate phases the final stage depends on are RUN -# steps, and a cached one is a check that did not run. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -10,15 +14,17 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. The VERSION build argument takes precedence over - # the version a build stage derives from the .git in the context. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/lint b/script/lint index 2d8b075..451a067 100755 --- a/script/lint +++ b/script/lint @@ -5,9 +5,14 @@ # make its answer untrustworthy. # # The phase is not the last stage in the file, so it is built only when -# --target names it. --no-cache because a cached lint layer is a lint -# that did not run. The tag makes each build replace the previous image -# instead of leaving a dangling one behind. +# --target names it. A new CACHEBUST value reruns every RUN step after +# the ARG CACHEBUST line, since a cached lint layer is a lint that did +# not run; unlike --no-cache, it keeps the layers above that line and +# the cache mounts. This relies on every stage of the Dockerfile +# declaring ARG CACHEBUST before every step that lints, tests or +# compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. +# --output type=cacheonly writes no image, since nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -15,9 +20,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --no-cache \ + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --target lint \ - -t "$("$SCRIPT_DIR/projectname")-lint" . + --output type=cacheonly . } main "$@" diff --git a/script/test b/script/test index cd239f2..1622a11 100755 --- a/script/test +++ b/script/test @@ -2,8 +2,13 @@ # script/test: run the test suite. Testing is a phase of the Dockerfile # and this builds that phase alone, on the same terms as script/lint: # --target because a phase that is not the last stage is built only when -# named, --no-cache because a cached test layer is a test that did not -# run, and a tag so each build replaces the previous image. +# named, and a new CACHEBUST value because a cached test layer is a test +# that did not run; unlike --no-cache, it keeps the layers above the ARG +# CACHEBUST line and the cache mounts. This relies on every stage of the +# Dockerfile declaring ARG CACHEBUST before every step that lints, tests +# or compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. --output type=cacheonly writes no +# image, since nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -11,9 +16,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --no-cache \ + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --target test \ - -t "$("$SCRIPT_DIR/projectname")-test" . + --output type=cacheonly . } main "$@"