diff --git a/.editorconfig b/.editorconfig index 92ec261..de84fae 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,3 +13,6 @@ indent_style = tab [*.go] indent_style = tab + +# This repository's own sections, such as one for another language it +# uses, go below this comment, and a re-vendor keeps them. diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index ee73864..2e96965 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -1,9 +1,20 @@ name: check on: [push] +# Free the shared runner: a new push cancels only the same branch's older run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: runs-on: ubuntu-latest + # Free the shared runner from a hung build. + timeout-minutes: 20 steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # script/cibuild needs no token, so none is left in .git/config. + with: + persist-credentials: false + # All history and tags, so git describe finds the version tag. + fetch-depth: 0 - run: script/cibuild diff --git a/.gitignore b/.gitignore index 41a70b3..5fedbfc 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ node_modules/ # Environment files. `*.env` covers bare `.env` and the `prod.env` # convention. Only the templates `example.env` and `sample.env` are # re-included below. A repository that commits any other template adds -# its own negation after these lines, for example `!.env.example`. +# its own negation at the end of this file, for example `!.env.example`. *.[eE][nN][vV] .[eE][nN][vV].* .[eE][nN][vV][rR][cC] @@ -46,6 +46,10 @@ node_modules/ [iI][dD]_[eE][dD]25519 [iI][dD]_[eE][dD]25519_[sS][kK] +# This repository's own entries, such as its build outputs, go below +# this comment, and a re-vendor keeps them. Anchor a binary built at the +# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`. + # Go. /secret is the built binary, anchored so that it does not also match # the internal/secret/ package directory. *.log diff --git a/.golangci.yml b/.golangci.yml index 1b73eb9..32db9fe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -25,6 +25,8 @@ linters: # silenced by disabling that name, not by enabling the successor. - wsl # Deprecated, replaced by wsl_v5 - gomodguard # Deprecated, replaced by gomodguard_v2 + # Misses findings at random in v2.14.0; back once a pinned release fixes it + - canonicalheader settings: lll: line-length: 88 diff --git a/Dockerfile b/Dockerfile index 3c03d0b..5a1d81b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,40 @@ +# Every stage declares ARG CACHEBUST first: script/ passes a new value +# on every build, which reruns every RUN step after it. The steps that +# compile mount Go's build cache, which survives from build to build. + # Lint phase. The linter is invoked directly rather than through `make # lint` or `script/lint`, which are themselves a docker build. # golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24 FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint +ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go vet ./... -RUN golangci-lint run --config .golangci.yml ./... +RUN --mount=type=cache,target=/root/.cache/go-build \ + go vet ./... +RUN --mount=type=cache,target=/root/.cache/go-build \ + golangci-lint run --config .golangci.yml ./... # The same checks on the code as a macOS build compiles it, which a Linux # build never compiles. Cgo is off, because compiling cgo code for macOS # needs Apple's SDK headers. -RUN GOOS=darwin CGO_ENABLED=0 go vet ./... -RUN GOOS=darwin CGO_ENABLED=0 golangci-lint run --config .golangci.yml ./... +RUN --mount=type=cache,target=/root/.cache/go-build \ + GOOS=darwin CGO_ENABLED=0 go vet ./... +RUN --mount=type=cache,target=/root/.cache/go-build \ + GOOS=darwin CGO_ENABLED=0 golangci-lint run --config .golangci.yml ./... # Test phase. -race needs cgo and so a C compiler, which the Debian Go -# image ships and the alpine one does not. +# image ships and the alpine one does not. -count=1 because the mounted +# cache also holds test results, which go test would otherwise replay. # golang:1.24.13-trixie, 2026-02-04 FROM golang@sha256:5835f052b784aa39f2fe9070def3568605c8bc3fcd810f10402066348b61e716 AS test +ARG CACHEBUST ENV CGO_ENABLED=1 WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -# Go's build cache goes in a cache mount, not the image layer, which would -# take seconds longer to export. --no-cache, on every build in script/, -# starts the mount empty; -count=1 keeps a build without it from taking -# test results from there. -RUN --mount=type=cache,id=sneak/secret/go-build-test,target=/root/.cache/go-build \ +RUN --mount=type=cache,target=/root/.cache/go-build \ go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ go test -count=1 -timeout 90s -race -v ./...; exit 1; } @@ -37,6 +44,7 @@ RUN --mount=type=cache,id=sneak/secret/go-build-test,target=/root/.cache/go-buil # lint and test passed. # golang 1.24.13-alpine, 2026-03-10 FROM golang@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191 AS builder +ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null # script/build compiles with cgo, so it needs a C compiler too. @@ -53,7 +61,8 @@ COPY . . # .git present, a version that is still empty, dev or unknown fails the # build: git is missing or could not read the checkout. ARG VERSION -RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ if [ -e .git ]; then \ case "$VERSION" in ""|dev|unknown) \ echo "version is '$VERSION' although .git is present" >&2; \ @@ -65,6 +74,7 @@ RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ # Runtime stage, and the last one # alpine 3.23, 2026-03-10 FROM alpine@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 +ARG CACHEBUST RUN apk add --no-cache ca-certificates gnupg diff --git a/README.md b/README.md index a716bb6..3ef7826 100644 --- a/README.md +++ b/README.md @@ -675,6 +675,10 @@ provide: - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` +Every stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after +`FROM`. The `script/` builds pass a new `CACHEBUST` value on every build, which +reruns every `RUN` step and keeps Go's build cache; `--no-cache` would empty it. + ## Features - **Multiple Authentication Methods**: Supports passphrase and PGP unlockers diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 20382d1..5b24098 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-10-04 +last_modified: 2026-10-15 --- This document covers repository structure, tooling, and workflow standards. Code @@ -54,34 +54,39 @@ style conventions are in separate documents: `cibuild`. `script/bootstrap` installs all dependencies idempotently and assumes nothing is present: base tools come from nix, apt, brew, or apk (detected in that order; apt runs noninteractive). For node it uses the - installed node if present; otherwise it installs a PINNED node version via - nvm, first installing nvm itself if missing — from a hash-verified GitHub - release archive (never `curl | sh`), with bash installed as an explicit - prerequisite since nvm requires bash. yarn is then pinned via - `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; - always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root, runs `script/bootstrap`, runs `script/check`, and builds the image - with the version; the Gitea workflow calls it. **`script/cibuild` runs + installed node only when its major version is the pinned one; otherwise (node + missing, or another major, such as the node 24 the CI runner image ships) it + installs the PINNED node version via nvm and installs yarn and the packages + under it. nvm itself is installed first if missing, from a hash-verified + GitHub release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. Only the major version is compared + because the `Dockerfile` stages start from a node image whose exact version is + not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned + via `corepack prepare yarn@ --activate`. Never install "latest" or + "lts"; always exact versions. `script/cibuild` runs the CI build: it changes + to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the + image with the version; the Gitea workflow calls it. **`script/cibuild` runs `script/bootstrap` first**, because the workflow checks out the repo and runs nothing else, while `script/fmt-check` runs the formatter on the host: on a pristine checkout with nothing installed the run dies there, after the - containerised gates have passed. **The bootstrap alone is not enough**: - `script/bootstrap` installs node and yarn under nvm and leaves neither on the - `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host - entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore - source nvm for the pinned node version before invoking it, exactly as - `script/bootstrap`'s own install step does. A runner carrying nothing but - docker and git then gets through `script/check`. Four further scripts are our - own extensions to the standard: `script/check` runs `script/test`, - `script/lint` and `script/fmt-check`; `script/precommit` is what the git - pre-commit hook runs, and it calls `script/check`; `script/install-precommit` - installs the git pre-commit hook (the `make hooks` target shims to it); and - `script/projectname` (literally that filename) simply outputs the project's - name. Scripts that need the name call `script/projectname` — e.g. - `script/docker` assembles its image tag from it — so those scripts stay - byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. - `go mod tidy` verification in Go repos) belong in `script/precommit`, not in - the hook itself. Model scripts are at + containerised gates have passed. **The bootstrap alone is not enough**: when + `script/bootstrap` installs node and yarn under nvm it leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs + under another node. The host entrypoints that need yarn — `script/fmt` and + `script/fmt-check` — therefore use the same test as `script/bootstrap`: when + the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, + and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the + pinned node. A runner carrying nothing but docker and git then gets through + `script/check`. Four further scripts are our own extensions to the standard: + `script/check` runs `script/test`, `script/lint` and `script/fmt-check`; + `script/precommit` is what the git pre-commit hook runs, and it calls + `script/check`; `script/install-precommit` installs the git pre-commit hook + (the `make hooks` target shims to it); and `script/projectname` (literally + that filename) simply outputs the project's name. Scripts that need the name + call `script/projectname` — e.g. `script/docker` assembles its image tag from + it — so those scripts stay byte-identical across all repos. Repo-type-specific + pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in + `script/precommit`, not in the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -118,8 +123,10 @@ style conventions are in separate documents: and nothing else: ```sh - docker build --no-cache --target lint -t "$(script/projectname)-lint" . - docker build --no-cache --target test -t "$(script/projectname)-test" . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint --output type=cacheonly . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test --output type=cacheonly . ``` **A stage that is not the last one in the file is built only when the final @@ -129,10 +136,15 @@ style conventions are in separate documents: plain `docker build .` builds the last stage alone and exits 0 having linted and tested nothing. - **Every `docker build` in `script/` is tagged**, here and in - `script/cibuild` and `script/docker`. An untagged build leaves a dangling - image behind on every invocation, on every developer host and every CI - runner; a tagged one replaces the previous image. + **The gate builds write no image.** With `--output type=cacheonly` the phase + runs and a failing step fails the build, but the result is not exported. + Nothing uses those images, and writing one out is slow: a Go test phase's + image holds the toolchain. A build given neither `--output` nor `-t` writes + an untagged image and leaves it dangling, on every developer host and every + CI runner. `script/cibuild` and `script/docker` build the image that ships + and tag it, so each build replaces the previous image; each assigns the tag + on its own line before the build, so `set -e` stops it where + `script/projectname` fails. Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, `eslint`, `prettier` — never through `make lint` or `script/test`, which are @@ -147,22 +159,33 @@ style conventions are in separate documents: one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs exit non-zero with `parallel golangci-lint is running` — a status a caller cannot tell from real findings. Both have produced wrong verdicts in this - org, in both directions. A container has its own cache, its own `TMPDIR` and - a digest-pinned binary, so neither is reachable. + org, in both directions. A container has its own golangci-lint cache, its + own `TMPDIR` and a digest-pinned binary, so neither is reachable. -- **Any build that runs checks is built with `--no-cache`.** Docker invalidates - a `COPY` layer only when the copied content changes, so on an unchanged tree - the check `RUN` is served from cache, nothing executes, and the build still - exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: - `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the - four, and there is no fifth — `script/check` runs the two gate phases and - `script/fmt-check`, and builds no image of its own. A bare `docker build .` is - not evidence that anything ran: a sub-second build reporting success is a - cache hit, not a result. Never invalidate by pruning — `docker builder prune` - and friends destroy a build cache shared with every other build on the host. - When a check is added or changed, prove it works by planting a defect it must - catch and watching the run fail on it, then revert the defect. A green run - alone shows neither that the check ran nor that it covers what it should. +- **Any build that runs checks is given a new `CACHEBUST` value.** Docker + invalidates a `COPY` layer only when the copied content changes, so on an + unchanged tree the check `RUN` is served from cache, nothing executes, and the + build still exits 0. Every `docker build` in `script/` therefore passes + `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process + id, so no two builds share a value, not even the three in one + `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and + `script/docker` are the four, and there is no fifth — `script/check` runs the + two gate phases and `script/fmt-check`, and builds no image of its own. Every + stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after + `FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps + are compared by content and need nothing. A stage without the `ARG` line is + served from cache on an unchanged tree, so a repository re-vendors the scripts + and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`: + it reruns every step too, but it also empties every cache mount, and Go's + build cache, which the canonical Go `Dockerfile` below mounts so that compiled + packages survive between check builds, would start empty every time. A bare + `docker build .` is not evidence that anything ran: a sub-second build + reporting success is a cache hit, not a result. Never invalidate by pruning — + `docker builder prune` and friends destroy a build cache shared with every + other build on the host. When a check is added or changed, prove it works by + planting a defect it must catch and watching the run fail on it, then revert + the defect. A green run alone shows neither that the check ran nor that it + covers what it should. - **The gate phases are separate stages, and the build stage depends on both.** The lint phase is based on the `golangci/golangci-lint` image (pinned by @@ -171,32 +194,42 @@ style conventions are in separate documents: `Dockerfile`: ```dockerfile + # Every stage declares ARG CACHEBUST first: script/ passes a new value + # on every build, which reruns every RUN step after it. The steps that + # compile mount Go's build cache, which survives from build to build. + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint + ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN golangci-lint run --config .golangci.yml ./... + RUN --mount=type=cache,target=/root/.cache/go-build \ + golangci-lint run --config .golangci.yml ./... # Test phase. -race needs cgo and so a C compiler, which the Debian Go - # image ships and the alpine one does not. + # image ships and the alpine one does not. -count=1 because the mounted + # cache also holds test results, which go test would otherwise replay. # golang:1.x, YYYY-MM-DD FROM golang@sha256:... AS test + ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN go test -timeout 90s -race -cover ./... || \ + RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 90s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } # Build stage. Nothing is wanted from either phase above; the copies # are what make BuildKit build them first, so this stage cannot run # unless lint and test passed. # golang:1.x-alpine, YYYY-MM-DD FROM golang@sha256:... AS builder + ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null RUN apk add --no-cache git @@ -212,7 +245,8 @@ style conventions are in separate documents: # .git present, a version that is still empty, dev or unknown fails the # build: git is missing or could not read the checkout. ARG VERSION - RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ if [ -e .git ]; then \ case "$VERSION" in ""|dev|unknown) \ echo "version is '$VERSION' although .git is present" >&2; \ @@ -225,6 +259,7 @@ style conventions are in separate documents: # Runtime stage, and the last one FROM alpine@sha256:... + ARG CACHEBUST COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` @@ -239,6 +274,16 @@ style conventions are in separate documents: - Keep the runtime stage last, and if you add a stage after it, give it the same two copies. A plain `docker build .` builds the last stage's chain and nothing else. + - Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps + it for root), and only on the steps that compile: `golangci-lint run`, + both `go test` runs and `go build`. It keeps compiled packages between + check builds, and Go reuses one only when all its inputs are unchanged, so + it saves the compile without skipping a check. Nothing else is kept: + golangci-lint's own cache would replay stored findings in place of a lint + run, and Go builds from whatever files a kept module cache holds without + checking them again (only `go mod verify` does), so a file changed there + would go unnoticed. Go's build cache also holds test results, which is why + both `go test` runs carry `-count=1`. - If the project uses `//go:embed` directives that reference build artifacts (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: @@ -278,20 +323,39 @@ style conventions are in separate documents: `.git` and the version still comes out empty, `dev` or `unknown`. A plain `docker build .` with no build arguments must succeed; a Dockerfile that refuses an empty build argument drops that refusal and keeps the argument. + A checkout whose `.git` is a file (a linked worktree, or a repository + checked out as a submodule) is the exception: that file points to a git + directory outside the build context, so the build cannot read the version + and a plain `docker build .` fails; pass the version with + `--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already + do. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` on push, and checks out the repo as its only other step. - That script bootstraps, runs the gate phases, and then builds the image, so a - successful run means every check passed; a bare `docker build .` does not + runs `script/cibuild` on push, and checks out the repo as its only other step, + with `persist-credentials: false`: `script/cibuild` needs no token, and + without it the checkout leaves the job's token in `.git/config` for every + later step. The checkout step also sets `fetch-depth: 0`, which fetches the + tags `git describe` needs: by default it clones shallow with no tags, and a + tagged repository's CI build would stamp a bare short commit id. The + workflow's `concurrency` block groups runs by workflow and branch + (`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`, + so a new push cancels the older run on the same branch, queued or running, and + no other: runs for replaced commits do not hold up the shared runner. + `script/cibuild` bootstraps, runs the gate phases, and then builds the image, + so a successful run means every check passed; a bare `docker build .` does not carry the same guarantee, because its gate phases may come from the cache. The - image build is uncached and so runs the gate phases a second time. That is the - price of the rule above, and it is worth paying: the image that ships is built - from a run of its own gates rather than from a cache entry. A separate - workflow limited to `main` by a `branches` list under `on: push` cannot be - checked by review: to try a change to it, add the feature branch to that list - and push, then remove the branch from the list again before merging. Keep any - job in it that publishes behind `if: github.ref_name == 'main'`, so the run - from the feature branch publishes nothing. + image build gets a `CACHEBUST` value of its own and so runs the gate phases a + second time. That is the price of the rule above, and it is worth paying: the + image that ships is built from a run of its own gates rather than from a cache + entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the + shared runner after 20 minutes. That allows for the three Docker builds + described above (the test phase, the lint phase, then the image), each held to + the 5-minute Docker build limit below, plus the bootstrap. A separate workflow + limited to `main` by a `branches` list under `on: push` cannot be checked by + review: to try a change to it, add the feature branch to that list and push, + then remove the branch from the list again before merging. Keep any job in it + that publishes behind `if: github.ref_name == 'main'`, so the run from the + feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -351,12 +415,11 @@ style conventions are in separate documents: That cache is Go's own, separate from Docker's layer cache. Go stores a passing result in its cache directory (`GOCACHE`), and when the same tests run again on unchanged code it prints that result, marked `(cached)`, - without running them. That matters on a developer's machine, where this - target runs and the directory lasts from one run to the next. The `test` - phase of the `Dockerfile` needs no `-count=1`: its base image holds no - result for this repo's tests and nothing before its `go test` step runs a - test, so there is nothing to replay. `--no-cache` (above) is what makes that - step run on an unchanged tree. + without running them. The directory lasts from one run to the next on a + developer's machine, where this target runs, and in the `test` phase of the + `Dockerfile`, which mounts it to keep compiled packages between check + builds, so the phase's `go test` runs carry `-count=1` too. A new + `CACHEBUST` value (above) is what makes that step run on an unchanged tree. Python example: @@ -384,9 +447,12 @@ style conventions are in separate documents: - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), - language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` - from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when - setting up a new repo. These patterns are written to `.gitignore`'s own + `node_modules/`, and the repo's own build outputs. Fetch the standard + `.gitignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up + a new repo. A repo's `.gitignore` is the standard file followed by the repo's + own entries, such as its binaries; a re-vendor replaces the standard part and + keeps those entries. These patterns are written to `.gitignore`'s own semantics, in which an unanchored pattern already matches at every depth; they are not a `.dockerignore` and must not be transplanted into one unmodified. @@ -434,13 +500,15 @@ style conventions are in separate documents: byte-identically across repos: ```sh - # Own line: a failing command substitution inside an argument does not - # trip `set -e`, so the inline form degrades to an empty constant. + # The version and the tag each get their own line: a failing command + # substitution inside an argument does not trip `set -e`, so the inline + # form degrades to an empty constant. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$(script/projectname)" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$(script/projectname)" . + -t "$tag" . ``` `--always` makes an untagged repo yield an abbreviated commit hash rather @@ -452,8 +520,8 @@ style conventions are in separate documents: there because `ARG` is stage-scoped; passing `VERSION` to a repo whose Dockerfile declares no such `ARG` is ignored and costs nothing, which is why the scripts stay byte-identical. One consequence for CI: the standard - checkout action clones shallow and fetches no tags, so a repo that embeds a - tag-derived version must set `fetch-depth: 0` on its checkout step. + checkout action clones shallow and fetches no tags, so the canonical + `.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step. - **Verify `.dockerignore` by enumerating the image, not by reading the patterns.** Plant files at the root _and_ at least two directories deep, build @@ -520,6 +588,11 @@ style conventions are in separate documents: function defined and never invoked has the same exit status and the same empty output as one that worked. + Node is the exception to the whole-token comparison: the canonical + `script/bootstrap` compares only its major version, because the node 22 + alpine image the `Dockerfile` stages start from is not the exact pin, and + nvm cannot install a prebuilt node on alpine. + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. A Go tool a repo needs on the host is installed with `go install` pinned to @@ -636,7 +709,11 @@ style conventions are in separate documents: Never edit existing migrations after release. - All repos should have an `.editorconfig` enforcing the project's indentation - settings. + settings: the standard file from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets + tabs for `Makefile` and Go files, followed by the repo's own sections, such as + one for another language it uses. A re-vendor replaces the standard part and + keeps those sections. - Avoid putting files in the repo root unless necessary. Root should contain only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, diff --git a/TODO.md b/TODO.md index fad1c4e..3b1b1e1 100644 --- a/TODO.md +++ b/TODO.md @@ -17,6 +17,18 @@ https://git.eeqj.de/sneak/secret/milestone/12 # Completed Steps +- 2026-10-15: The canonical files are re-vendored from `sneak/prompts` commit + `e3f79c4` (https://git.eeqj.de/sneak/secret/issues/129), so Go's build cache + survives between check builds and `make test` no longer compiles the standard + library and every dependency with `-race` on every run. `script/lint`, + `script/test`, `script/cibuild` and `script/docker` pass a new `CACHEBUST` + value instead of `--no-cache`, and every stage of the `Dockerfile` declares + `ARG CACHEBUST` first, so every `RUN` step still reruns on an unchanged tree. + The steps that compile mount Go's build cache, and both `go test` runs keep + `-count=1`, so no test result is reused. `script/lint` and `script/test` write + no image. The CI workflow cancels a branch's older run, stops after 20 + minutes, fetches the tags and leaves no token behind. `script/bootstrap` uses + the installed node only when its major version is the pinned one. - 2026-10-15: `secret mv` from one vault to another unlocks both vaults before it takes the state directory lock (https://git.eeqj.de/sneak/secret/issues/189). It unlocked them while holding diff --git a/package.json b/package.json index dc05cde..fb8b622 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,5 @@ { + "private": true, "devDependencies": { "prettier": "3.8.1" } diff --git a/script/bootstrap b/script/bootstrap index 0b414e7..22c605a 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -2,18 +2,18 @@ # script/bootstrap: install all dependencies needed to build and develop # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, -# or apk (detected in that order); assumes NOTHING is present (not git, -# make, node, yarn, go, or python). Node is used directly if installed; -# otherwise a pinned version is installed via nvm (installing nvm -# itself first, from a hash-verified release archive, never curl | sh). +# or apk (detected in that order); assumes nothing is present. The +# installed node is used only when its major version is the pinned one; +# otherwise (node missing, or another major) the pinned version is +# installed via nvm (installing nvm itself first, from a hash-verified +# release archive, never curl | sh), and yarn and the packages are +# installed under it. # golangci-lint is never installed: script/lint runs it in docker. -# -# Uncomment the language sections in main() that apply to this repo. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions. +# Pinned versions, 2026-07-06 NODE_VERSION="22.17.0" NVM_VERSION="0.40.3" # sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz @@ -22,6 +22,7 @@ YARN_VERSION="1.22.22" PKGMGR="" SUDO="" +APT_UPDATED="" detect_pkgmgr() { [ -n "$PKGMGR" ] && return 0 @@ -50,7 +51,14 @@ pkg_install() { detect_pkgmgr case "$PKGMGR" in nix) nix-env -iA "nixpkgs.$1" ;; - apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;; + apt) + # Package lists may be empty (fresh images); refresh once per run. + if [ -z "$APT_UPDATED" ]; then + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update + APT_UPDATED=1 + fi + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" + ;; brew) brew install "$3" ;; apk) apk add --no-cache "$4" ;; esac @@ -82,7 +90,7 @@ nvm_sh() { ensure_nvm() { [ -s "$HOME/.nvm/nvm.sh" ] && return 0 - # nvm prerequisites; nvm itself requires bash, so install it too + # nvm prerequisites; nvm itself requires bash if missing bash; then pkg_install bash bash bash bash; fi if missing curl; then pkg_install curl curl curl curl; fi if missing git; then pkg_install git git git git; fi @@ -95,56 +103,68 @@ ensure_nvm() { rm -rf "$tmp" } +# True when the node on PATH has the pinned major version. Only the +# major is compared: the Dockerfile stages start from a node 22 alpine +# image whose exact version is not the pin, and nvm cannot install a +# prebuilt node on alpine. Another major is not used: the pinned yarn 1 +# prints a deprecation warning under node 24. script/fmt and +# script/fmt-check carry this function unchanged, to run the yarn +# installed here. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + ensure_node() { - if ! missing node; then return 0; fi + if node_is_pinned_major; then + echo "bootstrap: using node $(node --version)" + return 0 + fi ensure_nvm nvm_sh "nvm install $NODE_VERSION" + nvm_sh "nvm use $NODE_VERSION >/dev/null && \ + echo \"bootstrap: using node \$(node --version)\"" } ensure_yarn() { + if ! node_is_pinned_major; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + return 0 + fi if ! missing yarn; then return 0; fi if ! missing corepack; then corepack enable corepack prepare "yarn@$YARN_VERSION" --activate - elif [ -s "$HOME/.nvm/nvm.sh" ]; then - nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ - corepack prepare yarn@$YARN_VERSION --activate" else npm install -g "yarn@$YARN_VERSION" fi } install_js_deps() { - if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + if node_is_pinned_major; then + yarn install --frozen-lockfile + else nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ yarn install --frozen-lockfile" - else - yarn install --frozen-lockfile fi } main() { cd "$ROOT" - # Base tooling (every repo) - if missing git; then pkg_install git git git git; fi if missing make; then pkg_install gnumake make make make; fi + if missing git; then pkg_install git git git git; fi - # ---- JS / docs repos ---- - # prettier, pinned in package.json and yarn.lock, formats the markdown ensure_node ensure_yarn install_js_deps - # ---- Go repos ---- if missing go; then pkg_install go golang go go; fi go mod download - # ---- Python repos ---- - # if missing python3; then pkg_install python3 python3 python3 python3; fi - # python3 -m venv .venv - # ./.venv/bin/pip install -e '.[dev]' - echo "bootstrap complete" } diff --git a/script/cibuild b/script/cibuild index d8d3200..66fcca0 100755 --- a/script/cibuild +++ b/script/cibuild @@ -2,9 +2,11 @@ # script/cibuild: run the CI build. It bootstraps first: a CI runner # checks out and runs this and nothing else, and script/fmt-check runs # the formatter on the host, which a pristine checkout cannot do. -# --no-cache for the same reason as script/docker: the gate phases the -# final stage depends on are RUN steps, and a cached one is a check that -# did not run. +# A new CACHEBUST value reruns every RUN step, since a cached gate phase +# is a check that did not run; unlike --no-cache, it keeps Go's build cache. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# first; a stage without that line is served from cache on an unchanged +# tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -14,15 +16,17 @@ main() { cd "$ROOT" "$SCRIPT_DIR/bootstrap" "$SCRIPT_DIR/check" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. The VERSION build argument takes precedence over - # the version a build stage derives from the .git in the context. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/docker b/script/docker index 07b626c..360037a 100755 --- a/script/docker +++ b/script/docker @@ -1,8 +1,11 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# --no-cache because the gate phases the final stage depends on are RUN -# steps, and a cached one is a check that did not run. +# A new CACHEBUST value reruns every RUN step, since a cached gate phase +# is a check that did not run; unlike --no-cache, it keeps Go's build cache. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# first; a stage without that line is served from cache on an unchanged +# tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -10,15 +13,17 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. The VERSION build argument takes precedence over - # the version a build stage derives from the .git in the context. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/fmt b/script/fmt index a9e7adf..9906883 100755 --- a/script/fmt +++ b/script/fmt @@ -8,15 +8,25 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" # Must match the pin in script/bootstrap. NODE_VERSION="22.17.0" -# script/bootstrap installs node and yarn under nvm and leaves neither -# on the PATH of the shell that called it, so resolve the pinned -# toolchain here the way bootstrap's own install step does. nvm is a -# bash script, hence the subshell. +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. run_yarn() { - if command -v yarn >/dev/null 2>&1; then + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then exec yarn "$@" fi - if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then echo "fmt: no yarn; run script/bootstrap first" >&2 exit 1 fi diff --git a/script/fmt-check b/script/fmt-check index ddbba8b..b110a76 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -8,15 +8,25 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" # Must match the pin in script/bootstrap. NODE_VERSION="22.17.0" -# script/bootstrap installs node and yarn under nvm and leaves neither -# on the PATH of the shell that called it, so resolve the pinned -# toolchain here the way bootstrap's own install step does. nvm is a -# bash script, hence the subshell. +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. run_yarn() { - if command -v yarn >/dev/null 2>&1; then + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then exec yarn "$@" fi - if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then echo "fmt-check: no yarn; run script/bootstrap first" >&2 exit 1 fi diff --git a/script/install-precommit b/script/install-precommit index 3519de6..bef6406 100755 --- a/script/install-precommit +++ b/script/install-precommit @@ -1,7 +1,6 @@ #!/bin/sh # script/install-precommit: install the git pre-commit hook that runs # script/precommit. Our own extension to scripts-to-rule-them-all. -# Generic: needs no adaptation. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/script/lint b/script/lint index 2d8b075..c4c459e 100755 --- a/script/lint +++ b/script/lint @@ -5,9 +5,12 @@ # make its answer untrustworthy. # # The phase is not the last stage in the file, so it is built only when -# --target names it. --no-cache because a cached lint layer is a lint -# that did not run. The tag makes each build replace the previous image -# instead of leaving a dangling one behind. +# --target names it. A new CACHEBUST value reruns every RUN step, since +# a cached lint layer is a lint that did not run; unlike --no-cache, it +# keeps Go's build cache. This relies on every stage of the Dockerfile +# declaring ARG CACHEBUST first; a stage without that line is served +# from cache on an unchanged tree, with no warning. +# --output type=cacheonly writes no image, since nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -15,9 +18,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --no-cache \ + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --target lint \ - -t "$("$SCRIPT_DIR/projectname")-lint" . + --output type=cacheonly . } main "$@" diff --git a/script/setup b/script/setup index 53327ba..4cc5b6b 100755 --- a/script/setup +++ b/script/setup @@ -1,7 +1,6 @@ #!/bin/sh # script/setup: set up the repo for development after a fresh clone: -# installs dependencies (script/bootstrap) and the git pre-commit hook. -# Add any repo-specific initialization (db init, .env template) here. +# installs dependencies and the git pre-commit hook. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/test b/script/test index cd239f2..a56c4a1 100755 --- a/script/test +++ b/script/test @@ -2,8 +2,12 @@ # script/test: run the test suite. Testing is a phase of the Dockerfile # and this builds that phase alone, on the same terms as script/lint: # --target because a phase that is not the last stage is built only when -# named, --no-cache because a cached test layer is a test that did not -# run, and a tag so each build replaces the previous image. +# named, and a new CACHEBUST value because a cached test layer is a test +# that did not run; unlike --no-cache, it keeps Go's build cache. This +# relies on every stage of the Dockerfile declaring ARG CACHEBUST first; +# a stage without that line is served from cache on an unchanged tree, +# with no warning. --output type=cacheonly writes no image, since +# nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -11,9 +15,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --no-cache \ + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --target test \ - -t "$("$SCRIPT_DIR/projectname")-test" . + --output type=cacheonly . } main "$@"