Run lint and tests as the lint and test phases of the Dockerfile, built with --no-cache (closes #202) #218

Merged
clawbot merged 1 commits from issue-202-dockerfile-check-phases into next 2026-10-05 03:41:51 +02:00
13 changed files with 175 additions and 156 deletions
Showing only changes of commit fe3063663c - Show all commits
+38 -30
View File
@@ -1,55 +1,62 @@
# Lint stage
# Same image as Dockerfile.lint: change both pins together.
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
# Lint phase. script/lint builds it alone. The linter is run directly:
# `make lint` and script/lint are themselves a docker build.
# golangci/golangci-lint:v2.12.2, 2026-10-04
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
# The linter compiles every package, and govips needs the libvips
# headers for that. REPO_POLICIES.md has the lint phase install them
# itself; this image is Debian, so with apt-get rather than apk.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN golangci-lint run --config .golangci.yml ./...
# Test phase. script/test builds it alone.
# golang:1.25.4-alpine, 2026-02-25
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test
WORKDIR /src
# script/bootstrap installs the build dependencies and downloads the Go
# modules. Only script/, go.mod and go.sum are copied first, so this
# layer is reused until one of them changes.
# script/bootstrap --cgo installs the build dependencies (a C compiler
# and the libvips and libheif headers) and downloads the Go modules.
COPY script/ ./script/
COPY go.mod go.sum ./
RUN script/bootstrap
RUN script/bootstrap --cgo
# Copy source code
COPY . .
# Tells script/lint it is inside a container, so it runs the linter.
ENV container=docker
# Without -v first; on a failure, again with -v for the details, and
# the step fails even if the second run passes.
RUN go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
# Run formatting check and linter. script/cibuild and script/docker pass
# a new CHECK_EPOCH on every run, and each check step names it in its
# command, so a new value reruns the step instead of reusing a cached
# success that checked nothing. A plain `docker build .` leaves it empty
# and reuses the check steps only for an identical build context.
ARG CHECK_EPOCH
RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
RUN echo "check epoch: ${CHECK_EPOCH}" && make lint
# Build stage
# Build stage. Nothing is wanted from the two phases above: these copies
# make BuildKit build them first, so this stage runs only when lint and
# test passed.
# golang:1.25.4-alpine, 2026-02-25
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
# Depend on lint stage passing
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
WORKDIR /src
# Build dependencies and Go modules, as in the lint stage
# Build dependencies and Go modules, as in the test phase
COPY script/ ./script/
COPY go.mod go.sum ./
RUN script/bootstrap
RUN script/bootstrap --cgo
# Copy source code
COPY . .
# Run tests; a new CHECK_EPOCH reruns them, as in the lint stage.
ARG CHECK_EPOCH
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
# VERSION is declared here, not earlier: a new value reruns only the
# build, not script/bootstrap or the tests. Given none, the version is
# build, not script/bootstrap. Given none, the version is
# `git describe --tags --always` of the .git in the build context (git
# comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH
# after one, the short commit when no tag is reachable. A context that
@@ -68,7 +75,8 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
-ldflags "-s -w -X main.Version=${version}" \
-o /pixad ./cmd/pixad
# Runtime stage
# Runtime stage, and the last one: a plain `docker build .` builds this
# stage and what it depends on, and nothing else.
# alpine:3.21, 2026-02-25
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
-34
View File
@@ -1,34 +0,0 @@
# Dockerfile.lint: the container script/lint builds to run golangci-lint,
# which is never installed on the host. Pinned to the same image as the
# Dockerfile lint stage: change both pins together, or the two run
# different linter versions.
#
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60
WORKDIR /src
# pixa is CGO/libvips: the type-aware linters compile every package, so
# this image needs the same C libraries the build does. script/bootstrap
# installs them and downloads the Go modules. Only script/, go.mod and
# go.sum are copied first; they settle this layer's result, so it may
# safely be reused between runs.
COPY script/ ./script/
COPY go.mod go.sum ./
RUN script/bootstrap
COPY . .
# Tells script/lint it is inside a container, so it runs the linter.
ENV container=docker
# script/lint passes a different CACHEBUST on every run, and BuildKit
# keys every RUN after this ARG on its value, so the lint step always
# runs instead of returning a cached success that linted nothing.
#
# Go's and golangci-lint's caches (/root/.cache, hundreds of MB) go on a
# tmpfs that is discarded after the step. Written into the layer, they
# would pile up as build cache on every run, since no later run, with
# its new CACHEBUST, can reuse that layer.
ARG CACHEBUST
RUN --mount=type=tmpfs,target=/root/.cache script/lint
+9 -9
View File
@@ -4,7 +4,7 @@ VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev
LDFLAGS := -X main.Version=$(VERSION)
# Use nix-shell to provide CGO dependencies unless they are already available
# (e.g. inside a Docker build or an existing nix-shell).
# (e.g. inside an existing nix-shell).
HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null)
ifdef HAS_PKGCONFIG
NIX_RUN_PREFIX =
@@ -32,11 +32,11 @@ fmt-check:
fmt:
@script/fmt
# Run linter
# Run linter (the lint phase of the Dockerfile)
lint:
@script/lint
# Run tests (30-second timeout)
# Run tests (the test phase of the Dockerfile)
test:
@script/test
@@ -64,20 +64,20 @@ docker-smoke:
loadtest:
@script/loadtest
# Build Docker image tagged pixad:$(VERSION) and pixad:latest
# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION)
docker-versioned:
docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
@script/docker
docker tag pixa pixa:$(VERSION)
# Run tests in Docker (needed for CGO/libvips)
# Run tests in Docker, as `make test` does
docker-test:
docker build --target builder --build-arg VERSION=$(VERSION) -t pixad-builder .
docker run --rm pixad-builder sh -c "CGO_ENABLED=1 GOTOOLCHAIN=auto go test -v ./..."
@script/test
# Run local dev server in Docker
devserver: docker-versioned devserver-stop
docker run -d --name pixad-dev -p 8080:8080 \
-v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
pixad:latest
pixa:latest
@echo "pixad running at http://localhost:8080"
# Stop dev server
+24 -11
View File
@@ -92,8 +92,11 @@ another part of pixa failed to stop. A request not finished by then is cut off.
Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
it uses libvips through CGO; building it also needs their development files,
`pkg-config` and a C compiler. `script/bootstrap` installs all of these with
nix, apt, brew or apk.
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
`script/setup` and `script/cibuild` run, installs only git, make and Go: the
checks compile pixa in Docker, so the host needs none of the C libraries. Docker
itself must already be installed.
## Running under upaas
@@ -565,28 +568,38 @@ standard: normalized scripts in `script/` are the entrypoints for the
development workflow, and the Makefile targets are thin shims that call
them. We provide:
- `script/bootstrap` — install all dependencies (idempotent)
- `script/bootstrap` — install git, make and Go and download the Go modules
(idempotent); with `--cgo`, also the C compiler and the libvips and libheif
libraries that compiling pixa needs
- `script/setup` — make a fresh clone ready for development
(bootstrap, then install-precommit)
- `script/projectname` — output the project name ("pixa")
- `script/test` — run the test suite
- `script/lint` — run golangci-lint, always in a container (builds
`Dockerfile.lint` when run outside one)
- `script/test` — run the test suite: build the `test` phase of the
`Dockerfile`, tagged `pixa-test`
- `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`,
tagged `pixa-lint`; the linter never runs on the host
- `script/fmt` — format all code (writes)
- `script/fmt-check` — check formatting (read-only)
- `script/fmt-check` — check formatting (read-only), on the host
- `script/check` — run test, lint, and fmt-check
- `script/docker` — build the Docker image tagged via `script/projectname`
- `script/docker` — build the Docker image tagged via `script/projectname`, with
the version from `git describe`; the image's build stage depends on the `lint`
and `test` phases, so this runs them too
- `script/docker-smoke` — build the image, start it, wait for it to be healthy
- `script/loadtest` — measure pixad's throughput, latency and peak memory; a
benchmark, not part of `script/check` (see Load Test)
- `script/cibuild` — CI entrypoint: `docker build .` with a new
`CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of
coming from the build cache, and a green run implies a green repo
- `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`),
then `script/check`, then build the image as `script/docker` does
- `script/precommit` — pre-commit checks (`go mod tidy` guard, then
`script/check`)
- `script/install-precommit` — install the git pre-commit hook that
runs `script/precommit`
Every `docker build` in these scripts passes `--no-cache`, so the lint and test
phases run on every build instead of coming from the build cache.
`script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`,
`script/setup` and `script/install-precommit` are the standard copies from
`sneak/prompts`, kept identical to them.
## Load Test
`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how
+18
View File
@@ -31,6 +31,24 @@ P2: security: per-IP rate limiting on the image routes
# Completed Steps
- 2026-10-05 lint and tests run as the `lint` and `test` phases of the
`Dockerfile`, built with `--no-cache` (closes #202): `script/check`,
`script/cibuild`, `script/docker`, `script/lint`, `script/test`,
`script/setup` and `script/install-precommit` are now the copies from
`sneak/prompts` `main`, unchanged. The `lint` phase runs golangci-lint from
the image `REPO_POLICIES.md` names, with `libvips-dev` from `apt-get`; the
`test` phase runs the tests with a 90-second timeout; the build stage depends
on both. `Dockerfile.lint` and the `CHECK_EPOCH` build argument are gone, the
formatting check runs on the host, and `make docker-versioned` and
`make docker-test` call the scripts. `script/bootstrap`, `script/fmt`,
`script/fmt-check`, `script/precommit` and `script/projectname` stay pixa's
own. `script/bootstrap` installs git, make and Go, refreshing apt's package
lists before its first apt install; with `--cgo`, which only the `test` phase
and the build stage pass, it also installs the C compiler and the libvips and
libheif libraries. The stage that compiles still takes the version from
`git describe` when no `VERSION` is given, per
https://git.eeqj.de/sneak/pixa/issues/166, so the copied scripts' comment that
`.dockerignore` leaves out `.git` does not hold for pixa.
- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]`
(`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check`
does not run, measures three scenarios, each against a new pixad container and
+26 -7
View File
@@ -4,11 +4,15 @@
# installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes NOTHING is present (not git,
# make, or go). The linter is never installed on the host: golangci-lint
# runs only inside a container, Dockerfile.lint or the Dockerfile lint
# stage (see script/lint). A C compiler and the CGO image libraries
# (pkg-config, vips, libheif) are installed for the govips bindings.
# Both Dockerfiles run this script too, so their build dependencies are
# the ones listed here.
# runs only in the lint phase of the Dockerfile (see script/lint).
#
# script/bootstrap git, make and Go, all the host needs: the
# checks compile pixa in Docker
# script/bootstrap --cgo also a C compiler and the CGO image
# libraries (pkg-config, vips, libheif) for
# the govips bindings, to compile pixa; the
# Dockerfile's test phase and build stage
# run this
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -35,6 +39,10 @@ detect_pkgmgr() {
if [ "$(id -u)" != "0" ]; then
SUDO="sudo"
fi
# This runs before the first install only. A fresh image, such
# as a CI runner's, has no package lists, and apt-get install
# finds no package without them.
$SUDO apt-get update
fi
}
@@ -71,7 +79,16 @@ ensure_cgo_deps() {
fi
}
usage() {
echo "usage: script/bootstrap [--cgo]" >&2
exit 2
}
main() {
case "$*" in
"" | --cgo) ;;
*) usage ;;
esac
cd "$ROOT"
# Base tooling
@@ -81,8 +98,10 @@ main() {
# Go toolchain
if missing go; then pkg_install go golang go go; fi
# CGO image libraries
ensure_cgo_deps
# CGO image libraries, only where pixa is compiled
if [ "$*" = "--cgo" ]; then
ensure_cgo_deps
fi
go mod download
+3 -2
View File
@@ -1,7 +1,8 @@
#!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. Must not modify any files.
# Generic: usually needs no adaptation.
# extension to scripts-to-rule-them-all. test and lint are Docker
# phases; fmt-check is native, because a formatter writes the working
# tree. Must not modify any files.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+20 -9
View File
@@ -1,18 +1,29 @@
#!/bin/sh
# script/cibuild: run the CI build. The Dockerfile runs the checks
# (make fmt-check, lint, test) as build steps. This script passes a new
# CHECK_EPOCH on every run, so Docker runs those steps instead of
# reusing cached results: a successful run means the checks ran and
# passed on this tree. Generic: needs no adaptation. The Gitea workflow
# runs this on push.
# script/cibuild: run the CI build. It bootstraps first: a CI runner
# checks out and runs this and nothing else, and script/fmt-check runs
# the formatter on the host, which a pristine checkout cannot do.
# --no-cache for the same reason as script/docker: the gate phases the
# final stage depends on are RUN steps, and a cached one is a check that
# did not run.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
epoch="$(date +%s)$$"
docker build --build-arg CHECK_EPOCH="$epoch" .
"$SCRIPT_DIR/bootstrap"
"$SCRIPT_DIR/check"
# Own line: a failing command substitution inside an argument does
# not trip `set -e`, so the inline form degrades silently to an
# empty constant. VERSION is computed here because .dockerignore
# excludes .git, so `git describe` in a build stage yields an empty
# version without failing.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$("$SCRIPT_DIR/projectname")" .
}
main "$@"
+12 -6
View File
@@ -1,9 +1,8 @@
#!/bin/sh
# script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname. Like
# script/cibuild, it passes a new CHECK_EPOCH, so the build runs the
# checks instead of reusing cached results. Generic: needs no
# adaptation.
# Identical in all repos; the tag comes from script/projectname.
# --no-cache because the gate phases the final stage depends on are RUN
# steps, and a cached one is a check that did not run.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -11,8 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
epoch="$(date +%s)$$"
docker build --build-arg CHECK_EPOCH="$epoch" \
# Own line: a failing command substitution inside an argument does
# not trip `set -e`, so the inline form degrades silently to an
# empty constant. VERSION is computed here because .dockerignore
# excludes .git, so `git describe` in a build stage yields an empty
# version without failing.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$("$SCRIPT_DIR/projectname")" .
}
+1 -1
View File
@@ -1,13 +1,13 @@
#!/bin/sh
# script/install-precommit: install the git pre-commit hook that runs
# script/precommit. Our own extension to scripts-to-rule-them-all.
# Generic: needs no adaptation.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
hook=".git/hooks/pre-commit"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
chmod +x .git/hooks/pre-commit
echo "pre-commit hook installed: runs script/precommit"
+13 -27
View File
@@ -1,37 +1,23 @@
#!/bin/sh
# script/lint: run golangci-lint over the whole tree. This is the only
# way the linter is run, everywhere; it is never installed on the host.
# script/lint: run the linter. Linting is a phase of the Dockerfile and
# this builds that phase alone; the linter is never installed or run on
# a developer host, where a shared result cache and a host-global lock
# make its answer untrustworthy.
#
# Inside a container it runs the linter. Anywhere else it builds
# Dockerfile.lint, whose last step runs this script again inside that
# container.
#
# Dockerfile.lint and the Dockerfile lint stage set container=docker
# (the systemd convention for marking a container) to say where we are.
# /.dockerenv cannot: it is missing inside build steps, and present on
# hosts that are themselves containers.
# The phase is not the last stage in the file, so it is built only when
# --target names it. --no-cache because a cached lint layer is a lint
# that did not run. The tag makes each build replace the previous image
# instead of leaving a dangling one behind.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
if [ "${container:-}" = docker ]; then
# `golangci-lint config verify` is not run: it fetches its JSON
# schema over an unpinned live HTTPS call, which REPO_POLICIES.md
# forbids.
echo "Running linter..."
golangci-lint run --config .golangci.yml ./...
else
# A new CACHEBUST on every run means the lint step is never
# served from cache (see Dockerfile.lint). The cacheonly output
# leaves no image behind.
docker build \
--progress=plain \
--build-arg CACHEBUST="$(date +%s)-$$" \
--output=type=cacheonly \
-f Dockerfile.lint .
fi
docker build --no-cache \
--target lint \
-t "$("$SCRIPT_DIR/projectname")-lint" .
}
main "$@"
+1 -2
View File
@@ -1,7 +1,6 @@
#!/bin/sh
# script/setup: set up the repo for development after a fresh clone:
# installs dependencies (script/bootstrap) and the git pre-commit hook.
# Add any repo-specific initialization (db init, .env template) here.
# installs dependencies and the git pre-commit hook.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+10 -18
View File
@@ -1,27 +1,19 @@
#!/bin/sh
# script/test: run the test suite. CGO dependencies (pkg-config, vips,
# libheif) come from nix-shell when not already available (e.g. inside
# a Docker build or an existing nix-shell).
# script/test: run the test suite. Testing is a phase of the Dockerfile
# and this builds that phase alone, on the same terms as script/lint:
# --target because a phase that is not the last stage is built only when
# named, --no-cache because a cached test layer is a test that did not
# run, and a tag so each build replaces the previous image.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
run_with_cgo_deps() {
if command -v pkg-config >/dev/null 2>&1; then
sh -c "$1"
else
nix-shell -p pkg-config vips libheif git --run "$1"
fi
}
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
echo "Running tests..."
# Run without -v first for clean output on success; on failure rerun
# with -v for full diagnostics, then exit non-zero (REPO_POLICIES.md
# conditional-verbose-rerun pattern). The first run already proved the
# tests broken, so the build fails even if the rerun happens to pass.
run_with_cgo_deps "CGO_ENABLED=1 go test -timeout 30s -race -cover ./... || { echo '--- Rerunning with -v for details ---'; CGO_ENABLED=1 go test -timeout 30s -race -v ./...; exit 1; }"
docker build --no-cache \
--target test \
-t "$("$SCRIPT_DIR/projectname")-test" .
}
main "$@"