Author SHA1 Message Date
sneak eb5025370f Test the remaining CLI cases (closes #16)
check / check (push) Failing after 10s
Most of the command-line contract was already tested through run. This
adds what was missing: report and trees with no database exit 1 with
the message telling the user to run scan; a scan that skips an
unreadable file prints its warning and counts the skip in its summary;
the report and trees summary lines are checked exactly. The fatal-path
test now takes its subcommands from the command tree, so a new
subcommand wired without runE fails it. The nonexistent-operand test
gets an accurate name.

Model: opus-5-5
2026-10-04 14:17:22 +00:00
35 changed files with 1743 additions and 2076 deletions
+5
View File
@@ -0,0 +1,5 @@
{
"worktree": {
"bgIsolation": "none"
}
}
+6 -73
View File
@@ -1,79 +1,12 @@
# .dockerignore does NOT use .gitignore semantics. Docker matches with
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
# `/` and an unprefixed pattern is anchored at the context root. Every
# depth-independent pattern therefore needs `**/`, or `config/.env` and
# `certs/server.key` still ship while this file reads as solved. Only
# genuinely root-anchored entries go unprefixed. Never transplant these
# into .gitignore, where `**/` is wrong.
#
# Matching is case-sensitive, so secrets use character ranges rather
# than an ALL-CAPS twin, which would still miss `Server.Key`.
#
# Extend with this repo's own host-built artifacts, written anchored:
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
# deletes the package directory from the context.
# .git is sent without its config. Without a VERSION build argument the # .git is sent without its config. Without a VERSION build argument the
# stage that compiles runs `git describe --tags --always` on .git, which # stage that compiles runs `git describe --tags --always` on .git, which
# does not need .git/config; that file can hold a credential, such as a # does not need .git/config; that file can hold a credential, such as a
# password in a remote URL or the token the CI checkout step stores there. # password in a remote URL or the token the CI checkout step stores there.
# Each submodule keeps a config with the same exposure in its git directory .git/config
# under .git/modules/, nested again for a submodule's own submodules, or in
# its own .git directory when it keeps one.
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
# `deploy/config`, `config/lib`) loses its whole git directory, because
# `**/.git/modules/**/config` also matches that segment's directory
# under .git/modules/. Go's version stamping then fails the build;
# nothing leaks. Name such a submodule without that segment:
# `git submodule add --name`.
**/.git/config
**/.git/modules/**/config
# Agent scratch: one full checkout of the repo per in-flight agent.
# Anchored because it occurs once where agents run at the repo root.
# KNOWN GAP: a repo running agents in subdirectories still ships
# `services/api/.claude/` and must add its own anchored entry.
.claude .claude
.DS_Store
# Environment files. `*.env` covers bare `.env` and the `prod.env` sfdupes
# convention. Re-include a committed template with a negation if the *.log
# build needs one: `!docs/example.env`. *.out
**/*.[eE][nN][vV] *.test
**/.[eE][nN][vV].*
**/.[eE][nN][vV][rR][cC]
# Private keys and the bundles carrying them. Public certificates
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
**/*.[pP][eE][mM]
**/*.[kK][eE][yY]
**/*.[pP]12
**/*.[pP][fF][xX]
**/[iI][dD]_[rR][sS][aA]
**/[iI][dD]_[dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
**/[iI][dD]_[eE][dD]25519
**/[iI][dD]_[eE][dD]25519_[sS][kK]
# Dependencies: restored inside the image, never copied in.
**/node_modules
# OS metadata.
**/.DS_Store
**/Thumbs.db
# Editor state: never a build input, and it churns COPY.
**/*.swp
**/*.swo
**/*~
**/*.bak
**/.idea
**/.vscode
**/*.sublime-*
# This repository's host-built artifacts: the binary `make build` writes,
# and test binaries, coverage output and logs.
/sfdupes
/*.test
/*.out
/*.log
-3
View File
@@ -13,6 +13,3 @@ indent_style = tab
[*.go] [*.go]
indent_style = tab indent_style = tab
# This repository's own sections, such as one for another language it
# uses, go below this comment, and a re-vendor keeps them.
-9
View File
@@ -1,20 +1,11 @@
name: check name: check
on: [push] on: [push]
# Free the shared runner: a new push cancels only the same branch's older run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs: jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Free the shared runner from a hung build.
timeout-minutes: 20
steps: steps:
# actions/checkout v4.2.2, 2026-02-22 # actions/checkout v4.2.2, 2026-02-22
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
# script/cibuild needs no token, so none is left in .git/config.
with: with:
persist-credentials: false
# All history and tags, so git describe finds the version tag.
fetch-depth: 0 fetch-depth: 0
- run: script/cibuild - run: script/cibuild
+12 -37
View File
@@ -11,50 +11,25 @@ Thumbs.db
.vscode/ .vscode/
*.sublime-* *.sublime-*
# Agent scratch (worktrees of this repo, created and destroyed by
# in-flight tooling). Unanchored: .gitignore patterns already match at
# every depth, so no prefix is wanted here. This is not a .dockerignore
# entry and must not be given a `**/` prefix on the way into one.
.claude/
# Node # Node
node_modules/ node_modules/
# Secrets. Unanchored like every entry above, so each matches at every # Environment / secrets
# depth. Matching is case-sensitive on Linux, so names use character .env
# ranges rather than a lowercase form that misses `Server.Key`. .env.*
*.pem
*.key
# Environment files. `*.env` covers bare `.env` and the `prod.env` # Go build artifacts
# convention. Only the templates `example.env` and `sample.env` are
# re-included below. A repository that commits any other template adds
# its own negation at the end of this file, for example `!.env.example`.
*.[eE][nN][vV]
.[eE][nN][vV].*
.[eE][nN][vV][rR][cC]
!example.env
!sample.env
# Private keys and the bundles carrying them.
*.[pP][eE][mM]
*.[kK][eE][yY]
*.[pP]12
*.[pP][fF][xX]
[iI][dD]_[rR][sS][aA]
[iI][dD]_[dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
[iI][dD]_[eE][dD]25519
[iI][dD]_[eE][dD]25519_[sS][kK]
# This repository's own entries, such as its build outputs, go below
# this comment, and a re-vendor keeps them. Anchor a binary built at the
# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
/sfdupes /sfdupes
*.log
*.out
*.test *.test
*.out
*.log
# A scan database lists every path it scanned. # Local scan data
*.sqlite *.sqlite
*.sqlite-shm *.sqlite-shm
*.sqlite-wal *.sqlite-wal
# Agent worktrees
.claude/worktrees/
-3
View File
@@ -17,7 +17,6 @@ linters:
disable: disable:
# Genuinely incompatible with project patterns # Genuinely incompatible with project patterns
- exhaustruct # Requires all struct fields - exhaustruct # Requires all struct fields
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
- godot # Requires comments to end with periods - godot # Requires comments to end with periods
- wrapcheck # Too verbose for internal packages - wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go - varnamelen # Short names like db, id are idiomatic Go
@@ -25,8 +24,6 @@ linters:
# silenced by disabling that name, not by enabling the successor. # silenced by disabling that name, not by enabling the successor.
- wsl # Deprecated, replaced by wsl_v5 - wsl # Deprecated, replaced by wsl_v5
- gomodguard # Deprecated, replaced by gomodguard_v2 - gomodguard # Deprecated, replaced by gomodguard_v2
# Misses findings at random in v2.14.0; back once a pinned release fixes it
- canonicalheader
settings: settings:
lll: lll:
line-length: 88 line-length: 88
-2
View File
@@ -1,2 +0,0 @@
node_modules/
yarn.lock
-4
View File
@@ -1,4 +0,0 @@
{
"tabWidth": 4,
"proseWrap": "always"
}
+120 -49
View File
@@ -1,64 +1,136 @@
# Lint phase, built alone by script/lint. The tools are invoked directly # Lint stage — fast feedback on formatting and lint issues
# rather than through `make lint`, which runs docker itself and so cannot # golangci/golangci-lint:v2.12.2, 2026-08-07
# run inside a build step. FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
# golangci/golangci-lint:v2.14.0, 2026-10-07
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
# The gofmt half of `make fmt-check`. gofmt's output is assigned to a # Cache-buster for the gate layers, and only for them. Docker
# variable first so that its own exit status, as when it cannot parse a # invalidates COPY only when the copied content changes, so on an
# file, still fails the step. # unchanged tree the gates below would be served from cache and the
RUN files="$(gofmt -s -l .)" && \ # build would exit 0 having run nothing. script/cibuild and
if [ -n "$files" ]; then \ # script/docker pass a fresh CHECK_EPOCH on every invocation.
echo "gofmt: files not formatted:" >&2; echo "$files" >&2; exit 1; \
fi
# Validates .golangci.yml against the schema the pinned binary embeds.
RUN golangci-lint config verify --config .golangci.yml
RUN golangci-lint run --config .golangci.yml ./...
# Test phase, built alone by script/test. -race needs cgo and so a C
# compiler, which the Debian Go image ships and the alpine one does not.
# #
# The tests run as nobody: several of them make a file unreadable and # Two properties this depends on. ARG is per-stage, so the build stage
# expect reading it to fail, and root reads it anyway. nobody has no home # below declares it again; one declaration here would leave that
# directory, so HOME is /tmp, where Go puts its build cache. # stage's gate cacheable. And each gate RUN must reference the value,
# golang:1.25-trixie, 2026-10-04 # because BuildKit hashes the expanded command: a declared but
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test # unreferenced ARG invalidates nothing.
USER nobody #
ENV HOME=/tmp # It sits below the dependency layers deliberately. Everything above it
WORKDIR /src # (the pinned base image, go mod download) keeps its cache; only the
COPY go.mod go.sum ./ # gates go cold.
RUN go mod download ARG CHECK_EPOCH
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies are # The linter is invoked directly here, not through `make lint`. That
# what make BuildKit build them first, so this stage cannot run unless # target now runs `docker build -f Dockerfile.lint`, and a docker build
# lint and test passed. # cannot run a docker build: routing the gate through make would mean
# nesting docker inside this image. Same reason `make check` is gone
# from the build stage below. `make fmt-check` stays as it is — it is a
# gate, not the aggregate, and it shells out to nothing.
RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check
# The FROM above and the one in Dockerfile.lint pin the same linter
# twice, and nothing else keeps them in sync; this fails the build when
# they disagree. See the script for why it restates neither pin.
RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \
script/verify-lint-image-pin
# Same config-schema check Dockerfile.lint runs, kept here so this build
# gates on exactly what script/lint gates on. It validates against a
# schema the pinned binary embeds, so it needs no network.
RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \
golangci-lint config verify --config .golangci.yml
RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \
golangci-lint run --config .golangci.yml ./...
# Build stage
# golang:1.25-alpine, 2026-07-23 # golang:1.25-alpine, 2026-07-23
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null # We never build or run as root. Create an unprivileged user and point
RUN apk add --no-cache git make # HOME and the build cache at its home so go build and go test can write
# A tar-stream context keeps the sender's file owners, which git refuses. # it when we drop to it below. $GOPATH/bin is deliberately not on PATH:
RUN git config --system --add safe.directory /src # script/bootstrap no longer `go install`s anything (the linter runs
# from a pinned image, never from a host install), so nothing lands
# there and adding it would only widen what this image resolves.
#
# The module cache is kept outside that home, at the base image's
# default /go/pkg/mod, and belongs to root: script/bootstrap fills it as
# root. Do not move it into the home and hand it over with `chown -R`:
# that walks every file in it, which took from about 80 s to over ten
# minutes on a shared host, depending on load.
RUN adduser -D -u 1000 builder
ENV HOME=/home/builder
ENV GOPATH=/home/builder/go
ENV GOMODCACHE=/go/pkg/mod
ENV GOCACHE=/home/builder/.cache/go-build
WORKDIR /src WORKDIR /src
# No-op file copy whose only purpose is the build-graph edge: it is what
# makes this stage depend on the lint stage, and so what forces BuildKit
# to finish fmt-check, the pin guard and lint before compilation and
# tests start. Remove it and the fail-fast design dies silently — the
# build stops gating on lint and still exits 0. It replaces a copy of
# the linter binary itself, which is no longer wanted here: nothing in
# this stage runs the linter, because `make lint` is now a docker build
# and a docker build cannot run inside one.
COPY --from=lint /src/go.sum /dev/null
# Install development prerequisites the same way a developer does,
# rather than duplicating the installs inline. Only script/ and the
# dependency manifests are copied first, nothing else, so this layer
# stays cached until the scripts or the dependencies change — bootstrap
# ends in `go mod download`, which is why there is no separate
# invocation of it here.
COPY script/ script/
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN script/bootstrap
COPY . .
# Hand builder only what it writes to, without walking the module cache.
# This layer stays cached with bootstrap.
# - /src itself: make build writes the binary into it, and git refuses
# a repository whose top directory belongs to another user.
# - the module cache's cache/download directory itself, not what is in
# it: Go only reads the downloaded modules, but make build saves its
# lookup of this module's own version from git there, in a new
# directory named after the module path.
# - builder's home: the go commands bootstrap ran as root left Go's
# telemetry files there, a few small files.
RUN chown builder:builder /src /go/pkg/mod/cache/download && \
chown -R builder:builder /home/builder
# The sources are handed to builder as they are copied, so no layer has
# to walk them. Then drop root before running any checks or builds.
COPY --chown=builder:builder . .
USER builder
# Fail the build unless the branch is green. Runs as non-root so the
# permission-denied test paths are exercised legitimately (root would
# bypass the chmod(0) the tests rely on).
#
# The gates are the individual targets, not `make check`: that aggregate
# runs `script/lint`, which is now a docker build, and nothing inside an
# image build may shell out to docker. Lint is not skipped by this — it
# ran in the lint stage above, which this stage's COPY --from makes a
# prerequisite. `make`, not the scripts directly, because the Makefile's
# `export CGO_ENABLED = 0` applies only to what it invokes.
#
# Second per-stage declaration of the gate cache-buster; see the lint
# stage above for why one is not enough. It is placed after USER so the
# drop to the unprivileged user still happens before the checks run.
ARG CHECK_EPOCH
RUN echo "gate test, epoch ${CHECK_EPOCH}" && make test
RUN echo "gate fmt-check, epoch ${CHECK_EPOCH}" && make fmt-check
# The version stamped into the binary: the VERSION build argument when # The version stamped into the binary: the VERSION build argument when
# one is given, otherwise `git describe --tags --always` of the .git in # one is given, otherwise `git describe --tags --always` of the .git in
# the build context. A context that carries .git and still yields no # the build context (git is installed by script/bootstrap above). A
# version fails the build; with neither, as from a source tarball, it is # context that carries .git and still yields no version fails the build;
# "dev". `make build` rather than `go build`, so the image and a host # with neither, as from a source tarball, it is "dev".
# build share one compile recipe, cgo disabled included.
ARG VERSION ARG VERSION
RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \ RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
@@ -68,8 +140,7 @@ RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
fi; \ fi; \
make build VERSION="$version" make build VERSION="$version"
# Runtime stage, and the last one: a plain `docker build .` builds this # Runtime stage
# stage's chain and nothing else.
# alpine:3.22, 2026-07-23 # alpine:3.22, 2026-07-23
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+59
View File
@@ -0,0 +1,59 @@
# Lint-only image: this is how the linter runs, everywhere. The repo is
# COPYed into the pinned golangci-lint image and the linter runs as a
# build step, so a successful build IS a clean lint. golangci-lint is
# never installed on a host — one toolchain, pinned by digest, identical
# on a laptop and in CI — and this works even when the docker daemon is
# remote and bind mounts are impossible.
#
# script/lint builds this file. It is a separate image from the lint
# stage of the main Dockerfile because script/lint must not depend on
# the rest of that build; the two FROM lines are kept identical by
# script/verify-lint-image-pin, run as a gate below.
# golangci/golangci-lint:v2.12.2, 2026-08-07
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240
WORKDIR /src
# Dependency layers first, so they stay cached across lint runs.
COPY go.mod go.sum ./
RUN go mod download
COPY . .
# Cache-buster for the gate layers, and only for them. Caching of the
# lint run is waived by ruling: COPY is invalidated only by changed
# content, so on an unchanged tree the gates below would be served from
# cache and this build would exit 0 in under a second having run no
# linter at all. That exact false green has bitten this repo twice
# already (#32, #39). script/lint passes a fresh value on every
# invocation.
#
# Each gate RUN must reference the value, because BuildKit hashes the
# expanded command and not the ARG declaration: a declared but
# unreferenced ARG invalidates nothing. The ARG sits below the
# dependency layers deliberately — everything above it keeps its cache,
# only the gates go cold.
ARG CHECK_EPOCH
# The linter version is pinned in two places, here and in the main
# Dockerfile's lint stage. Nothing else keeps them in sync, so a
# half-applied bump is a build failure; see the script.
RUN echo "gate lint-image-pin, epoch ${CHECK_EPOCH}" && \
script/verify-lint-image-pin
# Validates .golangci.yml against golangci-lint's JSON schema. The
# concern about this step was that it fetches that schema over a live,
# unpinned HTTPS call; measured on the pinned image, it does not. The
# binary carries the schema for its own version, so under
# `--network none` this both passes on a valid config and still rejects
# an invalid one with the jsonschema error. That holds for the gate
# steps generally — none of them makes a network call — but not for
# this build as a whole: `go mod download` above needs the network on a
# cold cache, and under `--network none` a first build fails there
# before reaching any gate. That layer stays cached, so only a warm
# cache lints offline, until go.mod or go.sum changes.
RUN echo "gate config verify, epoch ${CHECK_EPOCH}" && \
golangci-lint config verify --config .golangci.yml
RUN echo "gate lint, epoch ${CHECK_EPOCH}" && \
golangci-lint run --config .golangci.yml ./...
+721 -622
View File
File diff suppressed because it is too large Load Diff
+82 -386
View File
@@ -1,6 +1,6 @@
--- ---
title: Repository Policies title: Repository Policies
last_modified: 2026-10-07 last_modified: 2026-07-06
--- ---
This document covers repository structure, tooling, and workflow standards. Code This document covers repository structure, tooling, and workflow standards. Code
@@ -60,28 +60,17 @@ style conventions are in separate documents:
prerequisite since nvm requires bash. yarn is then pinned via prerequisite since nvm requires bash. yarn is then pinned via
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts"; `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
always exact versions. `script/cibuild` runs the CI build: it changes to the always exact versions. `script/cibuild` runs the CI build: it changes to the
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image repo root and runs `docker build .`; the Gitea workflow calls it. Four further
with the version; the Gitea workflow calls it. **`script/cibuild` runs scripts are our own extensions to the standard: `script/check` runs
`script/bootstrap` first**, because the workflow checks out the repo and runs `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
nothing else, while `script/fmt-check` runs the formatter on the host: on a what the git pre-commit hook runs, and it calls `script/check`;
pristine checkout with nothing installed the run dies there, after the `script/install-precommit` installs the git pre-commit hook (the `make hooks`
containerised gates have passed. **The bootstrap alone is not enough**: target shims to it); and `script/projectname` (literally that filename) simply
`script/bootstrap` installs node and yarn under nvm and leaves neither on the outputs the project's name. Scripts that need the name call
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host `script/projectname` — e.g. `script/docker` assembles its image tag from it —
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore so those scripts stay byte-identical across all repos. Repo-type-specific
source nvm for the pinned node version before invoking it, exactly as pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/bootstrap`'s own install step does. A runner carrying nothing but `script/precommit`, not in the hook itself. Model scripts are at
docker and git then gets through `script/check`. Four further scripts are our
own extensions to the standard: `script/check` runs `script/test`,
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
installs the git pre-commit hook (the `make hooks` target shims to it); and
`script/projectname` (literally that filename) simply outputs the project's
name. Scripts that need the name call `script/projectname` — e.g.
`script/docker` assembles its image tag from it — so those scripts stay
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the must document the provided scripts in an **Entrypoints** section (see the
README requirements below). README requirements below).
@@ -100,222 +89,87 @@ style conventions are in separate documents:
contributor should be able to understand the entire development workflow by contributor should be able to understand the entire development workflow by
reading the Makefile. reading the Makefile.
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a - Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
`lint` phase and a `test` phase, with the final stage depending on both so the as a build step so the build fails if the branch is not green. For non-server
image cannot be built unless they pass. For non-server repos the final stage repos, the Dockerfile should bring up a development environment and run
brings up a development environment; for server repos it is the runtime image. `make check`. For server repos, `make check` should run as an early build
The gate phases and the build stage start from their pinned base images and stage before the final image is assembled. Dockerfiles install development
install what those images lack either inline, as the canonical Go `Dockerfile` prerequisites by running `script/bootstrap` rather than duplicating installs
below does for `git`, or by running `script/bootstrap`, as the `prompts` inline; COPY `script/` and the dependency manifests (`package.json` +
repo's own `Dockerfile` does for its yarn packages. The development `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
environment stage installs development prerequisites by running layer stays cached until dependencies change.
`script/bootstrap` rather than duplicating its installs inline. A stage that
runs `script/bootstrap` COPYs `script/` and the dependency manifests
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is - **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
no separate lint file. `script/lint` and `script/test` each build one phase repos use a multistage build where linting runs in an independent stage based
and nothing else: on the `golangci/golangci-lint` image (pinned by hash). This stage runs
`make fmt-check` and `make lint` before the full build begins. The build stage
then declares an explicit dependency on the lint stage via
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
linting before proceeding to compilation and tests. This ensures lint failures
surface in seconds rather than minutes, without blocking on dependency
download or compilation in the build stage.
```sh The standard pattern for a Go repo Dockerfile is:
docker build --no-cache --target lint --output type=cacheonly .
docker build --no-cache --target test --output type=cacheonly .
```
**A stage that is not the last one in the file is built only when the final
stage's chain depends on it, or when `--target` names it.** That is why the
two gates are always invoked by name here, and why the final stage carries a
`COPY --from=` of a harmless file from each of them: without that edge a
plain `docker build .` builds the last stage alone and exits 0 having linted
and tested nothing.
**The gate builds write no image.** With `--output type=cacheonly` the phase
runs and a failing step fails the build, but the result is not exported.
Nothing uses those images, and writing one out is slow: a Go test phase's
image holds the toolchain and every compiled package. A build given neither
`--output` nor `-t` writes an untagged image and leaves it dangling, on
every developer host and every CI runner. `script/cibuild` and
`script/docker` build the image that ships and tag it, so each build
replaces the previous image; each assigns the tag on its own line before the
build, so `set -e` stops it where `script/projectname` fails.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
themselves a `docker build` and would recurse into a daemon that does not
exist in a build step. Formatting is the exception and stays on the host:
`script/fmt` writes the working tree, and `script/fmt-check` is its
read-only twin.
**No lint verdict may come from a host invocation of the linter.** On a
shared host golangci-lint reads a result cache keyed on file content rather
than location, so a second checkout of the same content is served the first
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own cache, its own `TMPDIR` and
a digest-pinned binary, so neither is reachable.
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
a `COPY` layer only when the copied content changes, so on an unchanged tree
the check `RUN` is served from cache, nothing executes, and the build still
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
four, and there is no fifth — `script/check` runs the two gate phases and
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
not evidence that anything ran: a sub-second build reporting success is a
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
and friends destroy a build cache shared with every other build on the host.
When a check is added or changed, prove it works by planting a defect it must
catch and watching the run fail on it, then revert the defect. A green run
alone shows neither that the check ran nor that it covers what it should.
- **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by
hash), so lint failures surface in seconds rather than after a full compile,
and the test phase is based on the Debian Go image. The canonical Go repo
`Dockerfile`:
```dockerfile ```dockerfile
# Lint phase # Lint stage — fast feedback on formatting and lint issues
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN golangci-lint run --config .golangci.yml ./... RUN make fmt-check
RUN make lint
# Test phase. -race needs cgo and so a C compiler, which the Debian Go # Build stage
# image ships and the alpine one does not.
# golang:1.x, YYYY-MM-DD
FROM golang@sha256:... AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
# unless lint and test passed.
# golang:1.x-alpine, YYYY-MM-DD # golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder FROM golang@sha256:... AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src WORKDIR /src
# Force BuildKit to run the lint stage before proceeding
COPY --from=lint /src/go.sum /dev/null
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN make test
# The VERSION build arg when one is given, otherwise ARG VERSION=dev
# `git describe --tags --always` on the .git in the build context. With RUN CGO_ENABLED=0 go build -trimpath \
# .git present, a version that is still empty, dev or unknown fails the
# build: git is missing or could not read the checkout.
ARG VERSION
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
if [ -e .git ]; then \
case "$VERSION" in ""|dev|unknown) \
echo "version is '$VERSION' although .git is present" >&2; \
exit 1 ;; \
esac; \
fi; \
CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \ -ldflags="-s -w -X main.Version=${VERSION}" \
-o /app ./cmd/app/ -o /app ./cmd/app/
# Runtime stage, and the last one # Runtime stage
FROM alpine@sha256:... FROM alpine@sha256:...
COPY --from=builder /app /usr/local/bin/app COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"] ENTRYPOINT ["app"]
``` ```
Key points: Key points:
- The lint phase uses the `golangci/golangci-lint` image directly (it has - The lint stage uses the `golangci/golangci-lint` image directly (it
both Go and the linter), so nothing needs installing. includes both Go and the linter), so there is no need to install the
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only linter separately.
purpose is the ordering edge. BuildKit runs stages in parallel by default, - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
and a stage nothing depends on is not built at all, so without these two a stage dependency. BuildKit runs stages in parallel by default; without
lines a red gate would not fail the build. this line, the build stage would not wait for lint to finish and a lint
- Keep the runtime stage last, and if you add a stage after it, give it the failure might not fail the overall build.
same two copies. A plain `docker build .` builds the last stage's chain
and nothing else.
- If the project uses `//go:embed` directives that reference build artifacts - If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must (e.g. a web frontend compiled in a separate stage), the lint stage must
create placeholder files so the embed directives resolve. Example: create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
- If the project requires CGO or system libraries for linting, install them The lint stage should not depend on the actual build output — it exists to
in the lint phase. The `golangci/golangci-lint` image is Debian-based and fail fast.
has no `apk`, so install with `apt-get` under the Debian package name - If the project requires CGO or system libraries for linting (e.g.
(`libvips-dev`, where alpine says `vips-dev`), and delete the package `vips-dev`), install them in the lint stage with `apk add`.
lists in the same `RUN`, so the layer does not keep them: - The build stage runs `make test` after compilation setup. Tests run in the
build stage, not the lint stage, because they may require compiled
```dockerfile artifacts or heavier dependencies.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/*
```
- `.dockerignore` lets `.git` into the build context. It keeps out every git
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
repository's own, each submodule's under `.git/modules/`, and that of a
submodule keeping its own `.git` directory. `git describe` does not need
them, and each can hold a credential: a password in a remote URL, or the
token the CI checkout step stores there. A submodule whose name has a
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
git directory to `**/.git/modules/**/config`, and Go's version stamping
then fails the build: give it a name without that segment
(`git submodule add --name`). The stage that compiles has `git` (the
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
takes the version from the `VERSION` build argument when one is given,
otherwise from `git describe --tags --always`. That gives the tag on a
tagged commit; on a later commit, the tag, the number of commits since it
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
tag is reachable. The stage that compiles also marks its working directory
safe for git (`git config --system --add safe.directory /src`): a context
sent as a tar stream keeps the sender's file owners, and git refuses a
checkout owned by another user, so the version would come out empty.
`ARG VERSION` has no default, and the build fails if the context carries
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
`docker build .` with no build arguments must succeed; a Dockerfile that
refuses an empty build argument drops that refusal and keeps the argument.
A checkout whose `.git` is a file (a linked worktree, or a repository
checked out as a submodule) is the exception: that file points to a git
directory outside the build context, so the build cannot read the version
and a plain `docker build .` fails; pass the version with
`--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already
do.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
runs `script/cibuild` on push, and checks out the repo as its only other step, runs `script/cibuild` (which runs `docker build .`) on push. Since the
with `persist-credentials: false`: `script/cibuild` needs no token, and Dockerfile already runs `make check`, a successful build implies all checks
without it the checkout leaves the job's token in `.git/config` for every pass.
later step. The checkout step also sets `fetch-depth: 0`, which fetches the
tags `git describe` needs: by default it clones shallow with no tags, and a
tagged repository's CI build would stamp a bare short commit id. The
workflow's `concurrency` block groups runs by workflow and branch
(`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`,
so a new push cancels the older run on the same branch, queued or running, and
no other: runs for replaced commits do not hold up the shared runner.
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
so a successful run means every check passed; a bare `docker build .` does not
carry the same guarantee, because its gate phases may come from the cache. The
image build is uncached and so runs the gate phases a second time. That is the
price of the rule above, and it is worth paying: the image that ships is built
from a run of its own gates rather than from a cache entry. The `check` job
sets `timeout-minutes: 20`, so a hung build frees the shared runner after 20
minutes. That allows for the three Docker builds described above (the test
phase, the lint phase, then the image), each held to the 5-minute Docker build
limit below, plus the bootstrap. A separate workflow limited to `main` by a
`branches` list under `on: push` cannot be checked by review: to try a change
to it, add the feature branch to that list and push, then remove the branch
from the list again before merging. Keep any job in it that publishes behind
`if: github.ref_name == 'main'`, so the run from the feature branch publishes
nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for - Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -335,21 +189,14 @@ style conventions are in separate documents:
module under test to verify it compiles/parses. There is no excuse for module under test to verify it compiles/parses. There is no excuse for
`make test` to be a no-op. `make test` to be a no-op.
- `make test` must complete in under 60 seconds. That is the hard cap, and a - `make test` must complete in under 20 seconds. Add a 30-second timeout in the
suite that exceeds it fails. Under 20 seconds is the target. A suite between Makefile.
20 and 60 seconds is still green, but the overage must be filed as an
improvement bug against that repo. Add a 90-second timeout to the test
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
hard cap so that it catches a genuinely hung test rather than a merely slow
one.
- **The test command should use the conditional verbose rerun pattern.** Run - **`make test` should use the conditional verbose rerun pattern.** Run tests
tests without `-v` (verbose) first. If tests fail, automatically rerun with without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
`-v` to show full output. This keeps CI logs and `docker build` output clean show full output. This keeps CI logs and `docker build` output clean on
on success (just package/suite summaries) while providing full diagnostic success (just package/suite summaries) while providing full diagnostic detail
detail on failure (every test case, every assertion). The command lives in the on failure (every test case, every assertion). The general shell pattern:
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
Makefile form below is the same pattern for any repo-local invocation:
```makefile ```makefile
test: test:
@@ -362,26 +209,11 @@ style conventions are in separate documents:
```makefile ```makefile
test: test:
@go test -count=1 -timeout 90s -race -cover ./... || \ @go test -timeout 30s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; } go test -timeout 30s -race -v ./...; exit 1; }
``` ```
`-count=1` is required on both invocations: it defeats Go's test _result_
cache, so neither run can report a stored pass in place of running the
tests. It leaves the build cache alone, so it costs the runtime of the suite
and no recompilation.
That cache is Go's own, separate from Docker's layer cache. Go stores a
passing result in its cache directory (`GOCACHE`), and when the same tests
run again on unchanged code it prints that result, marked `(cached)`,
without running them. That matters on a developer's machine, where this
target runs and the directory lasts from one run to the next. The `test`
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
result for this repo's tests and nothing before its `go test` step runs a
test, so there is nothing to replay. `--no-cache` (above) is what makes that
step run on an unchanged tree.
Python example: Python example:
```makefile ```makefile
@@ -407,89 +239,10 @@ style conventions are in separate documents:
must be in `.gitignore`. No exceptions. must be in `.gitignore`. No exceptions.
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
`node_modules/`, and the repo's own build outputs. Fetch the standard Fetch the standard `.gitignore` from
`.gitignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
a new repo. A repo's `.gitignore` is the standard file followed by the repo's a new repo.
own entries, such as its binaries; a re-vendor replaces the standard part and
keeps those entries. These patterns are written to `.gitignore`'s own
semantics, in which an unanchored pattern already matches at every depth; they
are not a `.dockerignore` and must not be transplanted into one unmodified.
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
across unmodified leaves secrets in the build context.** Docker matches with
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
therefore excludes only the copies at the repository root, while `config/.env`
and `certs/server.key` still reach the context and can land in an image layer
— which is more dangerous than a short file with no secret patterns at all,
because it reads as solved and stops anyone looking. Give every
depth-independent pattern the `**/` prefix and leave only genuinely
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
binary, written `/myapp` and never `**/myapp`, which would also match
`cmd/myapp/` and delete the package directory from the context. Matching is
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
also catches something the build needs, re-include it with a negation
(`!docs/example.env`); deleting the pattern reopens the exposure for every
other file it covers. Fetch the standard `.dockerignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
it with the repo's own artifacts.
- **In-repo agent scratch belongs in both files, written to each file's own
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
additional checkout of the repo — so under `COPY . .` the build context
inflates by a multiple of the repo and another session's unreviewed work can
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
prefix, because the prefixed form would also delete any nested directory of
that name from the build. Anchoring carries a known gap that the canonical
`.dockerignore` states in its own comment, since consuming repos receive the
file and not the tracker: the directory is created in the agent's working
directory, so a repo running agents in subdirectories still ships
`services/api/.claude/` and must add its own anchored entry there.
- **A plain `docker build .` of a clone stamps the version that
`git describe --tags --always` gives**, derived from the `.git` in the build
context as the canonical `Dockerfile` above shows. Without its failure check,
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
and the build would still exit 0. `script/docker` and `script/cibuild` pass
the version they compute on the host; it takes precedence. They do this
byte-identically across repos:
```sh
# The version and the tag each get their own line: a failing command
# substitution inside an argument does not trip `set -e`, so the inline
# form degrades to an empty constant.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$(script/projectname)"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$tag" .
```
`--always` makes an untagged repo yield an abbreviated commit hash rather
than failing, and the `[ -n "$version" ]` line is the single place the
fallback is applied — a live check that fires on a build from an export with
no `.git` and on a repository with no commits yet. Do not fold it into the
substitution as `|| echo unknown`, which makes the guard unreachable. The
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
the scripts stay byte-identical. One consequence for CI: the standard
checkout action clones shallow and fetches no tags, so the canonical
`.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step.
- **Verify `.dockerignore` by enumerating the image, not by reading the
patterns.** Plant files at the root _and_ at least two directories deep, build
a probe image that does `COPY . .`, and list what actually landed
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
size is not a substitute: a nested secret is a few bytes, and BuildKit
transfers only the delta from the previous build.
- **No build artifacts in version control.** Code-derived data (compiled - **No build artifacts in version control.** Code-derived data (compiled
bundles, minified output, generated assets) must never be committed to the bundles, minified output, generated assets) must never be committed to the
@@ -505,56 +258,9 @@ style conventions are in separate documents:
- Make all changes on a feature branch. You can do whatever you want on a - Make all changes on a feature branch. You can do whatever you want on a
feature branch. feature branch.
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must - `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
_NEVER_ be modified by an agent: fetch it from manually by the user. Fetch from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
byte-identical, so that no repo can quietly loosen its own linting. Linter
configuration changes are made to the canonical copy in the `prompts` repo and
reach consuming repos by re-vendoring; an agent may open a PR against
canonical, which only the user merges. One list is exempt from byte-identity,
because it cannot be written once for every repo: the `deny` list of the
`test-support` depguard rule, where a repo names its own test-support packages
by full import path. A repo adds entries there and changes nothing else, and a
re-vendor carries its entries forward. The canonical golangci-lint version is
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
image
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
directive must not name a newer Go minor version than the one golangci-lint
was built with, or golangci-lint refuses to lint it: this release lints
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
installs golangci-lint on the host. A repo sets the lint phase digest to the
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
the two prompted the change: the canonical copy can name linters that an older
golangci-lint rejects, and a newer golangci-lint can add linters that
`default: all` switches on until the canonical copy disables them.
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
`PATH` only, so on an already-provisioned machine the pin is inert and a
version bump is a silent no-op — while the Dockerfile, installing into a clean
image, gets the pinned version, so a local `make check` and `make docker` can
disagree about what the tool even is. The canonical form:
- compares the installed version against the pin over the **whole** version
token; a parser that stops at the first `-` reports `2.12.2` for a host
running `2.12.2-rc1` and skips the install;
- treats absent, non-zero, empty or unrecognised `--version` output as a
mismatch, so the failure direction is a redundant install and never a
skipped one;
- after installing, re-resolves the binary the way callers do — `hash -r`,
then through `PATH`, not through the directory the installer wrote to —
and fails naming the resolved path, since an install that a shadowing
binary hides succeeds while changing nothing any caller sees;
- is actually called, and prints the version on both success paths: a
function defined and never invoked has the same exit status and the same
empty output as one that worked.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
A Go tool a repo needs on the host is installed with `go install` pinned to
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
a `go.mod` tool dependency or through a `tools.go` file, either of which
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
- When pinning images or packages by hash, add a comment above the reference - When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD). with the version and date (YYYY-MM-DD).
@@ -665,21 +371,15 @@ style conventions are in separate documents:
Never edit existing migrations after release. Never edit existing migrations after release.
- All repos should have an `.editorconfig` enforcing the project's indentation - All repos should have an `.editorconfig` enforcing the project's indentation
settings: the standard file from settings.
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets
tabs for `Makefile` and Go files, followed by the repo's own sections, such as
one for another language it uses. A re-vendor replaces the standard part and
keeps those sections.
- Avoid putting files in the repo root unless necessary. Root should contain - Avoid putting files in the repo root unless necessary. Root should contain
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
and language-specific config). Everything else goes in a subdirectory. language-specific config). Everything else goes in a subdirectory. Canonical
Canonical subdirectory names: subdirectory names:
- `bin/` — executable scripts and tools - `bin/` — executable scripts and tools
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose - `cmd/` — Go command entrypoints
body is a single call into `internal/` or `pkg/`, no project logic in
`cmd/`
- `configs/` — configuration templates and examples - `configs/` — configuration templates and examples
- `deploy/` — deployment manifests (k8s, compose, terraform) - `deploy/` — deployment manifests (k8s, compose, terraform)
- `docs/` — documentation and markdown (README.md stays in root) - `docs/` — documentation and markdown (README.md stays in root)
@@ -706,7 +406,3 @@ style conventions are in separate documents:
- Go: `go.mod`, `go.sum`, `.golangci.yml` - Go: `go.mod`, `go.sum`, `.golangci.yml`
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
- Python: `pyproject.toml` - Python: `pyproject.toml`
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
is never committed under a file or directory named after one agent tool, such
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
+454 -285
View File
@@ -1,98 +1,57 @@
# Workflow # Workflow
- take an issue from the `1.0.0` milestone on the tracker; work not yet on the - take an issue from the `1.0.0` milestone on the tracker; work not
tracker gets filed as an issue first yet on the tracker gets filed as an issue first
- branch from `next` - branch (from `main`)
- do the work, with tests, in small focused commits - do the work, with tests, in small focused commits
- record it at the top of Completed Steps (`TODO.md` changes in the same commit - record it at the top of Completed Steps (`TODO.md` changes in the
as the work) same commit as the work)
- push the branch and open a PR against `next` whose title ends with - push the branch and open a PR whose title ends with
` (closes #N)` ` (closes #N)`
- an independent review gates each merge to `next`; every finding is addressed - an independent review gates the merge; every finding is addressed
or explicitly rebutted on the PR or explicitly rebutted on the PR
- only the owner merges `next` to `main` - merge to `main` once the review passes
# Status # Status
- pre-1.0 - pre-1.0
- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues - the Gitea tracker is authoritative for the pre-1.0 backlog: the
under the `1.0.0` milestone are what remains before the tag, and this file open issues under the `1.0.0` milestone are what remains before
records history and process, not the queue the tag, and this file records history and process, not the queue
# Next Step # Next Step
- take the next issue from the `1.0.0` milestone on the tracker: - take the next issue from the `1.0.0` milestone on the tracker:
https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is
of truth for what is left before 1.0.0. Individual issues are deliberately not the source of truth for what is left before 1.0.0. Individual
restated here; a copy in this file drifts out of date the moment the tracker issues are deliberately not restated here; a copy in this file
moves drifts out of date the moment the tracker moves
# Completed Steps # Completed Steps
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at - tests cover a missing database, the `scan` skip warning, the `report`
`c55a0cb`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile` and `trees` summary lines, and every subcommand going through `runE`
that write no image, and `make test` runs the suite under the race detector, (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
gone; every `docker build` in `script/` passes `--no-cache`; prettier runs on
the host, from the node and yarn `script/bootstrap` installs, so the
`prettier` and `markdown` stages are gone and the build stage installs `git`
and `make` itself; a new push cancels the workflow's older run on the same
branch, and a run stops after 20 minutes; `.claude/settings.json` is deleted
(2026-10-08, https://git.eeqj.de/sneak/sfdupes/issues/95)
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
within the same second is re-hashed (2026-10-07,
https://git.eeqj.de/sneak/sfdupes/issues/12)
- cut the narration from `TODO.md` Completed Steps and from the comments in
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
- `make test-race` ran the test suite under the race detector in a cgo-enabled
container, outside `make check` (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/18). Since
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
`make test-race` is gone
- a bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
rather than serve the gates from cache (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
`script/` pass `--no-cache`, so theirs always run
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
it; all Markdown reformatted (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/19)
- `script/lint` writes no image, so a run no longer leaves an untagged one
behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48)
- tests cover a missing database, `scan` keeping stdout empty, its skip warning,
the `report` and `trees` summary lines, and every subcommand going through
`runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
- `.golangci.yml` replaced with the current canonical copy, which uses - `.golangci.yml` replaced with the current canonical copy, which uses
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04, `gomodguard_v2`, so lint no longer prints a deprecation warning
https://git.eeqj.de/sneak/sfdupes/issues/26) (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/26)
- a test fails when either `hashWorker` cancellation check in `scan.go` is - a test fails when either `hashWorker` cancellation check in `scan.go`
removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83) is removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83)
- a database path holding `?`, `#` or `%` opens exactly the file it names - a database path holding `?`, `#` or `%` opens exactly the file it names
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55) (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
- `scan` rejects `--workers` below 1 as a usage error instead of running - `scan` rejects `--workers` below 1 as a usage error instead of
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10) running single-threaded (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/10)
- a test fails when either walk cancellation check in `scan.go` is removed - a test fails when either walk cancellation check in `scan.go` is
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81) removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
- test that `scan` refuses a database with another schema version (2026-10-04, - test that `scan` refuses a database with another schema version
https://git.eeqj.de/sneak/sfdupes/issues/64) (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64)
- correct four inaccurate comments in `cancel_test.go` and rename - correct four inaccurate comments in `cancel_test.go` and rename
`walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04, `walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04,
@@ -105,269 +64,479 @@
`files` table is refused with a clear schema-version error (2026-10-04, `files` table is refused with a clear schema-version error (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/11) https://git.eeqj.de/sneak/sfdupes/issues/11)
- README documents install, Docker, a daily cron scan and how to read and check - README documents install, Docker, a daily cron scan and how to read
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54) and check the reports (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/54)
- the `Dockerfile` build stage kept the Go module cache out of `builder`'s home - the `Dockerfile` build stage keeps the Go module cache out of `builder`'s
and copied the sources with `--chown`, so no `chown -R` walked them home and copies the sources with `--chown`, so no `chown -R` walks them
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43). Since (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43)
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
nothing changes owner: the build stage only compiles, as root, and the tests
run as `nobody` in the `test` phase
- `--version` prints `sfdupes VERSION` to stdout; README documents it and - `--version` prints `sfdupes VERSION` to stdout; README documents it and
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15) `--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed, - `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has
deletes nothing more, exits 1 (2026-10-04, hashed, deletes nothing more, exits 1 (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/5) https://git.eeqj.de/sneak/sfdupes/issues/5)
- `report` and `trees` stream the records instead of holding them all in memory; - `report` and `trees` stream the records instead of holding them all in
the schema gains the `files_signature` index (2026-10-04, memory; the schema gains the `files_signature` index (2026-10-04,
https://git.eeqj.de/sneak/sfdupes/issues/14) https://git.eeqj.de/sneak/sfdupes/issues/14)
- progress prints at once on a non-terminal, uses a real terminal test, and - progress prints at once on a non-terminal, uses a real terminal test,
prints warnings through a spinner instead of racing its redraw (2026-10-03, and prints warnings through a spinner instead of racing its redraw
https://git.eeqj.de/sneak/sfdupes/issues/13) (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/13)
- warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping - warn about and skip symlink, socket, FIFO, device and `.zfs`
the records beneath them (2026-10-03, operands, keeping the records beneath them (2026-10-03,
https://git.eeqj.de/sneak/sfdupes/issues/9) https://git.eeqj.de/sneak/sfdupes/issues/9)
- `scan` holds a lock on a lock file beside the database for its whole run, so a - `scan` holds a lock on a lock file beside the database for its whole run,
second `scan` fails at once with exit 1 (2026-10-03, so a second `scan` fails at once with exit 1 (2026-10-03,
https://git.eeqj.de/sneak/sfdupes/issues/53) https://git.eeqj.de/sneak/sfdupes/issues/53)
- test stdout write failures in `report` and `trees`; README states that - test stdout write failures in `report` and `trees`; README states that
`| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null` `| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null`
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30) (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30)
- `report` and `trees` open the database read-only, and `scan` leaves it out of - `report` and `trees` open the database read-only, and `scan` leaves it
WAL mode, so reading needs only read access (2026-10-03, closes out of WAL mode, so reading needs only read access (2026-10-03, closes
https://git.eeqj.de/sneak/sfdupes/issues/8) https://git.eeqj.de/sneak/sfdupes/issues/8)
- escape tabs, newlines, carriage returns and backslashes in report, trees and - escape tabs, newlines, carriage returns and backslashes in report,
warning paths; the root directory's path is `/` (2026-10-03, trees and warning paths; the root directory's path is `/`
https://git.eeqj.de/sneak/sfdupes/issues/7) (2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/7)
- stamp the git tag or short commit in a plain `docker build .` instead of `dev` - stamp the git tag or short commit in a plain `docker build .`
(2026-10-02, branch `next`, closes instead of `dev` (2026-10-02, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` sends `.git` https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` now
without `.git/config`; the build stage stamps the `VERSION` build argument, sends `.git`, without `.git/config`, and the `Dockerfile` build
else `git describe --tags --always`, and fails if the context carries `.git` stage takes the `VERSION` build argument when one is given,
and the version is still empty, `dev` or `unknown`. CI checks out the full otherwise `git describe --tags --always` of that `.git`. The build
history (`fetch-depth: 0`) so it stamps the same value as `make build`. fails if the context carries `.git` and the version still comes out
empty, `dev` or `unknown`. The CI checkout step fetches the full
history (`fetch-depth: 0`) so CI sees the tag and stamps the same
value as `make build`.
- replace the 1 KiB end-window sampling with the head/tail plus content-hash - replace the 1 KiB end-window sampling with the head/tail plus
ladder (2026-09-22, branch `next`, closes content-hash ladder (2026-09-22, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/61); README "Duplicate detection" https://git.eeqj.de/sneak/sfdupes/issues/61): a file under 10 MiB is
documents every rung. A file under 10 MiB is hashed in full, and its `head`, hashed in full and compared directly, with no end-window step — its
`tail` and `content` all hold that hash. A larger file gets only its 64 KiB `head`, `tail`, and `content` all hold the whole-file hash. A file at
`head` and `tail` in the hash phase; the content phase, after the update 10 MiB or above gets only the 64 KiB `head` and `tail` in the hash
phase, reads it for `content` (the whole file below 50 MiB, gigabyte-spaced 1 phase; a new content phase, after the update phase, reads it for its
MiB samples at or above) only when its size, `head` and `tail` match another `content` hash — the whole file below 50 MiB, gigabyte-spaced 1 MiB
record's from this scan or an earlier one, and never reads a file gone or samples at or above — only when its size, `head`, and `tail` match
changed since its record was written. `report` and `trees` leave out any another record's, from the same scan or stored by an earlier one, so
record without a `content` hash. The `content` column is part of the version 1 a stored file gains its content hash when it gains a match. A file
schema. that is gone or has changed since its record was written is not
read. The `content` column is part of the version 1 schema. `report`
and `trees` group by the extended signature and leave out any record
without a `content` hash, so the ladder is applied across the whole
database. README "Duplicate detection" documents every rung including
the probabilistic large-file path.
- remove the dead `files.dat` references from `Makefile`, `.gitignore` and - remove the dead `files.dat` references from `Makefile`, `.gitignore`
`.dockerignore` (2026-09-21, branch `next`, closes and `.dockerignore` (2026-09-21, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/22) https://git.eeqj.de/sneak/sfdupes/issues/22)
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and - fix the lint-image pin comments and `FROM` form in `Dockerfile` and
`Dockerfile.lint` (2026-08-10, branch `next`, closes `Dockerfile.lint` (2026-08-10, branch `next`, closes
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy https://git.eeqj.de/sneak/sfdupes/issues/25): dropped the false
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`, `(Debian-based)` parenthetical (v2.12.1 was Debian too) and the
without the false `(Debian-based)` note or the tag. Since redundant tag, so both pins are the policy `# image:vX.Y.Z,
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase YYYY-MM-DD` comment over a bare `FROM image@sha256:...`. Digest
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and unchanged. `script/verify-lint-image-pin` parses those `FROM` lines
`script/verify-lint-image-pin`, which compared the two pins, are gone. and still matches the tagless form; its advice line lost the now
meaningless "tag and digest". With no tag in either reference, a
tag-only disagreement no longer exists — a one-sided tag is caught as
a plain mismatch.
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10, - run all linting in Docker via `Dockerfile.lint` and `script/lint`
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the (2026-08-10, branch `next`, closes
owner ruling the linter is never installed on a host, and https://git.eeqj.de/sneak/sfdupes/issues/46): per the owner ruling, the
`golangci-lint config verify` runs before `golangci-lint run`. linter runs inside a container invoked through the `script/`
`script/bootstrap` stopped installing or pinning the linter, and entrypoint and is never installed on a host. New root
`script/verify-linter-pin` was retired. Since `Dockerfile.lint` COPYs the repo into the digest-pinned
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the `golangci/golangci-lint:v2.12.2` image and runs
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build `golangci-lint config verify` and `golangci-lint run` as build
stage depends on through `COPY --from=lint /src/go.sum /dev/null`; steps, so a successful build IS a clean lint; `script/lint` is
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside reduced to building it. `script/bootstrap` loses the `go install`,
an image build may run docker, so the phase calls `golangci-lint` directly. the pin constants, the version parser and `verify_golangci_lint`
outright rather than hardening them — with nothing linting on the
host, the `$GOPATH/bin` versus `PATH` problem that motivated them has
no subject — and now warns rather than fails when `docker` is absent.
Two traps handled. A lint build on an unchanged tree returns success
in well under a second having run no linter, which is
https://git.eeqj.de/sneak/sfdupes/issues/32 and
https://git.eeqj.de/sneak/sfdupes/issues/39 again, so
`Dockerfile.lint` carries `ARG CHECK_EPOCH` referenced
inside every gate `RUN` (BuildKit hashes the expanded command, not
the declaration) and `script/lint` passes `"$(date +%s)-$$"` — the
PID matters because two lint runs land inside the same second easily.
And nothing inside an image build may shell out to docker, so the
main `Dockerfile`'s lint stage now invokes `golangci-lint` directly
instead of `make lint`, and its build stage runs `make test` and
`make fmt-check` instead of the `make check` aggregate (`make`, not
the scripts bare, because the Makefile's `export CGO_ENABLED = 0`
only reaches what it invokes). `COPY --from=lint`
`/usr/bin/golangci-lint` is replaced by
`COPY --from=lint /src/go.sum /dev/null`: the copied binary was the
only edge forcing BuildKit to finish linting before the build stage
starts, and dropping it without replacing the edge would have ended
fail-fast linting silently under a still-green build. That is
canonical `REPO_POLICIES.md:107`'s ordering edge, restored.
`ENV PATH=/home/builder/go/bin:$PATH` is gone with the `go install`
that justified it. `script/verify-linter-pin` is retired, deleted
along with its README entry, because both of its subjects ceased to
exist in the same change: it compared a linter binary against
`GOLANGCI_LINT_VERSION` in `script/bootstrap`, and there is now
neither a binary crossing between stages nor a version pin in
bootstrap. The drift it guarded has not gone away, it has moved — the
linter is still pinned twice, now as the `FROM` line of
`Dockerfile.lint` and the `FROM` line of the `Dockerfile` lint stage,
with nothing syncing them, which is exactly what
https://git.eeqj.de/sneak/sfdupes/issues/42 made a build failure. Its
replacement is one new `script/verify-lint-image-pin`,
run as a gate in both files, which compares the two references to
each other and deliberately restates neither: a hardcoded expected
digest would be a third copy and the same drift one file further out.
`golangci-lint config verify` is included per the ruling, and the
concern about its unpinned live HTTPS schema fetch was measured
rather than assumed — under `--network none` the pinned binary both
passes a valid config and rejects an invalid one with the jsonschema
error, so it validates from an embedded schema and makes no network
call of its own. The README scopes that to the gate steps rather
than to linting as a whole: `Dockerfile.lint` runs `go mod download`
above them, so a cold cache still needs the network and only a warm
one lints offline. Verified: `make lint` green with every `PATH`
directory containing a `golangci-lint` removed
(`/home/user/go/bin`, `/home/user/.local/bin`, `/usr/local/bin`;
`command -v golangci-lint` empty); two consecutive `script/lint` runs
on an untouched tree both executed the linter, 27.7s and 28.7s in the
lint step under distinct epochs with the `COPY . .` layer `CACHED`
above them, at 42.2s and 41.8s wall clock — the no-cache rule was not
weakened to shorten that. Negative control: a planted
`var unusedIssue46Sentinel = 1` failed `script/lint` with
`report.go:173:5: var unusedIssue46Sentinel is unused (unused)`, and
failed `make docker` at `[lint 9/9]` with the build stage stopped at
`[builder 3/12]` — `COPY --from=lint`, `script/bootstrap`, the test
gate and `make build` all zero occurrences — then reverted clean. The
drift guard fails on a tag-only disagreement, on a digest-only
disagreement, and on an unreadable reference, naming both sides.
`make docker` green in 5m35s with all six gates executing under one
epoch (lint 37.6s, test 25.2s reporting
`ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not `(cached)`).
The non-root quirk still holds: in the builder image with the Go test
cache off, `--user 0:0` fails `TestScanHardlinkRunFailsTogether`
(exit 1) where the unprivileged user passes (exit 0). Noted for
follow-up, not fixed here: `golangci-lint` warns that the
`gomodguard` linter is deprecated since v2.12.0 in favour of
`gomodguard_v2`.
- install the Docker build stage's prerequisites by running `script/bootstrap` - install the Docker build stage's prerequisites by running
instead of `apk add --no-cache make` inline (2026-08-09, branch `script/bootstrap` instead of `apk add --no-cache make` inline
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42), (2026-08-09, branch `dockerfile-bootstrap`, closes #42): canonical
with `script/verify-linter-pin` failing the build unless the linter copied `REPO_POLICIES.md:97` requires it, and the inline install left the
from the lint stage was the version `script/bootstrap` pinned. Builds then build stage maintaining its own notion of the toolchain — exactly
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as the divergence #24 exists to close, one layer down. The stage now
https://git.eeqj.de/sneak/sfdupes/issues/43. Since copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`,
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and which ends in `go mod download`, so the separate invocation of that
`make` with `apk add --no-cache` and only compiles; the `lint` and `test` is gone. `COPY --from=lint /usr/bin/golangci-lint` stays, and moves
phases are the gates above the bootstrap layer. It is the only edge making this stage
- bust the Docker layer cache for the gate steps, so `script/cibuild` and depend on the lint stage, so deleting it as redundant would end
`script/docker` cannot report a green they did not earn (2026-08-09, branch fail-fast linting silently. Letting bootstrap install its own linter
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the here would have reintroduced the second toolchain and paid for a
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker from-source build of it. What makes the two stages provably one
served them from cache and the build exited 0 having run nothing. The fix was toolchain rather than two that happen to agree is a new
a `CHECK_EPOCH` build argument; since `script/verify-linter-pin`, run in the build stage on the binary
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/` that arrives from the lint stage, before bootstrap: it fails the
passes `--no-cache` instead. Run as root, the tests fail build naming both versions unless that binary is the version
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)` `script/bootstrap` pins. Bootstrap's own check could not serve that
the test relies on, so the `test` phase runs them as `nobody` purpose — it reinstalls its pin from source and then verifies
- check the installed golangci-lint version in `script/bootstrap` instead of whatever `PATH` resolves, so drift self-heals silently and a lint
only its presence (2026-08-09, branch `bootstrap-version-check`, closes stage image bumped on its own would lint at the new version while
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in `make check` ran at the old one, green. The linter version is pinned
`GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it, and in two independent places (the lint stage image digest and
any installed version that is not the pin — older, newer, absent or `GOLANGCI_LINT_VERSION`) and nothing else keeps them in sync, so a
unparseable — is reinstalled. `go install` writes into `GOBIN` (or half-applied bump is now a build failure. The pin is read out of
`GOPATH/bin`) while `make lint` runs the first `golangci-lint` on `PATH`, so `script/bootstrap`, which stays the single source of truth; a pin
bootstrap re-reads the effective version after installing and, on a mismatch, that cannot be read is a hard failure, not a skip. The check needs
prints both paths and both versions and exits non-zero; it does not reorder no `CHECK_EPOCH`: its only inputs are the copied binary and
`PATH` or delete anyone's binary. The `--version` call keeps its stderr and is `script/`, so Docker invalidates the layer exactly when a cached
bounded by `timeout(1)` where that exists. `git`, `make` and `go` keep result would stop being true, and it is documented with the other
presence-only checks. Verified by bootstrapping this host from v2.10.1 to entrypoints in the README. `$GOPATH/bin` joins `PATH` because
v2.12.2 and again to a no-op, and by stub runs of the script under `dash` that is where bootstrap's `go install` lands and bootstrap verifies
covering a thirteen-input version-parse matrix, a shadowed install that must its installs against what `PATH` resolves — nothing in the image is
exit non-zero, an install destination not on `PATH`, `GOBIN` set, and a wedged shadowed by it, the directory does not exist until bootstrap runs.
binary that must hit the timeout; `make check` and `make lint` are clean at Everything added sits above `ARG CHECK_EPOCH`, and the `chown` and
v2.12.2, so v2.10.1 was not hiding any findings on `main` `USER builder` still precede `make check`. Verified: the guard fails
the build with both versions named when the lint stage's linter is
faked to a different version, and an unmodified build still passes
it; bootstrap runs clean under Alpine's `sh` and its `apk` branch,
installing `git` and `make` and finding the copied
linter already at the pin; a second build served the bootstrap and
dependency layers `CACHED` while both gates ran with a fresh epoch;
a planted `unused` finding failed the build at the lint gate in
48.9s with the build stage's `make check` never starting; and the
suite run in the image as `--user 0:0` fails
`TestScanHardlinkRunFailsTogether`, so the drop to the unprivileged
user is still load-bearing. That last check needs the Go test cache
disabled — the first attempt reported `ok ... (cached)` as root,
reusing the result the build-time run had left in the shared cache,
which would have read as a pass. Build wall time, on a shared host
running many concurrent builds and so noisy: 2m13s on an unchanged
tree, 2m17s and 4m29s for two builds after a source change, 5m14s
cold. Only the cold one breaches the policy ceiling, and not because
of this change — `chown -R builder:builder /src /home/builder` walks
the module cache and re-runs on every source change, and it alone
varied between 77s and 210s across those four builds, which is also
the whole spread in the totals. The same cold measurement against
`main` is 5m03s with a 209s `chown`. Filed as #43
- bust the Docker layer cache for the gate steps, so `script/cibuild`
and `script/docker` cannot report a green they did not earn
(2026-08-09, branch `cibuild-cache-bust`, closes #32): both scripts
were bare `docker build` invocations with no cache control, and the
`Dockerfile` copies the tree before running its gates, so on an
unchanged tree Docker served those layers from cache and the build
exited 0 having executed nothing. That is not hypothetical here —
every merge this repo has done is a non-fast-forward merge of an
undiverged branch, so each merge commit's tree is byte-identical to
the branch head's and each merge CI run was almost certainly a full
cache hit; and PR #31's reviewer found `make docker` returning
success as a 17-layer cache hit, catching it only by being
suspicious. The fix is `ARG CHECK_EPOCH` with the scripts passing
`--build-arg CHECK_EPOCH="$(date +%s)"`. Two details make or break
it. `ARG` is scoped per stage and this `Dockerfile` has three gates
across two — `make fmt-check` and `make lint` in the lint stage,
`make check` in the build stage — so a single declaration would have
left one stage silently cacheable; it is declared in both. And
BuildKit hashes the expanded command, not the declaration, so a
declared-but-unreferenced `ARG` invalidates nothing: each gate `RUN`
echoes the epoch, which also puts the value in the build log as
evidence the layer really ran. Placement is below the dependency
layers on purpose — a build that goes cold every time would be a
different bug, not a fix. Verified by running each script twice back
to back on an unchanged tree under `BUILDKIT_PROGRESS=plain`: all
three gates executed on all four runs, each with a fresh epoch in
the log (`script/cibuild` 78.8s then 61.1s; `script/docker` 61.1s
then 53.4s), and twelve steps were still served `CACHED` in the
steady state — both `go mod download`s, `apk add`, `adduser`, the
`chown`, every `go.mod`/`go.sum` and source copy, the linter copy
out of the lint stage, and the binary copy into the runtime stage.
The lint stage still gates the build stage: with a deliberate
`unused` finding planted in the tree, the build failed at
`make lint` in 36.1s and the build-stage `make check` never started.
The build stage also still drops to the unprivileged `builder` user
before `make check`, which the suite depends on rather than merely
prefers: forcing the same image to run the tests as root fails
`TestScanHardlinkRunFailsTogether`, because root reads straight
through the `chmod(0)` the test uses to prove hard links are read
once. This is the local fix only; propagating it to the canonical
templates is `prompts` #26
- check the installed golangci-lint version in `script/bootstrap`
instead of only its presence (2026-08-09, branch
`bootstrap-version-check`, closes #24): `missing golangci-lint` meant
any linter already on `PATH` satisfied the check, so the pin was never
consulted and the v2.12.2 bump from #3 was inert on every host that
already had one — this host ran v2.10.1 against a v2.12.2 pin,
`make check` went green, and `make docker` then rejected the same
commit with findings the local gate never saw. The version now lives
in one place, `GOLANGCI_LINT_VERSION`, with the `go install` module
ref derived from it so a bump cannot half-apply; a
`golangci_lint_version` helper parses `golangci-lint --version`
(taking the field after the word `version` and tolerating an optional
leading `v`, which the module ref carries and the binary's output does
not), and any version that is not the pin — older, newer, absent or
unparseable — is reinstalled. The install is then verified against the
binary `PATH` actually resolves: `go install` writes into `GOBIN` (or
`GOPATH/bin`) while `make lint` runs whichever `golangci-lint` comes
first on `PATH`, so a wrong-version one sitting ahead of it — nix,
apt, brew, apk, or the `/usr/local/bin` copy the `Dockerfile` builder
stage makes — would swallow the install and leave the local gate
disagreeing with CI under an affirmative `bootstrap complete`.
Bootstrap now re-reads the effective version after installing and, on
a mismatch, prints both paths and both versions to stderr and exits
non-zero instead of claiming success; it does not reorder anyone's
`PATH` or delete their binary. The `--version` call keeps its stderr
connected, so a present-but-broken binary says why rather than
reinstalling forever in silence, and is bounded by `timeout(1)` where
that exists, so a wedged binary cannot hang bootstrap. `git`, `make`
and `go` keep their presence-only checks and now say why in a
comment: they are host package-manager tools the repo deliberately
does not pin, with `go.mod` governing the language version and the
digest-pinned images covering reproducible builds. Verified on this
host by bootstrapping from v2.10.1 to v2.12.2 and running it again to
a no-op, plus stub runs of the real script under `dash` covering a
thirteen-input parse matrix (absent, older, newer, host-style,
image-style, leading-`v`, stderr-only, empty, non-zero exit, impostor
binary, `(devel)`, trailing `version`), a shadowed install that must
exit non-zero, an install destination not on `PATH` at all, `GOBIN`
set, and a wedged binary that must hit the timeout; `make check` and
`make lint` are clean at v2.12.2, so v2.10.1 was not hiding any
findings on `main`
- unwind the hash worker pool on the error path (2026-08-09, branch - unwind the hash worker pool on the error path (2026-08-09, branch
`hash-pool-cleanup`, closes https://git.eeqj.de/sneak/sfdupes/issues/6): the `hash-pool-cleanup`, closes #6): `hashPhase` used to return the
pool is now an owned, context-aware `hashPool`: every blocking send in the moment `recordRun` failed and abandon the pool — the feeder parked
feeder and the workers selects on `ctx.Done()`, `jobs` is closed on every path forever on a full `jobs` channel and every worker on a full
out, and `hashPhase` defers `pool.stop()`, which cancels and then drains `results` channel. That only stopped being invisible when #4 landed
`results` until the last goroutine has exited — draining is what frees a and `runScan` began unwinding instead of calling `os.Exit`. The
worker already parked on a send. `ctx` is threaded from `cmd.Context()` pool is now an owned, context-aware `hashPool`: every blocking send
through `runScan`, `syncScan`, both worker pools and the whole database layer, in the feeder and the workers selects on `ctx.Done()`, `jobs` is
as the first parameter everywhere. The walk pool gets the same treatment plus closed on every path out, and `hashPhase` defers `pool.stop()`,
a `ctx.Err()` guard after the walk: a cancelled walk yields a partial size which cancels and then drains `results` until the last goroutine
census, and every file it never reached looks vanished to the update phase. has exited — draining is what frees a worker already parked on a
That phase's own `BeginTx` also fails on the cancelled context before deleting send. `ctx` is threaded from `cmd.Context()` through `runScan`,
anything, but the guard is the barrier that still holds once an interrupted `syncScan`, both worker pools and the whole database layer (it is
scan may commit what it has. Tests drive `run(scan)` against a database whose the first parameter everywhere), so #5 can hand this path a signal
insert trigger aborts and assert that the scan fails instead of hanging and and needs to add nothing else. The walk pool never leaked, because
that `runtime.NumGoroutine()` polls back to its pre-scan baseline; others `walkPhase` always drains its events to close, but it has the same
cancel a scan part-way through the walk, deterministically, by counting its unbounded-send shape and #5 will give it an early return, so it
own consultations of `ctx.Done()`, and assert that it stops at the guard gets the same treatment plus a `ctx.Err()` guard after the walk: a
holding a partial census and a still-populated record index, with every record cancelled walk yields a partial size census, and every file it never
intact. Direct tests of `sendEvent`, the walk workers, `dispatchDirs`, reached looks vanished to the update phase. That phase's own
`feedHashJobs`, `hashWorker` and `hashPhase` cover the remaining cancellation `BeginTx` fails on the same cancelled context before deleting
branches of both pools anything, so the guard is defence in depth rather than the only
- guarantee the database is closed on every fatal exit path (2026-08-09, branch barrier — but it is the one that survives #5 deciding an interrupted
`db-close-on-fatal`, closes https://git.eeqj.de/sneak/sfdupes/issues/4): scan may commit what it has. Tests drive `run(scan)` against a
`fatalf` and its `os.Exit(1)` are gone, so the deferred `db.Close()` — and database whose insert trigger aborts, and assert both that the scan
with it the SQLite WAL checkpoint — now actually runs when a subcommand fails; fails instead of hanging and that `runtime.NumGoroutine()` polls
`runScan`, `runReport`, `runTrees`, `loadRecords` and `resolveRoots` return back to its pre-scan baseline; a second set cancels a scan part-way
errors instead. The single exit point is `run` in `main.go`: it maps a through the walk — deterministically, by counting the scan's own
`fatalError` (anything a subcommand returned) to exit 1 and cobra's own consultations of `ctx.Done()` rather than racing a timer — and
argument and flag errors to exit 2, which keeps a runtime failure from being asserts that it stops at the guard holding a partial census and a
reported as a usage error or printing the usage text. New `main_test.go` still-populated record index, with every record intact. The
drives the CLI in-process and asserts the exit codes from README §Error remaining cancellation branches of both pools are covered by direct
handling plus the stdout/stderr split, including that a fatal error raised tests of `sendEvent`, the walk workers, `dispatchDirs`,
after the database is open leaves no `-wal`/`-shm` sidecar behind for `scan`, `feedHashJobs`, `hashWorker` and `hashPhase`
`report` or `trees` - guarantee the database is closed on every fatal exit path
- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch (2026-08-09, branch `db-close-on-fatal`, closes #4): `fatalf` and
`golangci-v2.12.2`, merged as `38a01bd`, closes its `os.Exit(1)` are gone, so the deferred `db.Close()` — and with
https://git.eeqj.de/sneak/sfdupes/issues/3): bumped the pinned linter in the it the SQLite WAL checkpoint — now actually runs when a subcommand
`Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, and fails; `runScan`, `runReport`, `runTrees`, `loadRecords` and
replaced `.golangci.yml` with the canonical file — the linter settings (`lll`, `resolveRoots` return errors instead. The single exit point is `run`
`funlen`, `cyclop`, `dupl` thresholds) now live under `linters.settings` per in `main.go`: it maps a `fatalError` (anything a subcommand
the v2 schema, so they are actually applied; no new lint findings surfaced returned) to exit 1 and cobra's own argument and flag errors to exit
- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints 2, which keeps a runtime failure from being reported as a usage
like the other managed repos (2026-07-26, commit `3abeacf`, closes error or printing the usage text. New `main_test.go` drives the CLI
https://git.eeqj.de/sneak/sfdupes/issues/1): all 12 `script/` entrypoints in-process and asserts the exit codes from README §Error handling
exist (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`, plus the stdout/stderr split, including that a fatal error raised
`fmt-check`, `check`, `docker`, `cibuild`, `precommit`, `install-precommit`) after the database is open leaves no `-wal`/`-shm` sidecar behind
and every Makefile target is now a thin shim over them for `scan`, `report` or `trees`
- update golangci-lint to v2.12.2 with the canonical config
(2026-08-09, branch `golangci-v2.12.2`, merged as `38a01bd`,
closes #3): bumped the pinned linter in the `Dockerfile` lint
stage and `script/bootstrap` from v2.12.1 to v2.12.2, and replaced
`.golangci.yml` with the canonical file — the linter settings
(`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under
`linters.settings` per the v2 schema, so they are actually
applied; no new lint findings surfaced
- convert Makefile targets to scripts-to-rule-them-all `script/`
entrypoints like the other managed repos (2026-07-26, commit
`3abeacf`, closes #1): all 12 `script/` entrypoints exist
(`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`,
`fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
`install-precommit`) and every Makefile target is now a thin shim
over them, matching the other managed repos
- make the binary the default Make target (2026-07-24, branch - make the binary the default Make target (2026-07-24, branch
`make-default-target`): plain `make` now builds `sfdupes` (previously it ran `make-default-target`): plain `make` now builds `sfdupes`
`check` plus `build`); `make build` remains as an alias (previously it ran `check` plus `build`); `make build` remains as
- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch an alias
`scan-wide-phases`): all operands seed the shared walk pool and every pass - scan-wide phases, concurrent operands, batched updates (2026-07-24,
runs once over the whole scan, so totals and ETAs are scan-global; the branch `scan-wide-phases`): all operands seed the shared walk pool
per-operand walk/hash/update cycles and their stderr announcements are gone; and every pass runs once over the whole scan, so totals and ETAs
the update pass commits in batched transactions — the filesystem is are scan-global; the per-operand walk/hash/update cycles and their
authoritative and the database an eventually-consistent reflection, so stderr announcements are gone; the update pass commits in batched
scan-level atomicity is not required transactions — the filesystem is authoritative and the database an
eventually-consistent reflection, so scan-level atomicity is not
required
- split the stat pass back out of the walk (2026-07-24, branch - split the stat pass back out of the walk (2026-07-24, branch
`parallel-phases`): phases are strictly sequential again — walk, stat, hash, `parallel-phases`): phases are strictly sequential again — walk,
update per operand — with parallelism only inside each phase; the walk stat, hash, update per operand — with parallelism only inside each
enumerates paths with per-directory workers and the stat pass lstats them with phase; the walk enumerates paths with per-directory workers and the
per-file workers, restoring the exact total/ETA stat bar stat pass lstats them with per-file workers, restoring the exact
- announce each operand on stderr before its passes (2026-07-24, branch total/ETA stat bar
`scan-operand-progress`): with per-operand walk/hash/update cycles, a - announce each operand on stderr before its passes (2026-07-24,
multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the branch `scan-operand-progress`): with per-operand walk/hash/update
whole run's cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals
- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single that looked like the whole run's — an operator watching operand 3 of
goroutine and took hours at ~20M files on a busy pool (observed: 22M files in 14 hash 300k files concluded 20M files were being skipped
4h on a ZFS server); it is now a per-directory worker-pool traversal that - parallel walk (2026-07-24, branch `parallel-walk`): the walk pass
records size/mtime during the walk (folding away the separate stat pass, was a single goroutine and took hours at ~20M files on a busy pool
halving metadata I/O), and each `PATH` operand commits in its own transaction (observed: 22M files in 4h on a ZFS server); it is now a
so an interrupted scan keeps completed operands per-directory worker-pool traversal that records size/mtime during
the walk (folding away the separate stat pass, halving metadata
I/O), and each `PATH` operand commits in its own transaction so an
interrupted scan keeps completed operands
- persistent scan database (2026-07-24, branch `persistent-database`): `scan` - persistent scan database (2026-07-24, branch `persistent-database`):
now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure
disabled) keyed by absolute path that survives between runs — a rescan hashes Go, cgo stays disabled) keyed by absolute path that survives between
only new or changed files (by mtime/size), deletes records for files vanished runs — a rescan hashes only new or changed files (by mtime/size),
from under the scanned operands, and leaves records outside them untouched, so deletes records for files vanished from under the scanned operands,
`scan` can be cronned daily; `report` and `trees` read the database (no and leaves records outside them untouched, so `scan` can be cronned
positional arguments) instead of a scan stream. Database at daily; `report` and `trees` read the database (no positional
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL arguments) instead of a scan stream. Database at
journaling plus a single-transaction update keep a report run during a scan `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`;
safe WAL journaling plus a single-transaction update keep a report run
- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`, during a scan safe
and push `main` plus tags (2026-07-23) - add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag
`v0.0.1`, and push `main` plus tags (2026-07-23)
- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
`PATH...` operands via cobra flags replacing the `/srv` `-root` default; new `PATH...` operands via cobra flags replacing the `/srv` `-root`
`-x`/`--one-file-system` flag (GNU convention) to stop at filesystem default; new `-x`/`--one-file-system` flag (GNU convention) to stop
boundaries, which are crossed by default at filesystem boundaries, which are crossed by default
- bring the repo into full policy compliance (2026-07-23, branch - bring the repo into full policy compliance (2026-07-23, branch
`repo-policy-compliance`; checklist below) `repo-policy-compliance`; checklist below)
- `git init` with README-only first commit; code baseline committed on `main` - `git init` with README-only first commit; code baseline committed on
(2026-07-22) `main` (2026-07-22)
- implement `scan`, `report`, and `trees` subcommands (pre-git history) - implement `scan`, `report`, and `trees` subcommands (pre-git history)
# Future Steps # Future Steps
- possible later features (explicitly out of scope per README): full-content - possible later features (explicitly out of scope per README):
verification of candidates, removal-script helpers full-content verification of candidates, removal-script helpers
# Repo Policy Compliance # Repo Policy Compliance
Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing
checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone repo checklist, and the Go styleguide. Code is already gofmt-clean, so no
formatting commit is needed. standalone formatting commit is needed.
- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit - [x] `.gitignore` missing — the compiled `sfdupes` binary and
untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns `files.dat` sit untracked in the tree; needs OS/editor/Go
artifacts plus secrets patterns
- [x] `.editorconfig` missing - [x] `.editorconfig` missing
- [x] `LICENSE` missing and README has no License section (MIT assumed from - [x] `LICENSE` missing and README has no License section (MIT assumed
house convention — user to confirm) from house convention — user to confirm)
- [x] `REPO_POLICIES.md` missing from repo root - [x] `REPO_POLICIES.md` missing from repo root
- [x] `.golangci.yml` missing (install canonical copy); code must then pass - [x] `.golangci.yml` missing (install canonical copy); code must then
`make lint` (150 findings fixed; `make lint` is clean) pass `make lint` (150 findings fixed; `make lint` is clean)
- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`, - [x] `Makefile` lacks required targets `test`, `lint`, `fmt`,
`docker`, `hooks`; `check` currently depends on `build`, which writes the `fmt-check`, `docker`, `hooks`; `check` currently depends on
binary (`make check` must not modify files) `build`, which writes the binary (`make check` must not modify
- [x] no tests — `go test ./...` has nothing to run; policy requires real tests files)
with a 30-second timeout and the conditional `-v` rerun pattern (suite - [x] no tests — `go test ./...` has nothing to run; policy requires
covers parsing, grouping, digests, suppression, hashing, and the scan real tests with a 30-second timeout and the conditional `-v`
pipeline; 64% coverage) rerun pattern (suite covers parsing, grouping, digests,
- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast suppression, hashing, and the scan pipeline; 64% coverage)
lint stage, build stage running `make check` - [x] `Dockerfile` missing — Go multistage with hash-pinned images:
fail-fast lint stage, build stage running `make check`
- [x] `.dockerignore` missing - [x] `.dockerignore` missing
- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout - [x] `.gitea/workflows/check.yml` missing (`docker build .` on push,
action pinned by commit SHA) checkout action pinned by commit SHA)
- [x] README lacks required sections: Description first line - [x] README lacks required sections: Description first line
(name/purpose/category/license/author), Getting Started, Rationale, TODO, (name/purpose/category/license/author), Getting Started,
License, Author Rationale, TODO, License, Author
- [x] README non-goal "no git repository setup and no CI" is stale now that the - [x] README non-goal "no git repository setup and no CI" is stale now
repo is under git with CI that the repo is under git with CI
- [x] pre-commit hook not installed (`make hooks` once the target exists) - [x] pre-commit hook not installed (`make hooks` once the target
exists)
Accepted divergences (no action): Accepted divergences (no action):
- flat single-package layout with `.go` files in the repo root — fine for a - flat single-package layout with `.go` files in the repo root — fine
small single-binary tool per the Go styleguide; the tracker audit agrees for a small single-binary tool per the Go styleguide; the tracker
audit agrees
- `go test` runs without `-race` — the repo mandates `CGO_ENABLED=0`
(pure-Go builds) and the race detector requires cgo
+5 -15
View File
@@ -344,14 +344,13 @@ func storedPaths(t *testing.T, path string) []string {
// TestRunScanInterrupted calls the scan entrypoint with a context that // TestRunScanInterrupted calls the scan entrypoint with a context that
// is already cancelled, as when a signal arrives at once. It must return // is already cancelled, as when a signal arrives at once. It must return
// errInterrupted promptly with its one line on stderr and nothing on // errInterrupted promptly with its one line on stderr, leave the
// stdout, leave the database valid and as it was, and leave nothing in // database valid and as it was, and leave nothing in the way of the
// the way of the next scan, which must bring the database up to date. // next scan, which must bring the database up to date.
func TestRunScanInterrupted(t *testing.T) { func TestRunScanInterrupted(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
stdout := captureStdout(t)
stderr := captureStderr(t) stderr := captureStderr(t)
dir := buildSmokeTree(t) dir := buildSmokeTree(t)
@@ -394,10 +393,6 @@ func TestRunScanInterrupted(t *testing.T) {
t.Errorf("stderr = %q, want %q", got, want) t.Errorf("stderr = %q, want %q", got, want)
} }
if got := stdout(); got != "" {
t.Errorf("stdout = %q, want nothing (data only)", got)
}
assertNoSidecars(t, path) assertNoSidecars(t, path)
if got := storedPaths(t, path); !slices.Equal(got, before) { if got := storedPaths(t, path); !slices.Equal(got, before) {
@@ -420,13 +415,12 @@ func TestRunScanInterrupted(t *testing.T) {
// TestRunScanInterruptedMidHash interrupts the scan entrypoint part-way // TestRunScanInterruptedMidHash interrupts the scan entrypoint part-way
// through its hash phase, after the database is open. It must return // through its hash phase, after the database is open. It must return
// errInterrupted, release the lock, end stderr with its line counting // errInterrupted, release the lock, end stderr with its line counting
// every file the walk reached, write nothing to stdout, close the // every file the walk reached, close the database out of WAL mode, and
// database out of WAL mode, and keep the records it hashed. // keep the records it hashed.
func TestRunScanInterruptedMidHash(t *testing.T) { func TestRunScanInterruptedMidHash(t *testing.T) {
path := testDBPath(t) path := testDBPath(t)
t.Setenv(databaseEnv, path) t.Setenv(databaseEnv, path)
stdout := captureStdout(t)
stderr := captureStderr(t) stderr := captureStderr(t)
dir := buildWalkCancelTree(t) dir := buildWalkCancelTree(t)
@@ -444,10 +438,6 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
t.Errorf("stderr = %q, want it to end with %q", got, want) t.Errorf("stderr = %q, want it to end with %q", got, want)
} }
if got := stdout(); got != "" {
t.Errorf("stdout = %q, want nothing (data only)", got)
}
assertNoSidecars(t, path) assertNoSidecars(t, path)
db, err := openReportDatabase(t.Context(), path) db, err := openReportDatabase(t.Context(), path)
+15 -25
View File
@@ -11,7 +11,6 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"strconv" "strconv"
"time"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo // The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
@@ -41,15 +40,12 @@ const dbDirPerm = 0o755
const lockFilePerm = 0o600 const lockFilePerm = 0o600
// createTableSQL is the schema applied to a fresh database. Paths are // createTableSQL is the schema applied to a fresh database. Paths are
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
// second.
const createTableSQL = ` const createTableSQL = `
CREATE TABLE files ( CREATE TABLE files (
path BLOB PRIMARY KEY, path BLOB PRIMARY KEY,
size INTEGER NOT NULL, size INTEGER NOT NULL,
mtime INTEGER NOT NULL, mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL,
head TEXT NOT NULL, head TEXT NOT NULL,
tail TEXT NOT NULL, tail TEXT NOT NULL,
content TEXT NOT NULL content TEXT NOT NULL
@@ -65,11 +61,10 @@ CREATE INDEX files_signature ON files (size, head, tail, content)
// upsertSQL inserts one file record, replacing any existing record for // upsertSQL inserts one file record, replacing any existing record for
// the same path. // the same path.
const upsertSQL = ` const upsertSQL = `
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content) INSERT INTO files (path, size, mtime, head, tail, content)
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (path) DO UPDATE SET ON CONFLICT (path) DO UPDATE SET
size = excluded.size, mtime = excluded.mtime, size = excluded.size, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
head = excluded.head, tail = excluded.tail, head = excluded.head, tail = excluded.tail,
content = excluded.content content = excluded.content
` `
@@ -361,8 +356,8 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
// which is the order of the primary key, so SQLite does not sort. // which is the order of the primary key, so SQLite does not sort.
func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error { func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
rows, err := db.QueryContext(ctx, rows, err := db.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+ "SELECT path, size, mtime, head, tail, content FROM files "+
"FROM files ORDER BY path") "ORDER BY path")
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return fmt.Errorf("read records: %w", err)
} }
@@ -372,18 +367,16 @@ func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
for rows.Next() { for rows.Next() {
var ( var (
path []byte path []byte
sec, nsec int64
r scanRec r scanRec
) )
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail,
&r.content) &r.content)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return fmt.Errorf("read record: %w", err)
} }
r.path = string(path) r.path = string(path)
r.mtime = time.Unix(sec, nsec)
fn(r) fn(r)
} }
@@ -473,10 +466,10 @@ func loadDupeRows(ctx context.Context, db *sql.DB,
// values, and skipping the hash columns keeps the scan's in-memory // values, and skipping the hash columns keeps the scan's in-memory
// index small on multi-million-file databases. // index small on multi-million-file databases.
func loadFileMeta(ctx context.Context, db *sql.DB, func loadFileMeta(ctx context.Context, db *sql.DB,
fn func(path string, size int64, mtime time.Time, hashed bool), fn func(path string, size, mtime int64, hashed bool),
) error { ) error {
rows, err := db.QueryContext(ctx, rows, err := db.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files") "SELECT path, size, mtime, head <> '' FROM files")
if err != nil { if err != nil {
return fmt.Errorf("read records: %w", err) return fmt.Errorf("read records: %w", err)
} }
@@ -486,16 +479,16 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
for rows.Next() { for rows.Next() {
var ( var (
path []byte path []byte
size, sec, nsec int64 size, mtime int64
hashed int64 hashed int64
) )
err = rows.Scan(&path, &size, &sec, &nsec, &hashed) err = rows.Scan(&path, &size, &mtime, &hashed)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return fmt.Errorf("read record: %w", err)
} }
fn(string(path), size, time.Unix(sec, nsec), hashed != 0) fn(string(path), size, mtime, hashed != 0)
} }
err = rows.Err() err = rows.Err()
@@ -514,7 +507,7 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
// memory; the rows come ordered by size, head, and tail, so each // memory; the rows come ordered by size, head, and tail, so each
// group's rows arrive together. // group's rows arrive together.
const contentCandidatesSQL = ` const contentCandidatesSQL = `
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> '' SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
FROM files AS f FROM files AS f
JOIN ( JOIN (
SELECT size, head, tail SELECT size, head, tail
@@ -541,19 +534,16 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
for rows.Next() { for rows.Next() {
var ( var (
path []byte path []byte
sec, nsec int64
r scanRec r scanRec
hashed int64 hashed int64
) )
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
&hashed)
if err != nil { if err != nil {
return fmt.Errorf("read record: %w", err) return fmt.Errorf("read record: %w", err)
} }
r.path = string(path) r.path = string(path)
r.mtime = time.Unix(sec, nsec)
fn(r, hashed != 0) fn(r, hashed != 0)
} }
@@ -637,8 +627,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
defer func() { _ = st.Close() }() defer func() { _ = st.Close() }()
for _, r := range upserts { for _, r := range upserts {
_, err = st.ExecContext(ctx, []byte(r.path), r.size, _, err = st.ExecContext(ctx,
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content) []byte(r.path), r.size, r.mtime, r.head, r.tail, r.content)
if err != nil { if err != nil {
return fmt.Errorf("upsert %s: %w", r.path, err) return fmt.Errorf("upsert %s: %w", r.path, err)
} }
+5 -10
View File
@@ -10,7 +10,6 @@ import (
"slices" "slices"
"strings" "strings"
"testing" "testing"
"time"
) )
// testDBPath returns a database path inside a fresh temp dir. // testDBPath returns a database path inside a fresh temp dir.
@@ -261,13 +260,10 @@ func TestApplyChangesRoundTrip(t *testing.T) {
// written. // written.
recs := []scanRec{ recs := []scanRec{
{ {
size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2", size: 2, mtime: 20, head: "h2", tail: "t2", content: "c2",
content: "c2", path: "/a/tab\tnew\nline", path: "/a/tab\tnew\nline",
},
{
size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
content: "c1", path: "/a/x",
}, },
{size: 1, mtime: 10, head: "h1", tail: "t1", content: "c1", path: "/a/x"},
} }
err := applyChanges(t.Context(), db, recs, nil, err := applyChanges(t.Context(), db, recs, nil,
@@ -285,8 +281,7 @@ func TestApplyChangesRoundTrip(t *testing.T) {
// An upsert for an existing path updates in place; a delete // An upsert for an existing path updates in place; a delete
// removes exactly its path. // removes exactly its path.
upd := scanRec{ upd := scanRec{
size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3", size: 3, mtime: 30, head: "h3", tail: "t3", content: "c3", path: "/a/x",
path: "/a/x",
} }
err = applyChanges(t.Context(), db, []scanRec{upd}, err = applyChanges(t.Context(), db, []scanRec{upd},
@@ -313,7 +308,7 @@ func TestApplyChangesBatching(t *testing.T) {
recs := make([]scanRec, 0, n) recs := make([]scanRec, 0, n)
for i := range n { for i := range n {
recs = append(recs, scanRec{ recs = append(recs, scanRec{
size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t", size: int64(i), mtime: 1, head: "h", tail: "t",
path: fmt.Sprintf("/batch/%07d", i), path: fmt.Sprintf("/batch/%07d", i),
}) })
} }
+17 -41
View File
@@ -91,35 +91,16 @@ func makeReadOnly(t *testing.T, path string) {
func captureStderr(t *testing.T) func() string { func captureStderr(t *testing.T) func() string {
t.Helper() t.Helper()
return capture(t, &os.Stderr) f, err := os.Create(filepath.Join(t.TempDir(), "stderr"))
}
// captureStdout does for os.Stdout what captureStderr does for
// os.Stderr. scan is never given run's stdout writer, so anything it
// printed would go straight to os.Stdout. The scan tests pass os.Stdout
// as run's stdout too, so the one capture sees both.
func captureStdout(t *testing.T) func() string {
t.Helper()
return capture(t, &os.Stdout)
}
// capture redirects *std, which is os.Stdout or os.Stderr, to a file
// for the rest of the test and returns a function reading back
// everything written to it.
func capture(t *testing.T, std **os.File) func() string {
t.Helper()
f, err := os.Create(filepath.Join(t.TempDir(), "output"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
saved := *std saved := os.Stderr
*std = f os.Stderr = f
t.Cleanup(func() { t.Cleanup(func() {
*std = saved os.Stderr = saved
_ = f.Close() _ = f.Close()
}) })
@@ -220,17 +201,15 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
args = append(args, t.TempDir()) args = append(args, t.TempDir())
} }
stdout := captureStdout(t) var stdout, stderr bytes.Buffer
var stderr bytes.Buffer code := run(args, &stdout, &stderr)
code := run(args, os.Stdout, &stderr)
if code != exitFatal { if code != exitFatal {
t.Errorf("run(%v) = %d, want %d", args, code, exitFatal) t.Errorf("run(%v) = %d, want %d", args, code, exitFatal)
} }
assertNoSidecars(t, path) assertNoSidecars(t, path)
assertFatalOutput(t, stderr.String(), stdout()) assertFatalOutput(t, stderr.String(), stdout.String())
// Proof that the failure happened after the open: only a // Proof that the failure happened after the open: only a
// query against the opened database can report this. // query against the opened database can report this.
@@ -248,18 +227,16 @@ func TestRunNonexistentPathIsFatalNotUsage(t *testing.T) {
// must not dump the usage text. // must not dump the usage text.
t.Setenv(databaseEnv, testDBPath(t)) t.Setenv(databaseEnv, testDBPath(t))
stdout := captureStdout(t) var stdout, stderr bytes.Buffer
var stderr bytes.Buffer
missing := filepath.Join(t.TempDir(), "nope") missing := filepath.Join(t.TempDir(), "nope")
code := run([]string{cmdScan, missing}, os.Stdout, &stderr) code := run([]string{cmdScan, missing}, &stdout, &stderr)
if code != exitFatal { if code != exitFatal {
t.Errorf("run(scan %s) = %d, want %d", missing, code, exitFatal) t.Errorf("run(scan %s) = %d, want %d", missing, code, exitFatal)
} }
assertFatalOutput(t, stderr.String(), stdout()) assertFatalOutput(t, stderr.String(), stdout.String())
} }
// assertFatalOutput checks that a fatal error was reported the way // assertFatalOutput checks that a fatal error was reported the way
@@ -488,16 +465,17 @@ func scanFixture(t *testing.T) []string {
func scanOK(t *testing.T, operands ...string) string { func scanOK(t *testing.T, operands ...string) string {
t.Helper() t.Helper()
stdout := captureStdout(t) var stdout bytes.Buffer
stderr := captureStderr(t) stderr := captureStderr(t)
code := run(append([]string{cmdScan}, operands...), os.Stdout, os.Stderr) code := run(append([]string{cmdScan}, operands...), &stdout, os.Stderr)
if code != exitOK { if code != exitOK {
t.Fatalf("run(scan %q) = %d, want %d; stderr: %s", t.Fatalf("run(scan %q) = %d, want %d; stderr: %s",
operands, code, exitOK, stderr()) operands, code, exitOK, stderr())
} }
if got := stdout(); got != "" { if got := stdout.String(); got != "" {
t.Errorf("scan stdout = %q, want nothing (data only)", got) t.Errorf("scan stdout = %q, want nothing (data only)", got)
} }
@@ -829,11 +807,9 @@ func TestRunSecondScanFails(t *testing.T) {
holdScanLock(t, path) holdScanLock(t, path)
stdout := captureStdout(t) var stdout, stderr bytes.Buffer
var stderr bytes.Buffer code := run([]string{cmdScan, t.TempDir()}, &stdout, &stderr)
code := run([]string{cmdScan, t.TempDir()}, os.Stdout, &stderr)
if code != exitFatal { if code != exitFatal {
t.Errorf("run(scan) = %d, want %d", code, exitFatal) t.Errorf("run(scan) = %d, want %d", code, exitFatal)
} }
@@ -844,7 +820,7 @@ func TestRunSecondScanFails(t *testing.T) {
t.Errorf("stderr = %q, want %q", got, want) t.Errorf("stderr = %q, want %q", got, want)
} }
if got := stdout(); got != "" { if got := stdout.String(); got != "" {
t.Errorf("stdout = %q, want nothing (data only)", got) t.Errorf("stdout = %q, want nothing (data only)", got)
} }
-5
View File
@@ -1,5 +0,0 @@
{
"devDependencies": {
"prettier": "3.8.1"
}
}
+1 -2
View File
@@ -7,7 +7,6 @@ import (
"io" "io"
"os" "os"
"strings" "strings"
"time"
) )
// ioBufSize is the buffer size for the buffered stdout writers. // ioBufSize is the buffer size for the buffered stdout writers.
@@ -22,7 +21,7 @@ const minGroupSize = 2
// only and used by scan for change detection. // only and used by scan for change detection.
type scanRec struct { type scanRec struct {
size int64 size int64
mtime time.Time mtime int64
head string head string
tail string tail string
content string content string
+2 -9
View File
@@ -11,7 +11,6 @@ import (
"slices" "slices"
"strings" "strings"
"testing" "testing"
"time"
) )
// awkwardDir is a directory name holding every byte the reports escape. // awkwardDir is a directory name holding every byte the reports escape.
@@ -314,14 +313,8 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
// mtime is informational only; records differing only in mtime // mtime is informational only; records differing only in mtime
// still group together. // still group together.
recs := []scanRec{ recs := []scanRec{
{ {size: 9, mtime: 100, head: "h", tail: "t", content: "c", path: "/m/1"},
size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t", {size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"},
content: "c", path: "/m/1",
},
{
size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
content: "c", path: "/m/2",
},
} }
groups := dupeGroupsOf(t, recs) groups := dupeGroupsOf(t, recs)
+7 -18
View File
@@ -17,7 +17,6 @@ import (
"strings" "strings"
"sync" "sync"
"syscall" "syscall"
"time"
) )
// The duplicate ladder (see hashSignature and README "Duplicate // The duplicate ladder (see hashSignature and README "Duplicate
@@ -69,7 +68,7 @@ var errInterrupted = errors.New("scan interrupted")
type fileRec struct { type fileRec struct {
path string path string
size int64 size int64
mtime time.Time mtime int64
dev uint64 dev uint64
ino uint64 ino uint64
} }
@@ -80,7 +79,7 @@ type fileRec struct {
// they would dominate the scan's memory. // they would dominate the scan's memory.
type fileMeta struct { type fileMeta struct {
size int64 size int64
mtime time.Time mtime int64
hashed bool hashed bool
} }
@@ -370,7 +369,7 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
s.existing = make(map[string]fileMeta) s.existing = make(map[string]fileMeta)
return loadFileMeta(ctx, s.db, return loadFileMeta(ctx, s.db,
func(path string, size int64, mtime time.Time, hashed bool) { func(path string, size, mtime int64, hashed bool) {
prog.increment() prog.increment()
if underAnyRoot(path, roots) { if underAnyRoot(path, roots) {
@@ -416,7 +415,7 @@ func (s *scanState) walkPhase(
old, ok := s.existing[ev.rec.path] old, ok := s.existing[ev.rec.path]
switch { switch {
case !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime): case !ok || old.size != ev.rec.size || old.mtime < ev.rec.mtime:
changed = append(changed, ev.rec) changed = append(changed, ev.rec)
case old.hashed: case old.hashed:
delete(s.existing, ev.rec.path) delete(s.existing, ev.rec.path)
@@ -808,7 +807,7 @@ func unchangedFile(r scanRec) (fileRec, bool, error) {
} }
if !fi.Mode().IsRegular() || fi.Size() != r.size || if !fi.Mode().IsRegular() || fi.Size() != r.size ||
mtimeAfter(fi.ModTime(), r.mtime) { fi.ModTime().Unix() > r.mtime {
return fileRec{}, false, nil return fileRec{}, false, nil
} }
@@ -819,16 +818,6 @@ func unchangedFile(r scanRec) (fileRec, bool, error) {
}, true, nil }, true, nil
} }
// mtimeAfter reports whether mtime a is later than mtime b.
// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
func mtimeAfter(a, b time.Time) bool {
if a.Unix() != b.Unix() {
return a.Unix() > b.Unix()
}
return a.Nanosecond() > b.Nanosecond()
}
// underAnyRoot reports whether path is any of the roots or lies under // underAnyRoot reports whether path is any of the roots or lies under
// one of them. // one of them.
func underAnyRoot(path string, roots []string) bool { func underAnyRoot(path string, roots []string) bool {
@@ -977,7 +966,7 @@ func seedRoot(ctx context.Context, root string,
sendEvent(ctx, events, walkEvent{rec: fileRec{ sendEvent(ctx, events, walkEvent{rec: fileRec{
path: root, path: root,
size: fi.Size(), size: fi.Size(),
mtime: fi.ModTime(), mtime: fi.ModTime().Unix(),
dev: dev, dev: dev,
ino: ino, ino: ino,
}}) }})
@@ -1135,7 +1124,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
sendEvent(ctx, events, walkEvent{rec: fileRec{ sendEvent(ctx, events, walkEvent{rec: fileRec{
path: p, path: p,
size: info.Size(), size: info.Size(),
mtime: info.ModTime(), mtime: info.ModTime().Unix(),
dev: dev, dev: dev,
ino: ino, ino: ino,
}}) }})
+7 -227
View File
@@ -20,8 +20,6 @@ import (
"syscall" "syscall"
"testing" "testing"
"time" "time"
"golang.org/x/sys/unix"
) )
// writeFile creates a file with the given content and returns its path. // writeFile creates a file with the given content and returns its path.
@@ -583,65 +581,6 @@ func TestScanContentHashedStalePartners(t *testing.T) {
} }
} }
// TestScanContentSameSecondRewrite is TestScanContentStalePartners for
// a stored file rewritten in place at the same size with an mtime later
// in the same second than recorded: the file counts as changed, so
// neither it nor its match inside the operand is read.
func TestScanContentSameSecondRewrite(t *testing.T) {
t.Parallel()
db := openTestDB(t)
dirA := t.TempDir()
changed := sparseFileWithoutMatch(t, dirA, "changed", headTailMin)
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(changed, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dirA)
before := dbRecords(t, db)
// Rewrite one byte in place, keeping the size.
pokeAt(t, changed, headTailMin/2, []byte{1})
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(changed, later, later)
if err != nil {
t.Fatal(err)
}
dirB := t.TempDir()
sparseFile(t, dirB, "changed-copy", headTailMin)
st := syncTree(t, db, dirB)
if st != (scanStats{walked: 1, added: 1}) {
t.Errorf("stats = %+v, want 1 added and nothing skipped", st)
}
recs := dbRecords(t, db)
for _, r := range recs {
if r.content != "" {
t.Errorf("%s: content = %q, want none: its only match is stale",
r.path, r.content)
}
}
for _, old := range before {
if r := recordByPath(t, recs, old.path); r != old {
t.Errorf("record = %+v, want it left as %+v", r, old)
}
}
if groups := dupeGroups(t, db); len(groups) != 0 {
t.Errorf("groups = %+v, want none", groups)
}
}
// TestScanContentReadFailure checks that a failed content read is // TestScanContentReadFailure checks that a failed content read is
// counted as skipped and leaves the record without a content hash, and // counted as skipped and leaves the record without a content hash, and
// that a later scan tries the read again. // that a later scan tries the read again.
@@ -844,8 +783,8 @@ func TestWalk(t *testing.T) {
t.Errorf("%s: size = %d, want 1..3", r.path, r.size) t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
} }
if r.mtime.Unix() <= 0 { if r.mtime <= 0 {
t.Errorf("%s: mtime = %v, want after 1970", r.path, r.mtime) t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
} }
} }
} }
@@ -1346,172 +1285,11 @@ func TestSyncScanMtimeBump(t *testing.T) {
t.Fatalf("mtime-bump stats = %+v, want 1 updated", st) t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
} }
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(future) { if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
t.Fatalf("mtime = %v, want %v", r.mtime, future) t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
} }
} }
// assertWholeFileHashed fails unless the record for path holds the
// whole-file hash of data as its head, tail, and content.
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
data []byte,
) {
t.Helper()
r := recordByPath(t, dbRecords(t, db), path)
if want := hexSum(data); r.head != want || r.tail != want ||
r.content != want {
t.Fatalf("head, tail, content = %q, %q, %q, want %q for each",
r.head, r.tail, r.content, want)
}
}
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
// size with an mtime later in the same second as the recorded one: the
// next scan must notice the change and re-hash the file.
func TestSyncScanSameSecondRewrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
writeFile(t, dir, "b.bin", pattern(2, 500))
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(a, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dir)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(a, later, later)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
// TestSyncScanOperandSameSecondRewrite is TestSyncScanSameSecondRewrite
// for files given to scan as operands, which scan stats without reading
// their directory.
func TestSyncScanOperandSameSecondRewrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
b := writeFile(t, dir, "b.bin", pattern(2, 500))
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(a, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, a, b)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(a, later, later)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, a, b)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
// TestSyncScanRewriteAfter2262 runs assertLateRewriteRehashed with an
// mtime after 2262, a time too late to count in nanoseconds in an int64.
func TestSyncScanRewriteAfter2262(t *testing.T) {
t.Parallel()
assertLateRewriteRehashed(t, time.Date(2300, 1, 2, 3, 4, 5, 0, time.UTC))
}
// TestSyncScanRewritePastTimeLimit runs assertLateRewriteRehashed with an
// mtime one second past the latest a time.Time holds without wrapping it
// to a time far in the past.
func TestSyncScanRewritePastTimeLimit(t *testing.T) {
t.Parallel()
assertLateRewriteRehashed(t, time.Unix(9223371974719179008, 0))
}
// assertLateRewriteRehashed scans a directory, rewrites a file in it in
// place at the same size, sets its mtime to late, and fails unless the
// next scan re-hashes the file. It skips where late does not fit the
// platform's timespec or the filesystem does not store it.
func assertLateRewriteRehashed(t *testing.T, late time.Time) {
t.Helper()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
writeFile(t, dir, "b.bin", pattern(2, 500))
syncTree(t, db, dir)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
// os.Chtimes cannot set such a time: it converts through UnixNano.
ts, err := unix.TimeToTimespec(late)
if err != nil {
t.Skipf("an mtime %d seconds after 1970 does not fit this platform's "+
"timespec: %v", late.Unix(), err)
}
err = unix.UtimesNano(a, []unix.Timespec{ts, ts})
if err != nil {
t.Fatal(err)
}
fi, err := os.Lstat(a)
if err != nil {
t.Fatal(err)
}
if fi.ModTime().Unix() != late.Unix() {
t.Skipf("the filesystem stored the mtime as %d seconds after 1970, "+
"not %d", fi.ModTime().Unix(), late.Unix())
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
func TestSyncScanAddRemove(t *testing.T) { func TestSyncScanAddRemove(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1557,7 +1335,9 @@ func TestSyncScanSizeChange(t *testing.T) {
writeFile(t, dir, "f", pattern(1, 200)) writeFile(t, dir, "f", pattern(1, 200))
err := os.Chtimes(p, old.mtime, old.mtime) mt := time.Unix(old.mtime, 0)
err := os.Chtimes(p, mt, mt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+23 -82
View File
@@ -2,21 +2,15 @@
# script/bootstrap: install all dependencies needed to build and develop # script/bootstrap: install all dependencies needed to build and develop
# this repo. Idempotent: every install is guarded by a check so already # this repo. Idempotent: every install is guarded by a check so already
# installed tools are skipped. Base tooling comes from nix, apt, brew, # installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes nothing is present. Node is # or apk (detected in that order); assumes nothing is present (not git,
# used directly if installed; otherwise it is installed at a pinned # make, or go). The linter is NOT installed: golangci-lint runs via
# version via nvm (installing nvm itself first, from a hash-verified # docker only (script/lint), pinned by image digest, so the only lint
# release archive, never curl | sh). # prerequisite is a working docker — which is warned about, not
# installed, because everything except linting works without it.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Pinned versions, 2026-07-06
NODE_VERSION="22.17.0"
NVM_VERSION="0.40.3"
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
YARN_VERSION="1.22.22"
PKGMGR="" PKGMGR=""
SUDO="" SUDO=""
APT_UPDATED="" APT_UPDATED=""
@@ -49,7 +43,6 @@ pkg_install() {
case "$PKGMGR" in case "$PKGMGR" in
nix) nix-env -iA "nixpkgs.$1" ;; nix) nix-env -iA "nixpkgs.$1" ;;
apt) apt)
# Package lists may be empty (fresh images); refresh once per run.
if [ -z "$APT_UPDATED" ]; then if [ -z "$APT_UPDATED" ]; then
$SUDO env DEBIAN_FRONTEND=noninteractive apt-get update $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update
APT_UPDATED=1 APT_UPDATED=1
@@ -65,82 +58,30 @@ missing() {
! command -v "$1" >/dev/null 2>&1 ! command -v "$1" >/dev/null 2>&1
} }
# verify_sha256 <file> <expected-hash>
verify_sha256() {
if command -v sha256sum >/dev/null 2>&1; then
actual="$(sha256sum "$1" | cut -d' ' -f1)"
else
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
fi
if [ "$actual" != "$2" ]; then
echo "bootstrap: sha256 mismatch for $1" >&2
echo " expected: $2" >&2
echo " actual: $actual" >&2
exit 1
fi
}
# nvm is a bash script; run a command in a bash with nvm loaded
nvm_sh() {
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
}
ensure_nvm() {
[ -s "$HOME/.nvm/nvm.sh" ] && return 0
# nvm prerequisites; nvm itself requires bash
if missing bash; then pkg_install bash bash bash bash; fi
if missing curl; then pkg_install curl curl curl curl; fi
if missing git; then pkg_install git git git git; fi
tmp="$(mktemp -d)"
curl -fsSL -o "$tmp/nvm.tar.gz" \
"https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz"
verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256"
mkdir -p "$HOME/.nvm"
tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1
rm -rf "$tmp"
}
ensure_node() {
if ! missing node; then return 0; fi
ensure_nvm
nvm_sh "nvm install $NODE_VERSION"
}
ensure_yarn() {
if ! missing yarn; then return 0; fi
if ! missing corepack; then
corepack enable
corepack prepare "yarn@$YARN_VERSION" --activate
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
corepack prepare yarn@$YARN_VERSION --activate"
else
npm install -g "yarn@$YARN_VERSION"
fi
}
install_js_deps() {
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
yarn install --frozen-lockfile"
else
yarn install --frozen-lockfile
fi
}
main() { main() {
cd "$ROOT" cd "$ROOT"
if missing make; then pkg_install gnumake make make make; fi # System tooling, deliberately unpinned: these come from the host
# package manager and whatever version it ships is what the host
# gets, so a presence check is the right check. The repo pins no
# system toolchain versions — the Go language version is governed by
# go.mod, and builds that must be reproducible run in the Docker
# image, whose base images are pinned by digest.
if missing git; then pkg_install git git git git; fi if missing git; then pkg_install git git git git; fi
# Go builds the binary and runs gofmt. Presence is the whole check: if missing make; then pkg_install gnumake make make make; fi
# go.mod names the Go version, and the tests and the linter run in
# digest-pinned images.
if missing go; then pkg_install go golang go go; fi if missing go; then pkg_install go golang go go; fi
ensure_node # Linting runs via docker only (script/lint), so docker is a lint
ensure_yarn # prerequisite rather than something bootstrap installs. Warn, do
install_js_deps # not fail: everything except `make lint` — and, through it,
# `make check`, `make docker` and the pre-commit hook — works
# without it.
if missing docker; then
echo "bootstrap: WARNING: docker not found; make lint, make check" >&2
echo "bootstrap: and make docker require it. Install docker to" >&2
echo "bootstrap: run the linter." >&2
fi
go mod download go mod download
echo "bootstrap complete" echo "bootstrap complete"
+1 -3
View File
@@ -1,8 +1,6 @@
#!/bin/sh #!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own # script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. test and lint are Docker # extension to scripts-to-rule-them-all. Must not modify any files.
# phases; fmt-check is native, because a formatter writes the working
# tree. Must not modify any files.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+25 -21
View File
@@ -1,30 +1,34 @@
#!/bin/sh #!/bin/sh
# script/cibuild: run the CI build. It bootstraps first: a CI runner # script/cibuild: run the CI build. The Gitea workflow runs this on
# checks out and runs this and nothing else, and script/fmt-check runs # push.
# the formatter on the host, which a pristine checkout cannot do. #
# --no-cache for the same reason as script/docker: the gate phases the # The Dockerfile runs the gates individually as build steps, not the
# final stage depends on are RUN steps, and a cached one is a check that # make check aggregate: the lint stage runs make fmt-check,
# did not run. # script/verify-lint-image-pin, golangci-lint config verify and
# golangci-lint run; the build stage, dropped to an unprivileged user,
# runs make test and make fmt-check. Neither make lint nor make check
# appears, because both reach script/lint, which is itself a docker
# build, and a docker build cannot run inside one. Lint is not skipped
# by that — the linter is invoked directly in the lint stage, and the
# build stage's COPY --from=lint makes that stage a prerequisite, so
# BuildKit must finish it first. Between the two stages everything
# make check would run has run, which is why a successful build here
# implies the repo is green.
#
# That implication holds only because of CHECK_EPOCH. A COPY layer is
# invalidated only by changed content, and a rebuild of an unchanged
# checkout sends the same content, so without a fresh value here Docker
# serves the gate layers from cache and the build reports a green it
# never earned. Passing the current epoch invalidates the gate
# layers on every run while leaving the pinned base images and
# go mod download cached; see the Dockerfile for the placement.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
"$SCRIPT_DIR/bootstrap" docker build --build-arg CHECK_EPOCH="$(date +%s)" .
"$SCRIPT_DIR/check"
# The version and the tag each get their own line: a failing
# command substitution inside an argument does not trip `set -e`,
# so the inline form degrades silently to an empty constant. The
# VERSION build argument takes precedence over the version a build
# stage derives from the .git in the context.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$tag" .
} }
main "$@" main "$@"
+13 -14
View File
@@ -1,8 +1,14 @@
#!/bin/sh #!/bin/sh
# script/docker: build the Docker image tagged with the project name. # script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname. # The tag comes from script/projectname.
# --no-cache because the gate phases the final stage depends on are RUN #
# steps, and a cached one is a check that did not run. # CHECK_EPOCH is passed for the same reason script/cibuild passes it:
# without it Docker serves the Dockerfile's gate layers from cache on an
# unchanged tree and this exits 0 having run neither the lint stage's
# gates nor the builder stage's test and fmt-check gates. This is the
# set of gates a developer or reviewer runs by hand, so a cached pass
# here is the most misleading result the repo can produce. Dependency
# layers sit above the ARG and stay cached.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -10,17 +16,10 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
# The version and the tag each get their own line: a failing docker build \
# command substitution inside an argument does not trip `set -e`, --build-arg CHECK_EPOCH="$(date +%s)" \
# so the inline form degrades silently to an empty constant. The -t "$("$SCRIPT_DIR/projectname")" \
# VERSION build argument takes precedence over the version a build .
# stage derives from the .git in the context.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
tag="$("$SCRIPT_DIR/projectname")"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$tag" .
} }
main "$@" main "$@"
-20
View File
@@ -4,29 +4,9 @@ set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0"
# script/bootstrap installs node and yarn under nvm and leaves neither
# on the PATH of the shell that called it, so resolve the pinned
# toolchain here the way bootstrap's own install step does. nvm is a
# bash script, hence the subshell.
run_yarn() {
if command -v yarn >/dev/null 2>&1; then
exec yarn "$@"
fi
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt: no yarn; run script/bootstrap first" >&2
exit 1
fi
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
}
main() { main() {
cd "$ROOT" cd "$ROOT"
gofmt -s -w . gofmt -s -w .
run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always
} }
main "$@" main "$@"
+4 -36
View File
@@ -1,50 +1,18 @@
#!/bin/sh #!/bin/sh
# script/fmt-check: check formatting (read-only). # script/fmt-check: check formatting (read-only). Same scope as
# script/fmt, but fails instead of writing.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Must match the pin in script/bootstrap.
NODE_VERSION="22.17.0"
# script/bootstrap installs node and yarn under nvm and leaves neither
# on the PATH of the shell that called it, so resolve the pinned
# toolchain here the way bootstrap's own install step does. nvm is a
# bash script, hence the subshell.
run_yarn() {
if command -v yarn >/dev/null 2>&1; then
exec yarn "$@"
fi
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
echo "fmt-check: no yarn; run script/bootstrap first" >&2
exit 1
fi
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
}
main() { main() {
cd "$ROOT" cd "$ROOT"
status=0 files="$(gofmt -s -l .)"
# gofmt and prettier both run every time, so the output names each
# one that fails. Under set -e a bare assignment would end the
# script when gofmt fails (a Go file it cannot parse).
if ! files="$(gofmt -s -l .)"; then
echo "gofmt: failed; see its errors above" >&2
status=1
fi
if [ -n "$files" ]; then if [ -n "$files" ]; then
echo "gofmt: files not formatted:" >&2 echo "gofmt: files not formatted:" >&2
echo "$files" >&2 echo "$files" >&2
status=1 exit 1
fi fi
# run_yarn ends in exec; the subshell returns here afterwards.
(run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always) ||
status=1
exit "$status"
} }
main "$@" main "$@"
+7 -3
View File
@@ -1,15 +1,19 @@
#!/bin/sh #!/bin/sh
# script/install-precommit: install the git pre-commit hook that runs # script/install-precommit: install the git pre-commit hook that runs
# script/precommit. Our own extension to scripts-to-rule-them-all. # script/precommit. Our own extension to scripts-to-rule-them-all.
# Hooks are shared between the main checkout and all worktrees, so
# resolve the common git dir instead of assuming .git is a directory.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
hook=".git/hooks/pre-commit" hooks_dir="$(git rev-parse --git-common-dir)/hooks"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit mkdir -p "$hooks_dir"
chmod +x .git/hooks/pre-commit hook="$hooks_dir/pre-commit"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
chmod +x "$hook"
echo "pre-commit hook installed: runs script/precommit" echo "pre-commit hook installed: runs script/precommit"
} }
+19 -13
View File
@@ -1,23 +1,29 @@
#!/bin/sh #!/bin/sh
# script/lint: run the linter. Linting is a phase of the Dockerfile and # script/lint: run the linter. golangci-lint is never installed on a
# this builds that phase alone; the linter is never installed or run on # host: it runs via docker only, one way, everywhere — this builds
# a developer host, where a shared result cache and a host-global lock # Dockerfile.lint, which COPYs the repo into the digest-pinned
# make its answer untrustworthy. # golangci-lint image and lints as a build step, so a successful build
# is a clean lint. The only prerequisite is a working docker. The gate
# steps make no network calls of their own, but Dockerfile.lint runs
# `go mod download` above them, so a cold cache does reach the network
# (as does pulling the pinned image); that layer stays cached, and once
# it is warm this runs offline until go.mod or go.sum changes.
# #
# The phase is not the last stage in the file, so it is built only when # CHECK_EPOCH is what makes the result mean anything. Without it docker
# --target names it. --no-cache because a cached lint layer is a lint # serves the gate layers from cache on an unchanged tree and this exits
# that did not run. --output type=cacheonly writes no image, since # 0 in well under a second having run no linter. The PID is in the value
# nothing uses one. # as well as the epoch because two lint runs land inside the same second
# easily, and `date +%s` alone would cache the second one.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
docker build --no-cache \ docker build \
--target lint \ --build-arg CHECK_EPOCH="$(date +%s)-$$" \
--output type=cacheonly . -f Dockerfile.lint \
.
} }
main "$@" main "$@"
+2 -2
View File
@@ -1,13 +1,13 @@
#!/bin/sh #!/bin/sh
# script/precommit: run by the git pre-commit hook; fails the commit if # script/precommit: run by the git pre-commit hook; fails the commit if
# checks fail. Our own extension to scripts-to-rule-them-all. # checks fail. Our own extension to scripts-to-rule-them-all. Go extra:
# go mod tidy must be a no-op before the checks run.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
# Go extra: go mod tidy must be a no-op before the checks run.
cd "$ROOT" cd "$ROOT"
go mod tidy go mod tidy
if ! git diff --exit-code -- go.mod go.sum; then if ! git diff --exit-code -- go.mod go.sum; then
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh #!/bin/sh
# script/setup: set up the repo for development after a fresh clone: # script/setup: set up the repo for development after a fresh clone:
# installs dependencies and the git pre-commit hook. # installs dependencies (script/bootstrap) and the git pre-commit hook.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+8 -10
View File
@@ -1,19 +1,17 @@
#!/bin/sh #!/bin/sh
# script/test: run the test suite. Testing is a phase of the Dockerfile # script/test: run the test suite. Reruns verbosely on failure so CI
# and this builds that phase alone, on the same terms as script/lint: # logs show which test failed.
# --target because a phase that is not the last stage is built only when
# named, and --no-cache because a cached test layer is a test that did
# not run. --output type=cacheonly writes no image, since nothing uses one.
set -eu set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() { main() {
cd "$ROOT" cd "$ROOT"
docker build --no-cache \ go test -timeout 30s -cover ./... || {
--target test \ echo "--- Rerunning with -v for details ---"
--output type=cacheonly . go test -timeout 30s -v ./...
exit 1
}
} }
main "$@" main "$@"
+84
View File
@@ -0,0 +1,84 @@
#!/bin/sh
# script/verify-lint-image-pin: fail unless the golangci-lint image
# referenced by Dockerfile.lint and the one referenced by the main
# Dockerfile's lint stage are the same image at the same digest. Our own
# extension to scripts-to-rule-them-all, not one of its entrypoints.
#
# The linter version is pinned in two independent files. That is the
# shape #42 turned into a build failure rather than tolerate: nothing
# else keeps the two in sync, and a bump applied to one file alone would
# leave `make lint` and the fail-fast lint stage of `make docker`
# linting the same tree against different rulesets, both green. This is
# the single guard that stops it, run as a gate in both files.
#
# It deliberately restates neither pin. A hardcoded expected digest here
# would be a third copy — one more thing to bump, and the same drift one
# file further out. It compares the two files to each other and knows
# nothing about which version is correct.
#
# A reference that cannot be read is a hard failure, not a skip: a
# comparison of two empty strings succeeds, which would turn this guard
# into exactly the unearned green it exists to prevent.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
LINT_DOCKERFILE="Dockerfile.lint"
MAIN_DOCKERFILE="Dockerfile"
# Echo the single golangci-lint image reference in the named Dockerfile.
# Scans every argument of every FROM instruction rather than assuming a
# field position, so `FROM --platform=... img AS stage` reads correctly.
# Exits non-zero, with a diagnosis, unless there is exactly one.
lint_image_ref() {
file="$1"
if [ ! -f "$file" ]; then
echo "verify-lint-image-pin: $file: not found" >&2
return 1
fi
refs="$(
awk '
toupper($1) == "FROM" {
for (i = 2; i <= NF; i++) {
if ($i ~ /^golangci\/golangci-lint[:@]/) {
print $i
}
}
}
' "$file"
)"
count="$(printf '%s' "$refs" | grep -c . || true)"
if [ "$count" -ne 1 ]; then
echo "verify-lint-image-pin: $file: expected exactly one" \
"golangci/golangci-lint FROM reference, found $count" >&2
return 1
fi
printf '%s\n' "$refs"
}
main() {
cd "$ROOT"
lint_ref="$(lint_image_ref "$LINT_DOCKERFILE")"
main_ref="$(lint_image_ref "$MAIN_DOCKERFILE")"
if [ "$lint_ref" != "$main_ref" ]; then
echo "verify-lint-image-pin: the linter image is pinned twice and" \
"the two pins disagree:" >&2
echo "verify-lint-image-pin: $LINT_DOCKERFILE: $lint_ref" >&2
echo "verify-lint-image-pin: $MAIN_DOCKERFILE: $main_ref" >&2
echo "verify-lint-image-pin: bump both FROM lines together so" \
"script/lint and the Dockerfile lint stage keep running the" \
"same linter" >&2
exit 1
fi
echo "verify-lint-image-pin: $LINT_DOCKERFILE and $MAIN_DOCKERFILE" \
"agree on $lint_ref"
}
main "$@"
-8
View File
@@ -1,8 +0,0 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
prettier@3.8.1:
version "3.8.1"
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173"
integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==