Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32e7fd095a | ||
|
|
92c02c3bd1 | ||
|
|
80bd7c0999 | ||
|
|
5dc6bfb6b7 | ||
|
|
1b26b5ce77 | ||
|
|
a4d76af8fe |
+20
-5
@@ -1,8 +1,13 @@
|
||||
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
||||
# on every build, which reruns every RUN step after it. The steps that
|
||||
# compile mount Go's build cache, which survives from build to build.
|
||||
|
||||
# Lint phase, built alone by script/lint. The tools are invoked directly
|
||||
# rather than through `make lint`, which runs docker itself and so cannot
|
||||
# run inside a build step.
|
||||
# golangci/golangci-lint:v2.14.0, 2026-10-07
|
||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||
ARG CACHEBUST
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
@@ -18,31 +23,39 @@ RUN files="$(gofmt -s -l .)" && \
|
||||
|
||||
# Validates .golangci.yml against the schema the pinned binary embeds.
|
||||
RUN golangci-lint config verify --config .golangci.yml
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase, built alone by script/test. -race needs cgo and so a C
|
||||
# compiler, which the Debian Go image ships and the alpine one does not.
|
||||
# -count=1 because the mounted cache also holds test results, which go
|
||||
# test would otherwise replay.
|
||||
#
|
||||
# The tests run as nobody: several of them make a file unreadable and
|
||||
# expect reading it to fail, and root reads it anyway. nobody has no home
|
||||
# directory, so HOME is /tmp, where Go puts its build cache.
|
||||
# directory, so HOME is /tmp, and Go keeps its build cache in
|
||||
# /tmp/.cache/go-build. That is where the cache is mounted, owned by
|
||||
# nobody (65534) so the tests can write to it.
|
||||
# golang:1.25-trixie, 2026-10-04
|
||||
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
|
||||
ARG CACHEBUST
|
||||
USER nobody
|
||||
ENV HOME=/tmp
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
|
||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies are
|
||||
# what make BuildKit build them first, so this stage cannot run unless
|
||||
# lint and test passed.
|
||||
# golang:1.25-alpine, 2026-07-23
|
||||
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
|
||||
ARG CACHEBUST
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
RUN apk add --no-cache git make
|
||||
@@ -60,7 +73,8 @@ COPY . .
|
||||
# "dev". `make build` rather than `go build`, so the image and a host
|
||||
# build share one compile recipe, cgo disabled included.
|
||||
ARG VERSION
|
||||
RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
|
||||
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
|
||||
[ "$version" = unknown ]; }; then \
|
||||
echo "no version could be derived although the build context carries .git" >&2; \
|
||||
@@ -72,6 +86,7 @@ RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
|
||||
# stage's chain and nothing else.
|
||||
# alpine:3.22, 2026-07-23
|
||||
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
|
||||
ARG CACHEBUST
|
||||
|
||||
COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
|
||||
|
||||
|
||||
@@ -271,16 +271,21 @@ All three subcommands operate on a single SQLite database file:
|
||||
error (exit 1) telling the user to run `scan` first.
|
||||
- Only one `scan` runs against a database at a time. For its whole run, `scan`
|
||||
holds an exclusive `flock(2)` lock on a lock file beside the database, named
|
||||
by appending `.lock` to the database path (`/var/lib/sfdupes/db.sqlite.lock`
|
||||
by default), taken before it walks the filesystem or opens the database. A
|
||||
second `scan` against the same database does not wait: it fails at once with a
|
||||
one-line error naming the lock file and exits 1, without walking anything or
|
||||
opening the database, and the running scan carries on. The lock file is
|
||||
created on first use, open to its owner only, and left in place: a leftover
|
||||
file blocks nothing, because the lock ends with the process holding it however
|
||||
it ends, a fatal error or an interrupt included, and deleting the file while a
|
||||
scan runs would let a second scan start. `report` and `trees` never take the
|
||||
lock, so they run during a scan.
|
||||
by appending `.lock` to the database's path with symlinks resolved
|
||||
(`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the
|
||||
filesystem or opens the database. Every path to one database, through a
|
||||
symlink to the file or to a directory above it, names the same lock file. Just
|
||||
before it takes the lock, `scan` creates the database file, empty, if it is
|
||||
missing (through a symlink at the database path, it creates the symlink's
|
||||
target), so the lock file is named the same way before the first scan sets up
|
||||
the database as after. A second `scan` against the same database does not
|
||||
wait: it fails at once with a one-line error naming the lock file and exits 1,
|
||||
without walking anything or opening the database, and the running scan carries
|
||||
on. The lock file is created on first use, open to its owner only, and left in
|
||||
place: a leftover file blocks nothing, because the lock ends with the process
|
||||
holding it however it ends, a fatal error or an interrupt included, and
|
||||
deleting the file while a scan runs would let a second scan start. `report`
|
||||
and `trees` never take the lock, so they run during a scan.
|
||||
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so
|
||||
running a report while a cron `scan` is in progress is safe. The filesystem is
|
||||
authoritative; the database is an eventually-consistent reflection of it.
|
||||
@@ -393,7 +398,11 @@ results do not depend on the working directory. All operands belong to a single
|
||||
scan and are enumerated concurrently: every operand seeds the shared walk worker
|
||||
pool. Overlapping operands are harmless — an operand that duplicates another or
|
||||
lies under another is dropped before walking, so every file is reached exactly
|
||||
once and produces one database record.
|
||||
once and produces one database record. With `-x`, though, an operand under
|
||||
another one is kept and walked on its own when any directory from it (from its
|
||||
directory, for a regular file) up to but not including the operand covering it
|
||||
is on a different filesystem than that operand, because that operand's walk does
|
||||
not enter such a directory and so does not reach it.
|
||||
|
||||
An operand that is a symlink (never followed, not even as an operand), socket,
|
||||
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
|
||||
@@ -402,9 +411,8 @@ drops it from the scanned operands before reading the database. Another operand
|
||||
beneath it is still scanned. The records stored beneath it are not deleted: they
|
||||
are treated like any other record outside the scanned operands, including the
|
||||
content-phase exception below. If it lies under another operand, they are under
|
||||
that operand instead, and are deleted like any other record there that this scan
|
||||
did not verify. This is not an error: a scan whose every operand is dropped
|
||||
walks nothing and exits 0.
|
||||
that operand instead, and are treated like any other record there. This is not
|
||||
an error: a scan whose every operand is dropped walks nothing and exits 0.
|
||||
|
||||
`scan` synchronizes the database with the filesystem state under the scanned
|
||||
operands:
|
||||
@@ -442,6 +450,12 @@ operands:
|
||||
a `content` hash is read for one, wherever it lies, once its size, `head`, and
|
||||
`tail` match another record's. If that file is gone or has changed since its
|
||||
record was written, the record is left as it is.
|
||||
- With `-x`, a record below a directory the walk did not enter because that
|
||||
directory is on another filesystem than its operand counts as outside the
|
||||
scanned operands: it is not deleted, and its size joins the size census. A
|
||||
filesystem mounted under an operand can therefore be scanned on its own
|
||||
schedule into the same database. Records below an operand at or inside such a
|
||||
directory are not kept this way, since that operand's own walk verifies them.
|
||||
|
||||
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
|
||||
inside each phase; batched database writes begin during the hash phase:
|
||||
@@ -508,15 +522,19 @@ Rules for the walk:
|
||||
- Filesystem boundaries are crossed by default. With `-x` (long form
|
||||
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
||||
into a directory on a different filesystem than its `PATH` operand; each
|
||||
operand is bounded by its own filesystem.
|
||||
operand is bounded by its own filesystem. The records below such a directory,
|
||||
unless they lie below an operand at or inside that directory, count as outside
|
||||
the scanned operands, as described in "`scan` mode" above: they are not
|
||||
deleted, and their sizes join the size census.
|
||||
- On any per-path error (permission denied, file vanished between passes,
|
||||
unreadable): print a one-line warning to stderr, skip the path, and continue.
|
||||
Per-file errors never abort the run; the final summary reports how many were
|
||||
skipped. As specified above, a skipped path that has a database record from an
|
||||
earlier scan loses that record, unless it failed only in the content phase, or
|
||||
is an operand dropped before the database was read that lies under no other
|
||||
operand; an unreadable directory subtree likewise loses its records (accepted:
|
||||
the database mirrors what the latest scan could actually verify).
|
||||
is an operand dropped before the database was read whose record counts as
|
||||
outside the scanned operands; an unreadable directory subtree likewise loses
|
||||
its records (accepted: the database mirrors what the latest scan could
|
||||
actually verify).
|
||||
|
||||
Concurrency: the walk phase (which also stats files), the hash phase, and the
|
||||
content phase each use a worker pool of `--workers` workers (default
|
||||
@@ -746,14 +764,16 @@ entrypoints are:
|
||||
- `script/bootstrap` — install everything needed to build and develop this
|
||||
repository, idempotently, assuming nothing is present. `git`, `make`, and `go`
|
||||
come from the first of nix, apt, brew, or apk found on the host, and are
|
||||
presence-checked only. An installed node is used as it is; otherwise node
|
||||
22.17.0 is installed through nvm, which comes from a release archive whose
|
||||
sha256 the script checks. A yarn already on `PATH` is used as it is; otherwise
|
||||
yarn 1.22.22 is activated through corepack, or installed with `npm` when there
|
||||
is no corepack. `yarn install --frozen-lockfile` then installs the prettier
|
||||
that `package.json` and `yarn.lock` pin. `golangci-lint` is never installed:
|
||||
it runs in Docker (see `script/lint`). `docker` is not installed either;
|
||||
testing, linting and the image build need it. Ends with `go mod download`.
|
||||
presence-checked only. An installed node is used when its major version is 22;
|
||||
otherwise node 22.17.0 is installed through nvm, which comes from a release
|
||||
archive whose sha256 the script checks, and yarn 1.22.22 is activated under it
|
||||
through corepack. With an installed node 22, a yarn already on `PATH` is used
|
||||
as it is; otherwise yarn 1.22.22 is activated through corepack, or installed
|
||||
with `npm` when there is no corepack. `yarn install --frozen-lockfile` then
|
||||
installs the prettier that `package.json` and `yarn.lock` pin. `golangci-lint`
|
||||
is never installed: it runs in Docker (see `script/lint`). `docker` is not
|
||||
installed either; testing, linting and the image build need it. Ends with
|
||||
`go mod download`.
|
||||
- `script/setup` — make a fresh clone ready for development: runs
|
||||
`script/bootstrap`, then `script/install-precommit`.
|
||||
- `script/projectname` — print this project's name (`sfdupes`). Scripts that
|
||||
@@ -774,8 +794,8 @@ entrypoints are:
|
||||
- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every
|
||||
Markdown file with prettier, at the settings in `.prettierrc` (4-space
|
||||
indents, prose wrapped at 80 columns). Both run on the host, prettier through
|
||||
yarn at the version `yarn.lock` pins. When yarn is not on `PATH`, this loads
|
||||
the node 22.17.0 that `script/bootstrap` installed through nvm.
|
||||
yarn at the version `yarn.lock` pins. When the node on `PATH` is not node 22,
|
||||
this loads the node 22.17.0 that `script/bootstrap` installed through nvm.
|
||||
- `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any
|
||||
unformatted file and exits non-zero instead of writing. gofmt and prettier
|
||||
both run every time, and each names itself when it fails. The `Dockerfile`'s
|
||||
@@ -798,10 +818,13 @@ entrypoints are:
|
||||
- `script/install-precommit` — install the git pre-commit hook, as
|
||||
`.git/hooks/pre-commit`, that runs `script/precommit`.
|
||||
|
||||
Every `docker build` in `script/` passes `--no-cache`. On an unchanged tree
|
||||
Docker would serve the gate steps from its cache, and the build would pass
|
||||
having run no test and no linter. Every run therefore downloads the Go modules
|
||||
again and needs the network.
|
||||
Every `docker build` in `script/` passes a new `CACHEBUST` build argument, which
|
||||
every stage of the `Dockerfile` declares first, so every `RUN` step runs again
|
||||
each time. On an unchanged tree Docker would otherwise serve the gate steps from
|
||||
its cache, and the build would pass having run no test and no linter. Go's build
|
||||
cache is kept between builds on the steps that compile, so only what changed is
|
||||
compiled again; the tests run with `-count=1`, so none reports a stored result.
|
||||
Every run downloads the Go modules again and needs the network.
|
||||
|
||||
## Build
|
||||
|
||||
|
||||
+117
-73
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-10-07
|
||||
last_modified: 2026-10-15
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
@@ -54,34 +54,39 @@ style conventions are in separate documents:
|
||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
||||
(detected in that order; apt runs noninteractive). For node it uses the
|
||||
installed node if present; otherwise it installs a PINNED node version via
|
||||
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
||||
release archive (never `curl | sh`), with bash installed as an explicit
|
||||
prerequisite since nvm requires bash. yarn is then pinned via
|
||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
||||
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||
installed node only when its major version is the pinned one; otherwise (node
|
||||
missing, or another major, such as the node 24 the CI runner image ships) it
|
||||
installs the PINNED node version via nvm and installs yarn and the packages
|
||||
under it. nvm itself is installed first if missing, from a hash-verified
|
||||
GitHub release archive (never `curl | sh`), with bash installed as an explicit
|
||||
prerequisite since nvm requires bash. Only the major version is compared
|
||||
because the `Dockerfile` stages start from a node image whose exact version is
|
||||
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
|
||||
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
|
||||
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
|
||||
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
|
||||
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
||||
pristine checkout with nothing installed the run dies there, after the
|
||||
containerised gates have passed. **The bootstrap alone is not enough**:
|
||||
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
||||
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
||||
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
||||
source nvm for the pinned node version before invoking it, exactly as
|
||||
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
||||
docker and git then gets through `script/check`. Four further scripts are our
|
||||
own extensions to the standard: `script/check` runs `script/test`,
|
||||
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
||||
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
||||
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
||||
`script/projectname` (literally that filename) simply outputs the project's
|
||||
name. Scripts that need the name call `script/projectname` — e.g.
|
||||
`script/docker` assembles its image tag from it — so those scripts stay
|
||||
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
||||
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
||||
the hook itself. Model scripts are at
|
||||
containerised gates have passed. **The bootstrap alone is not enough**: when
|
||||
`script/bootstrap` installs node and yarn under nvm it leaves neither on the
|
||||
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
|
||||
under another node. The host entrypoints that need yarn — `script/fmt` and
|
||||
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when
|
||||
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
|
||||
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
|
||||
pinned node. A runner carrying nothing but docker and git then gets through
|
||||
`script/check`. Four further scripts are our own extensions to the standard:
|
||||
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
|
||||
`script/precommit` is what the git pre-commit hook runs, and it calls
|
||||
`script/check`; `script/install-precommit` installs the git pre-commit hook
|
||||
(the `make hooks` target shims to it); and `script/projectname` (literally
|
||||
that filename) simply outputs the project's name. Scripts that need the name
|
||||
call `script/projectname` — e.g. `script/docker` assembles its image tag from
|
||||
it — so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||
`script/precommit`, not in the hook itself. Model scripts are at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||
must document the provided scripts in an **Entrypoints** section (see the
|
||||
README requirements below).
|
||||
@@ -118,8 +123,10 @@ style conventions are in separate documents:
|
||||
and nothing else:
|
||||
|
||||
```sh
|
||||
docker build --no-cache --target lint --output type=cacheonly .
|
||||
docker build --no-cache --target test --output type=cacheonly .
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--target lint --output type=cacheonly .
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--target test --output type=cacheonly .
|
||||
```
|
||||
|
||||
**A stage that is not the last one in the file is built only when the final
|
||||
@@ -132,12 +139,12 @@ style conventions are in separate documents:
|
||||
**The gate builds write no image.** With `--output type=cacheonly` the phase
|
||||
runs and a failing step fails the build, but the result is not exported.
|
||||
Nothing uses those images, and writing one out is slow: a Go test phase's
|
||||
image holds the toolchain and every compiled package. A build given neither
|
||||
`--output` nor `-t` writes an untagged image and leaves it dangling, on
|
||||
every developer host and every CI runner. `script/cibuild` and
|
||||
`script/docker` build the image that ships and tag it, so each build
|
||||
replaces the previous image; each assigns the tag on its own line before the
|
||||
build, so `set -e` stops it where `script/projectname` fails.
|
||||
image holds the toolchain. A build given neither `--output` nor `-t` writes
|
||||
an untagged image and leaves it dangling, on every developer host and every
|
||||
CI runner. `script/cibuild` and `script/docker` build the image that ships
|
||||
and tag it, so each build replaces the previous image; each assigns the tag
|
||||
on its own line before the build, so `set -e` stops it where
|
||||
`script/projectname` fails.
|
||||
|
||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
||||
@@ -152,22 +159,33 @@ style conventions are in separate documents:
|
||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
||||
a digest-pinned binary, so neither is reachable.
|
||||
org, in both directions. A container has its own golangci-lint cache, its
|
||||
own `TMPDIR` and a digest-pinned binary, so neither is reachable.
|
||||
|
||||
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
||||
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
||||
the check `RUN` is served from cache, nothing executes, and the build still
|
||||
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
||||
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
||||
four, and there is no fifth — `script/check` runs the two gate phases and
|
||||
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
||||
not evidence that anything ran: a sub-second build reporting success is a
|
||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
||||
and friends destroy a build cache shared with every other build on the host.
|
||||
When a check is added or changed, prove it works by planting a defect it must
|
||||
catch and watching the run fail on it, then revert the defect. A green run
|
||||
alone shows neither that the check ran nor that it covers what it should.
|
||||
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
||||
invalidates a `COPY` layer only when the copied content changes, so on an
|
||||
unchanged tree the check `RUN` is served from cache, nothing executes, and the
|
||||
build still exits 0. Every `docker build` in `script/` therefore passes
|
||||
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
|
||||
id, so no two builds share a value, not even the three in one
|
||||
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
||||
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
||||
two gate phases and `script/fmt-check`, and builds no image of its own. Every
|
||||
stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after
|
||||
`FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps
|
||||
are compared by content and need nothing. A stage without the `ARG` line is
|
||||
served from cache on an unchanged tree, so a repository re-vendors the scripts
|
||||
and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`:
|
||||
it reruns every step too, but it also empties every cache mount, and Go's
|
||||
build cache, which the canonical Go `Dockerfile` below mounts so that compiled
|
||||
packages survive between check builds, would start empty every time. A bare
|
||||
`docker build .` is not evidence that anything ran: a sub-second build
|
||||
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
||||
`docker builder prune` and friends destroy a build cache shared with every
|
||||
other build on the host. When a check is added or changed, prove it works by
|
||||
planting a defect it must catch and watching the run fail on it, then revert
|
||||
the defect. A green run alone shows neither that the check ran nor that it
|
||||
covers what it should.
|
||||
|
||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||
@@ -176,32 +194,42 @@ style conventions are in separate documents:
|
||||
`Dockerfile`:
|
||||
|
||||
```dockerfile
|
||||
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
||||
# on every build, which reruns every RUN step after it. The steps that
|
||||
# compile mount Go's build cache, which survives from build to build.
|
||||
|
||||
# Lint phase
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
ARG CACHEBUST
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not.
|
||||
# image ships and the alpine one does not. -count=1 because the mounted
|
||||
# cache also holds test results, which go test would otherwise replay.
|
||||
# golang:1.x, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS test
|
||||
ARG CACHEBUST
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed.
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
ARG CACHEBUST
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
RUN apk add --no-cache git
|
||||
@@ -217,7 +245,8 @@ style conventions are in separate documents:
|
||||
# .git present, a version that is still empty, dev or unknown fails the
|
||||
# build: git is missing or could not read the checkout.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
@@ -230,6 +259,7 @@ style conventions are in separate documents:
|
||||
|
||||
# Runtime stage, and the last one
|
||||
FROM alpine@sha256:...
|
||||
ARG CACHEBUST
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
@@ -244,6 +274,16 @@ style conventions are in separate documents:
|
||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||
same two copies. A plain `docker build .` builds the last stage's chain
|
||||
and nothing else.
|
||||
- Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps
|
||||
it for root), and only on the steps that compile: `golangci-lint run`,
|
||||
both `go test` runs and `go build`. It keeps compiled packages between
|
||||
check builds, and Go reuses one only when all its inputs are unchanged, so
|
||||
it saves the compile without skipping a check. Nothing else is kept:
|
||||
golangci-lint's own cache would replay stored findings in place of a lint
|
||||
run, and Go builds from whatever files a kept module cache holds without
|
||||
checking them again (only `go mod verify` does), so a file changed there
|
||||
would go unnoticed. Go's build cache also holds test results, which is why
|
||||
both `go test` runs carry `-count=1`.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
@@ -304,18 +344,18 @@ style conventions are in separate documents:
|
||||
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
|
||||
so a successful run means every check passed; a bare `docker build .` does not
|
||||
carry the same guarantee, because its gate phases may come from the cache. The
|
||||
image build is uncached and so runs the gate phases a second time. That is the
|
||||
price of the rule above, and it is worth paying: the image that ships is built
|
||||
from a run of its own gates rather than from a cache entry. The `check` job
|
||||
sets `timeout-minutes: 20`, so a hung build frees the shared runner after 20
|
||||
minutes. That allows for the three Docker builds described above (the test
|
||||
phase, the lint phase, then the image), each held to the 5-minute Docker build
|
||||
limit below, plus the bootstrap. A separate workflow limited to `main` by a
|
||||
`branches` list under `on: push` cannot be checked by review: to try a change
|
||||
to it, add the feature branch to that list and push, then remove the branch
|
||||
from the list again before merging. Keep any job in it that publishes behind
|
||||
`if: github.ref_name == 'main'`, so the run from the feature branch publishes
|
||||
nothing.
|
||||
image build gets a `CACHEBUST` value of its own and so runs the gate phases a
|
||||
second time. That is the price of the rule above, and it is worth paying: the
|
||||
image that ships is built from a run of its own gates rather than from a cache
|
||||
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
|
||||
shared runner after 20 minutes. That allows for the three Docker builds
|
||||
described above (the test phase, the lint phase, then the image), each held to
|
||||
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
|
||||
limited to `main` by a `branches` list under `on: push` cannot be checked by
|
||||
review: to try a change to it, add the feature branch to that list and push,
|
||||
then remove the branch from the list again before merging. Keep any job in it
|
||||
that publishes behind `if: github.ref_name == 'main'`, so the run from the
|
||||
feature branch publishes nothing.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
@@ -375,12 +415,11 @@ style conventions are in separate documents:
|
||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
||||
run again on unchanged code it prints that result, marked `(cached)`,
|
||||
without running them. That matters on a developer's machine, where this
|
||||
target runs and the directory lasts from one run to the next. The `test`
|
||||
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
||||
result for this repo's tests and nothing before its `go test` step runs a
|
||||
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
||||
step run on an unchanged tree.
|
||||
without running them. The directory lasts from one run to the next on a
|
||||
developer's machine, where this target runs, and in the `test` phase of the
|
||||
`Dockerfile`, which mounts it to keep compiled packages between check
|
||||
builds, so the phase's `go test` runs carry `-count=1` too. A new
|
||||
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
|
||||
|
||||
Python example:
|
||||
|
||||
@@ -467,7 +506,7 @@ style conventions are in separate documents:
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
tag="$(script/projectname)"
|
||||
docker build --no-cache \
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$tag" .
|
||||
```
|
||||
@@ -549,6 +588,11 @@ style conventions are in separate documents:
|
||||
function defined and never invoked has the same exit status and the same
|
||||
empty output as one that worked.
|
||||
|
||||
Node is the exception to the whole-token comparison: the canonical
|
||||
`script/bootstrap` compares only its major version, because the node 22
|
||||
alpine image the `Dockerfile` stages start from is not the exact pin, and
|
||||
nvm cannot install a prebuilt node on alpine.
|
||||
|
||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||
|
||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
||||
|
||||
@@ -30,15 +30,43 @@
|
||||
# Completed Steps
|
||||
|
||||
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
|
||||
`c55a0cb`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
||||
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
||||
that write no image, and `make test` runs the suite under the race detector,
|
||||
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
|
||||
gone; every `docker build` in `script/` passes `--no-cache`; prettier runs on
|
||||
the host, from the node and yarn `script/bootstrap` installs, so the
|
||||
`prettier` and `markdown` stages are gone and the build stage installs `git`
|
||||
and `make` itself; a new push cancels the workflow's older run on the same
|
||||
branch, and a run stops after 20 minutes; `.claude/settings.json` is deleted
|
||||
(2026-10-08, https://git.eeqj.de/sneak/sfdupes/issues/95)
|
||||
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
|
||||
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
|
||||
every build, while the steps that compile keep Go's build cache and the tests
|
||||
run with `-count=1`; prettier runs on the host, from the node and yarn
|
||||
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
|
||||
and the build stage installs `git` and `make` itself; a new push cancels the
|
||||
workflow's older run on the same branch, and a run stops after 20 minutes;
|
||||
`.claude/settings.json` is deleted (2026-10-15,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/95)
|
||||
|
||||
- with `-x`, an operand under another operand is walked on its own instead of
|
||||
being dropped when a directory from it (from its directory, for a regular
|
||||
file) up to but not including that operand is on a different filesystem than
|
||||
that operand, whose walk then cannot reach it, and the records of files gone
|
||||
from below it are deleted (2026-10-14,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
||||
|
||||
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
||||
under its operand, and counts their sizes in the size census, instead of
|
||||
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
||||
|
||||
- the hash and content phases open each file without blocking and check that it
|
||||
is still a regular file, so one replaced by a FIFO or device node after it was
|
||||
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
||||
|
||||
- the scan lock file is named from the database's path with symlinks resolved,
|
||||
after `scan` creates the database file if it is missing, so a scan through a
|
||||
symlink to the database fails while another scan holds the lock (2026-10-14,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/101)
|
||||
|
||||
- on a terminal, a warning for the last file of a phase no longer erases the
|
||||
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
||||
|
||||
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
||||
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
||||
@@ -62,7 +90,7 @@
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
||||
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
|
||||
`script/` pass `--no-cache`, so theirs always run
|
||||
`script/` pass a new `CACHEBUST` value, so theirs always run
|
||||
|
||||
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
||||
it; all Markdown reformatted (2026-10-04,
|
||||
@@ -213,7 +241,8 @@
|
||||
served them from cache and the build exited 0 having run nothing. The fix was
|
||||
a `CHECK_EPOCH` build argument; since
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
||||
passes `--no-cache` instead. Run as root, the tests fail
|
||||
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
||||
declares first. Run as root, the tests fail
|
||||
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
|
||||
the test relies on, so the `test` phase runs them as `nobody`
|
||||
- check the installed golangci-lint version in `script/bootstrap` instead of
|
||||
|
||||
@@ -35,6 +35,10 @@ const schemaVersion = 1
|
||||
// scan.
|
||||
const dbDirPerm = 0o755
|
||||
|
||||
// dbFilePerm is the mode for a database file created by scan, the mode
|
||||
// SQLite gives a database file it creates.
|
||||
const dbFilePerm = 0o644
|
||||
|
||||
// lockFilePerm is the mode for the scan lock file. Anyone who can open
|
||||
// the file can hold the lock and keep every scan from running, so it
|
||||
// is open to its owner only.
|
||||
@@ -140,19 +144,37 @@ func openDB(path, params string) (*sql.DB, error) {
|
||||
|
||||
// lockScanDatabase takes the lock that keeps a second scan off the
|
||||
// database at path: an exclusive flock(2) on the file beside it named
|
||||
// path with ".lock" appended, created along with the database's parent
|
||||
// directory if missing. A lock held by another scan fails at once
|
||||
// instead of waiting. The lock lasts until the returned file is closed
|
||||
// or the process ends. The file is never deleted: a scan that deleted
|
||||
// it would let the next scan lock a new file while another still holds
|
||||
// the old one.
|
||||
// by the database's path with symlinks resolved and ".lock" appended,
|
||||
// so every path to one database names the same lock file. If missing,
|
||||
// the database's parent directory and an empty database file are
|
||||
// created first, so the lock file is named the same way before the
|
||||
// database is set up as after. A lock held by another scan fails at
|
||||
// once instead of waiting. The lock lasts until the returned file is
|
||||
// closed or the process ends. The lock file is never deleted: a scan
|
||||
// that deleted it would let the next scan lock a new file while
|
||||
// another still holds the old one.
|
||||
func lockScanDatabase(path string) (*os.File, error) {
|
||||
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create database directory: %w", err)
|
||||
}
|
||||
|
||||
lockPath := path + ".lock"
|
||||
// Like SQLite, this follows a symlink at path and creates its
|
||||
// target. SQLite opens an empty file as a new database.
|
||||
//nolint:gosec // the operator chooses the database path
|
||||
dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create database: %w", err)
|
||||
}
|
||||
|
||||
_ = dbFile.Close()
|
||||
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve database path: %w", err)
|
||||
}
|
||||
|
||||
lockPath := resolved + ".lock"
|
||||
|
||||
//nolint:gosec // the operator chooses the database path
|
||||
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm)
|
||||
|
||||
+10
-2
@@ -13,11 +13,19 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// testDBPath returns a database path inside a fresh temp dir.
|
||||
// testDBPath returns a database path inside a fresh temp dir, with
|
||||
// symlinks resolved as the scan lock resolves them, so the lock file
|
||||
// is this path plus ".lock" even where the temp dir is reached through
|
||||
// a symlink.
|
||||
func testDBPath(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
return filepath.Join(t.TempDir(), "db.sqlite")
|
||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return filepath.Join(dir, "db.sqlite")
|
||||
}
|
||||
|
||||
// openTestDB creates a fresh scan database in a temp dir.
|
||||
|
||||
+93
-4
@@ -822,7 +822,7 @@ func holdScanLock(t *testing.T, path string) {
|
||||
|
||||
func TestRunSecondScanFails(t *testing.T) {
|
||||
// README §Database: while one scan holds the lock, a second scan
|
||||
// fails at once, naming the lock file, without creating the
|
||||
// fails at once, naming the lock file, without opening the
|
||||
// database.
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
@@ -848,10 +848,99 @@ func TestRunSecondScanFails(t *testing.T) {
|
||||
t.Errorf("stdout = %q, want nothing (data only)", got)
|
||||
}
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("stat %s = %v, want the database not created", path, err)
|
||||
// Taking the lock created the database file, empty; opening it as
|
||||
// a database would have written to it.
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if info.Size() != 0 {
|
||||
t.Errorf("database size = %d, want 0 (not opened)", info.Size())
|
||||
}
|
||||
}
|
||||
|
||||
// assertSecondScanFails runs a scan against the database
|
||||
// SFDUPES_DATABASE names while the test holds the lock on it. It fails
|
||||
// the test unless that scan fails at once, naming the lock file beside
|
||||
// path, the database's real path.
|
||||
func assertSecondScanFails(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
|
||||
if code != exitFatal {
|
||||
t.Errorf("run(scan) = %d, want %d", code, exitFatal)
|
||||
}
|
||||
|
||||
want := "sfdupes: another scan is running (lock held on " +
|
||||
path + ".lock)\n"
|
||||
if got := stderr.String(); got != want {
|
||||
t.Errorf("stderr = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||
// README §Database: every path to one database names the same lock
|
||||
// file, so while a scan holds it through one path, a scan through
|
||||
// another fails, naming the lock file beside the database's real
|
||||
// path.
|
||||
t.Run("symlink to the database", func(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
scanFixture(t)
|
||||
|
||||
link := filepath.Join(t.TempDir(), "link.sqlite")
|
||||
|
||||
err := os.Symlink(path, link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
holdScanLock(t, path)
|
||||
t.Setenv(databaseEnv, link)
|
||||
assertSecondScanFails(t, path)
|
||||
})
|
||||
|
||||
t.Run("symlink to its directory, no database yet", func(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
link := filepath.Join(t.TempDir(), "link")
|
||||
|
||||
err := os.Symlink(filepath.Dir(path), link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
holdScanLock(t, path)
|
||||
t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
|
||||
assertSecondScanFails(t, path)
|
||||
})
|
||||
|
||||
t.Run("symlink whose target does not exist yet", func(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
link := filepath.Join(t.TempDir(), "link.sqlite")
|
||||
|
||||
err := os.Symlink(path, link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The first scan takes the lock through the link before the
|
||||
// database exists, then creates the database through it.
|
||||
holdScanLock(t, link)
|
||||
|
||||
db, err := openScanDatabase(t.Context(), link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
closeScanDatabase(t.Context(), db, link)
|
||||
|
||||
t.Setenv(databaseEnv, link)
|
||||
assertSecondScanFails(t, path)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunScanReleasesLock(t *testing.T) {
|
||||
|
||||
@@ -139,6 +139,13 @@ func (p *progress) warnf(format string, args ...any) {
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stderr, msg)
|
||||
|
||||
// The library never draws a finished bar again, so the warning for
|
||||
// a pass's last item would erase it for good: draw its last frame
|
||||
// again below the warning.
|
||||
if p.bar != nil && p.bar.IsFinished() {
|
||||
fmt.Fprint(os.Stderr, p.bar.String())
|
||||
}
|
||||
}
|
||||
|
||||
// finish terminates the pass's display. A bar whose pass stopped short
|
||||
|
||||
@@ -187,3 +187,25 @@ func TestBarStoppedShortKeepsCount(t *testing.T) {
|
||||
t.Errorf("terminal shows %q, want a count of 2 of 10", shown)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarningAfterLastItemKeepsBar checks that a warning printed after
|
||||
// a pass's last item, as for a failed read of the last file, leaves the
|
||||
// pass's finished bar on the terminal below the warning.
|
||||
//
|
||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||
func TestWarningAfterLastItemKeepsBar(t *testing.T) {
|
||||
stderr := captureStderr(t)
|
||||
p := &progress{
|
||||
label: "hash", total: 2, start: time.Now(),
|
||||
bar: newBar("hash", 2),
|
||||
}
|
||||
|
||||
p.increment()
|
||||
p.increment()
|
||||
p.warnf("warning")
|
||||
p.finish()
|
||||
|
||||
if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/2,") {
|
||||
t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,10 +203,16 @@ func resolveRoots(roots []string) ([]string, error) {
|
||||
}
|
||||
|
||||
// pruneRoots drops operands already covered by another operand:
|
||||
// duplicates and any operand lying under another one. Every remaining
|
||||
// file is then reachable through exactly one root, so walked paths are
|
||||
// unique without keeping a scan-wide set of every path seen.
|
||||
func pruneRoots(roots []string) []string {
|
||||
// duplicates and any operand lying under another one. With -x, an
|
||||
// operand under another one is kept when the walk of the deepest kept
|
||||
// operand above it cannot reach it: when device puts any directory from
|
||||
// it (from its directory, for a regular file) up to but not including
|
||||
// that operand on a different filesystem than that operand. Every
|
||||
// remaining file is then reachable through exactly one root, so walked
|
||||
// paths are unique without keeping a scan-wide set of every path seen.
|
||||
func pruneRoots(roots []string, oneFS bool,
|
||||
device func(path string) (uint64, bool),
|
||||
) []string {
|
||||
// Shorter-first ordering guarantees an ancestor is kept before any
|
||||
// operand under it is considered.
|
||||
sorted := slices.Clone(roots)
|
||||
@@ -221,14 +227,58 @@ func pruneRoots(roots []string) []string {
|
||||
kept := make([]string, 0, len(sorted))
|
||||
|
||||
for _, r := range sorted {
|
||||
if !underAnyRoot(r, kept) {
|
||||
above := deepestRoot(r, kept)
|
||||
if above == "" {
|
||||
kept = append(kept, r)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Without -x, and always for a duplicate, r is dropped.
|
||||
if !oneFS || above == r {
|
||||
continue
|
||||
}
|
||||
|
||||
aboveDev, aboveOK := device(above)
|
||||
if !aboveOK {
|
||||
continue
|
||||
}
|
||||
|
||||
// The walk of above enters a directory only when it is on
|
||||
// above's filesystem, and then reaches every regular file in it
|
||||
// whatever device the file reports, so for a regular-file
|
||||
// operand the directories checked start at its directory.
|
||||
dir := r
|
||||
|
||||
fi, err := os.Lstat(r)
|
||||
if err == nil && fi.Mode().IsRegular() {
|
||||
dir = filepath.Dir(r)
|
||||
}
|
||||
|
||||
for ; dir != above; dir = filepath.Dir(dir) {
|
||||
dev, ok := device(dir)
|
||||
if ok && dev != aboveDev {
|
||||
kept = append(kept, r)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return kept
|
||||
}
|
||||
|
||||
// deviceOfPath returns the filesystem device ID of the file at path,
|
||||
// when lstat succeeds and the platform exposes one.
|
||||
func deviceOfPath(path string) (uint64, bool) {
|
||||
fi, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return deviceOfInfo(fi)
|
||||
}
|
||||
|
||||
// scanStats summarizes one scan's database synchronization for the
|
||||
// final stderr summary, or for the line an interrupted scan prints.
|
||||
type scanStats struct {
|
||||
@@ -288,21 +338,22 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
|
||||
// size, head, and tail match another record's). Records outside the
|
||||
// roots are never touched, except that the content phase fills in
|
||||
// their content hash. Operands the walk cannot start from are dropped
|
||||
// first, so the records beneath them count as outside the roots unless
|
||||
// they lie under another root.
|
||||
// first, so the records beneath them are treated as if those operands
|
||||
// had not been given.
|
||||
func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||
workers int, oneFS bool,
|
||||
) error {
|
||||
// Types are checked before pruning so that an operand under a
|
||||
// dropped one is still scanned, not dropped as lying under it.
|
||||
roots = pruneRoots(s.walkableRoots(roots))
|
||||
roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath)
|
||||
|
||||
err := s.loadIndex(ctx, roots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers))
|
||||
changed, unhashed := s.walkPhase(
|
||||
startWalk(ctx, roots, oneFS, workers), roots)
|
||||
|
||||
// A cancelled walk stops early, so its size census covers only part
|
||||
// of the roots, and every file it never reached would look vanished
|
||||
@@ -330,10 +381,10 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||
// walkableRoots returns the operands the walk can start from: regular
|
||||
// files, and directories not named .zfs. Every other operand is warned
|
||||
// about, counted as skipped, and dropped. A dropped operand is no
|
||||
// longer a root, so the records stored beneath it count as outside the
|
||||
// roots and are not deleted as unverified, unless it lies under another
|
||||
// root. An operand that fails lstat here is kept, and the walk warns
|
||||
// about it.
|
||||
// longer a root, so the records stored beneath it are treated as if it
|
||||
// had not been given: those under no other root count as outside the
|
||||
// roots and are not deleted as unverified. An operand that fails lstat
|
||||
// here is kept, and the walk warns about it.
|
||||
func (s *scanState) walkableRoots(roots []string) []string {
|
||||
kept := make([]string, 0, len(roots))
|
||||
|
||||
@@ -391,15 +442,27 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||
// (the content phase may still fill in its content hash). It returns
|
||||
// the new-or-changed files and the unchanged files whose records lack
|
||||
// hashes; both remain candidates until the census decides whether
|
||||
// their sizes are shared.
|
||||
// their sizes are shared. A record below a directory -x did not enter
|
||||
// counts as outside the roots: it is not deleted, and its size joins
|
||||
// the census. That does not hold below a root at or inside that
|
||||
// directory, since that root's own walk verifies the records below it.
|
||||
func (s *scanState) walkPhase(
|
||||
events <-chan walkEvent,
|
||||
events <-chan walkEvent, roots []string,
|
||||
) ([]fileRec, []fileRec) {
|
||||
prog := newProgress("walk", -1)
|
||||
|
||||
var changed, unhashed []fileRec
|
||||
var (
|
||||
changed, unhashed []fileRec
|
||||
otherFSDirs []string
|
||||
)
|
||||
|
||||
for ev := range events {
|
||||
if ev.otherFSDir != "" {
|
||||
otherFSDirs = append(otherFSDirs, ev.otherFSDir)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if ev.fail {
|
||||
s.st.skipped++
|
||||
|
||||
@@ -429,6 +492,19 @@ func (s *scanState) walkPhase(
|
||||
|
||||
prog.finish()
|
||||
|
||||
for path, old := range s.existing {
|
||||
// Kept only when a directory -x did not enter lies below the
|
||||
// deepest root above the record. A root at or below that
|
||||
// directory was walked on its own and did not find the file.
|
||||
// Both contain the record, so the longer path is the deeper.
|
||||
otherFSDir := deepestRoot(path, otherFSDirs)
|
||||
if len(otherFSDir) > len(deepestRoot(path, roots)) {
|
||||
delete(s.existing, path)
|
||||
|
||||
s.sizes = append(s.sizes, old.size)
|
||||
}
|
||||
}
|
||||
|
||||
return changed, unhashed
|
||||
}
|
||||
|
||||
@@ -856,6 +932,20 @@ func underRoot(path, root string) bool {
|
||||
return strings.HasPrefix(path, prefix)
|
||||
}
|
||||
|
||||
// deepestRoot returns the longest of the roots that path is or lies
|
||||
// under, or "" when it lies under none.
|
||||
func deepestRoot(path string, roots []string) string {
|
||||
deepest := ""
|
||||
|
||||
for _, root := range roots {
|
||||
if underRoot(path, root) && len(root) > len(deepest) {
|
||||
deepest = root
|
||||
}
|
||||
}
|
||||
|
||||
return deepest
|
||||
}
|
||||
|
||||
// dirJob is one directory awaiting traversal by the walk workers. It
|
||||
// carries its operand's filesystem device so -x can stop at
|
||||
// filesystem boundaries.
|
||||
@@ -866,19 +956,23 @@ type dirJob struct {
|
||||
}
|
||||
|
||||
// walkEvent is one walk result delivered to the walk phase: a regular
|
||||
// file's statted record, or a warning when fail is set.
|
||||
// file's statted record, a warning when fail is set, or, when
|
||||
// otherFSDir is set, a directory -x did not enter because it is on
|
||||
// another filesystem than its operand.
|
||||
type walkEvent struct {
|
||||
rec fileRec
|
||||
warn string
|
||||
fail bool
|
||||
rec fileRec
|
||||
warn string
|
||||
fail bool
|
||||
otherFSDir string
|
||||
}
|
||||
|
||||
// startWalk seeds every root into the shared walk worker pool and
|
||||
// returns the event stream: one record per regular file, one warning
|
||||
// event per per-path error. The channel is closed when the walk
|
||||
// completes, and also when ctx is cancelled — every goroutine in the
|
||||
// pool abandons its blocking send in that case, so the consumer sees a
|
||||
// truncated but properly terminated stream instead of a stalled one.
|
||||
// event per per-path error, and one event per directory -x did not
|
||||
// enter. The channel is closed when the walk completes, and also when
|
||||
// ctx is cancelled — every goroutine in the pool abandons its blocking
|
||||
// send in that case, so the consumer sees a truncated but properly
|
||||
// terminated stream instead of a stalled one.
|
||||
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
||||
workers int,
|
||||
) <-chan walkEvent {
|
||||
@@ -945,7 +1039,8 @@ func operandWarning(root string, fi fs.FileInfo) string {
|
||||
// operand becomes an initial job. walkableRoots has already dropped
|
||||
// every other operand. One that has changed into something else since
|
||||
// is warned about and skipped here; it is still a root, so the records
|
||||
// stored beneath it are deleted as unverified.
|
||||
// stored beneath it, other than those under another root, are deleted
|
||||
// as unverified.
|
||||
func seedRoot(ctx context.Context, root string,
|
||||
events chan<- walkEvent,
|
||||
) []dirJob {
|
||||
@@ -1144,7 +1239,9 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
||||
// subdirJob applies the descent rules to directory p: never enter
|
||||
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
|
||||
// snapshot), and with -x never enter a directory on a different
|
||||
// filesystem than its operand.
|
||||
// filesystem than its operand, sending an event that names it so the
|
||||
// scan keeps the records below it, except those below an operand at or
|
||||
// inside it.
|
||||
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||
parent dirJob, oneFS bool, events chan<- walkEvent,
|
||||
) (dirJob, bool) {
|
||||
@@ -1168,6 +1265,8 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||
}
|
||||
|
||||
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
|
||||
sendEvent(ctx, events, walkEvent{otherFSDir: p})
|
||||
|
||||
return dirJob{}, false
|
||||
}
|
||||
|
||||
@@ -1309,6 +1408,38 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
|
||||
}
|
||||
}
|
||||
|
||||
// errNotRegular reports a path that is no longer a regular file when it
|
||||
// is opened to be read.
|
||||
var errNotRegular = errors.New("not a regular file")
|
||||
|
||||
// openRegularFile opens path for reading and returns it only if it is
|
||||
// still a regular file, since the path may have been replaced after it
|
||||
// was statted. O_NONBLOCK keeps the open of a FIFO or a device node
|
||||
// from waiting forever, and fstat then rejects it. Reads of a regular
|
||||
// file ignore O_NONBLOCK.
|
||||
func openRegularFile(path string) (*os.File, error) {
|
||||
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !fi.Mode().IsRegular() {
|
||||
_ = f.Close()
|
||||
|
||||
return nil, errNotRegular
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
|
||||
// tail, and content hash of every zero-length file.
|
||||
const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
|
||||
@@ -1331,8 +1462,7 @@ func hashSignature(path string, size int64) (string, string, string, error) {
|
||||
return emptyHash, emptyHash, emptyHash, nil
|
||||
}
|
||||
|
||||
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||
f, err := os.Open(path)
|
||||
f, err := openRegularFile(path)
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
@@ -1363,8 +1493,7 @@ func hashSignature(path string, size int64) (string, string, string, error) {
|
||||
// tail are returned empty, because the content phase keeps the ones its
|
||||
// records already hold.
|
||||
func hashContentOnly(path string, size int64) (string, string, string, error) {
|
||||
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||
f, err := os.Open(path)
|
||||
f, err := openRegularFile(path)
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
|
||||
+378
-4
@@ -997,7 +997,7 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry {
|
||||
}
|
||||
|
||||
// callSubdirJob runs subdirJob against e under parent, collecting any
|
||||
// warning events it emits (subdirJob emits at most one).
|
||||
// events it emits (subdirJob emits at most one).
|
||||
func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
|
||||
parent dirJob, oneFS bool,
|
||||
) (dirJob, bool, []walkEvent) {
|
||||
@@ -1075,8 +1075,15 @@ func TestSubdirJobOneFilesystem(t *testing.T) {
|
||||
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
|
||||
}
|
||||
|
||||
if len(evs) != 0 {
|
||||
t.Fatalf("unexpected events: %+v", evs)
|
||||
// A refused descent names the directory, so the scan keeps
|
||||
// the records below it.
|
||||
var wantEvs []walkEvent
|
||||
if !ok {
|
||||
wantEvs = []walkEvent{{otherFSDir: sub}}
|
||||
}
|
||||
|
||||
if !slices.Equal(evs, wantEvs) {
|
||||
t.Fatalf("events = %+v, want %+v", evs, wantEvs)
|
||||
}
|
||||
|
||||
// The accepted job must carry the operand's device down, or -x
|
||||
@@ -1128,6 +1135,296 @@ func TestSubdirJobStatError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// syncOneFilesystem scans roots into db with -x and returns the scan
|
||||
// stats. Every device pruneRoots looks up, and the device each
|
||||
// operand's walk compares its subdirectories with, comes from devices
|
||||
// instead of from lstat, so a test can put a path on a filesystem that
|
||||
// no directory here is on. Unlike runPhases, it skips walkableRoots,
|
||||
// the check for a cancelled walk, and the content phase.
|
||||
func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
|
||||
devices map[string]uint64,
|
||||
) scanStats {
|
||||
t.Helper()
|
||||
|
||||
device := func(path string) (uint64, bool) {
|
||||
dev, ok := devices[path]
|
||||
|
||||
return dev, ok
|
||||
}
|
||||
|
||||
roots = pruneRoots(roots, true, device)
|
||||
|
||||
ctx := t.Context()
|
||||
s := &scanState{db: db}
|
||||
|
||||
err := s.loadIndex(ctx, roots)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
|
||||
|
||||
// seedRoot emits a regular-file operand itself and returns a job
|
||||
// only for a directory operand.
|
||||
var initial []dirJob
|
||||
|
||||
for _, root := range roots {
|
||||
for _, job := range seedRoot(ctx, root, events) {
|
||||
job.rootDev = devices[root]
|
||||
initial = append(initial, job)
|
||||
}
|
||||
}
|
||||
|
||||
dispatchDirs(ctx, initial, jobs, subdirs)
|
||||
|
||||
changed, unhashed := s.walkPhase(events, roots)
|
||||
s.partition(changed, unhashed)
|
||||
|
||||
err = s.hashPhase(ctx, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.updatePhase(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return s.st
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
|
||||
// record below a directory the walk does not enter, because it is on
|
||||
// another filesystem than its operand, counts as outside the operands:
|
||||
// it is not deleted, and its size joins the size census. The record of
|
||||
// a gone file that is not below that directory is still deleted, even
|
||||
// when its path begins with the directory's name. The operand is given
|
||||
// a device that no directory here is on, so no second real filesystem
|
||||
// is needed.
|
||||
func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
|
||||
// A scan without -x records the file below mnt, and mnt2, which is
|
||||
// beside mnt, not below it.
|
||||
mounted := writeFile(t, dir, "mnt/f", pattern(1, 100))
|
||||
gone := writeFile(t, dir, "mnt2", pattern(3, 50))
|
||||
syncTree(t, db, dir)
|
||||
|
||||
err := os.Remove(gone)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The same size as mnt/f, so it is hashed only if mnt/f's size is
|
||||
// in the census.
|
||||
added := writeFile(t, dir, "added", pattern(2, 100))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1})
|
||||
|
||||
recs := dbRecords(t, db)
|
||||
recordByPath(t, recs, mounted)
|
||||
|
||||
if recordByPath(t, recs, added).head == "" {
|
||||
t.Errorf("%s not hashed: the census lacks the size of %s",
|
||||
added, mounted)
|
||||
}
|
||||
|
||||
if slices.Contains(recordPaths(recs), gone) {
|
||||
t.Errorf("record for %s kept: the file is gone and not below mnt",
|
||||
gone)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemWalksNestedOperand checks that with -x an
|
||||
// operand under another one, on a different filesystem than it, is
|
||||
// walked on its own: a file added below it is recorded, and the record
|
||||
// of a file gone from below it is deleted. A record below a directory
|
||||
// that its own walk does not enter is still kept. Each operand is given
|
||||
// a device that no directory here is on, so neither walk enters a
|
||||
// subdirectory.
|
||||
func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
|
||||
// A scan without -x records both files below mnt.
|
||||
gone := writeFile(t, dir, "mnt/gone", pattern(1, 10))
|
||||
kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20))
|
||||
syncTree(t, db, dir)
|
||||
|
||||
err := os.Remove(gone)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
added := writeFile(t, dir, "mnt/added", pattern(3, 30))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
mnt := filepath.Join(dir, "mnt")
|
||||
syncOneFilesystem(t, db, []string{dir, mnt},
|
||||
map[string]uint64{dir: dev + 1, mnt: dev + 2})
|
||||
|
||||
paths := recordPaths(dbRecords(t, db))
|
||||
|
||||
if !slices.Contains(paths, added) {
|
||||
t.Errorf("no record for %s: the walk did not reach %s", added, mnt)
|
||||
}
|
||||
|
||||
if slices.Contains(paths, gone) {
|
||||
t.Errorf("record for %s kept: the file is gone", gone)
|
||||
}
|
||||
|
||||
if !slices.Contains(paths, kept) {
|
||||
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
|
||||
kept, mnt, filepath.Dir(kept))
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that
|
||||
// with -x an operand on the same filesystem as the operand covering it
|
||||
// is walked on its own when a directory between them is on another
|
||||
// filesystem, since the walk of the covering operand does not enter
|
||||
// that directory: a file added below the nested operand is recorded,
|
||||
// and the record of a file gone from below it is deleted. A record
|
||||
// below that directory but outside the nested operand is still kept.
|
||||
// Both operands are given one device that no directory here is on,
|
||||
// while mnt keeps its own.
|
||||
func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
|
||||
// A scan without -x records both files below mnt.
|
||||
gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10))
|
||||
kept := writeFile(t, dir, "mnt/kept", pattern(2, 20))
|
||||
syncTree(t, db, dir)
|
||||
|
||||
err := os.Remove(gone)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
added := writeFile(t, dir, "mnt/back/added", pattern(3, 30))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
mnt := filepath.Join(dir, "mnt")
|
||||
back := filepath.Join(mnt, "back")
|
||||
syncOneFilesystem(t, db, []string{dir, back},
|
||||
map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1})
|
||||
|
||||
paths := recordPaths(dbRecords(t, db))
|
||||
|
||||
if !slices.Contains(paths, added) {
|
||||
t.Errorf("no record for %s: the walk did not reach %s", added, back)
|
||||
}
|
||||
|
||||
if slices.Contains(paths, gone) {
|
||||
t.Errorf("record for %s kept: the file is gone", gone)
|
||||
}
|
||||
|
||||
if !slices.Contains(paths, kept) {
|
||||
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
|
||||
kept, dir, mnt)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
|
||||
// regular-file operand under another operand is dropped when the file
|
||||
// reports a different device than that operand but its directory does
|
||||
// not, as a file bind-mounted into the tree does: the walk of the other
|
||||
// operand enters the directory and reaches the file, so the file is
|
||||
// recorded once.
|
||||
func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
file := writeFile(t, dir, "sub/f", pattern(1, 10))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
|
||||
dir: dev, filepath.Dir(file): dev, file: dev + 1,
|
||||
})
|
||||
|
||||
if st.added != 1 {
|
||||
t.Errorf("%s recorded %d times, want once", file, st.added)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a
|
||||
// regular-file operand under another operand is kept when its directory
|
||||
// is on a different filesystem than that operand: the walk of the other
|
||||
// operand does not enter the directory, so the file is recorded once,
|
||||
// as its own operand. The other operand is given a device that no
|
||||
// directory here is on.
|
||||
func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
file := writeFile(t, dir, "sub/f", pattern(1, 10))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
|
||||
dir: dev + 1, filepath.Dir(file): dev,
|
||||
})
|
||||
|
||||
if st.added != 1 {
|
||||
t.Errorf("%s recorded %d times, want once", file, st.added)
|
||||
}
|
||||
}
|
||||
|
||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||
// with deterministic content and returns the tree root.
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
@@ -1797,6 +2094,54 @@ func TestScanHardlinkRunFailsTogether(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file
|
||||
// with a FIFO before the hash phase reads it. Opening a FIFO that has
|
||||
// no writer waits for one, so a blocking open would hang the phase. The
|
||||
// path must be warned about once and skipped, with no record written.
|
||||
//
|
||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||
func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) {
|
||||
stderr := captureStderr(t)
|
||||
dir := t.TempDir()
|
||||
p := writeFile(t, dir, "a.bin", pattern(1, 300))
|
||||
|
||||
walked, _ := collectWalk(t, []string{dir}, false, 1)
|
||||
|
||||
err := os.Remove(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = unix.Mkfifo(p, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := &scanState{db: openTestDB(t), toHash: walked}
|
||||
|
||||
err = s.hashPhase(t.Context(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("hashPhase = %v, want nil", err)
|
||||
}
|
||||
|
||||
if s.st.skipped != 1 || len(s.batch) != 0 {
|
||||
t.Errorf("skipped = %d, records = %d, want 1 skipped and none",
|
||||
s.st.skipped, len(s.batch))
|
||||
}
|
||||
|
||||
warning := "hash " + p + ": not a regular file\n"
|
||||
if got := stderr(); strings.Count(got, warning) != 1 {
|
||||
t.Errorf("stderr = %q, want %q once", got, warning)
|
||||
}
|
||||
|
||||
// The content phase opens files the same way.
|
||||
_, _, content, err := hashContentOnly(p, headTailMin)
|
||||
if content != "" || !errors.Is(err, errNotRegular) {
|
||||
t.Errorf("hashContentOnly on a FIFO = %q, %v; want no hash and %v",
|
||||
content, err, errNotRegular)
|
||||
}
|
||||
}
|
||||
|
||||
// injectedWriteFailure is the message the injected database trigger
|
||||
// aborts with, so the test can recognize its own failure in the error
|
||||
// the scan reports.
|
||||
@@ -1990,14 +2335,43 @@ func TestHashRuns(t *testing.T) {
|
||||
func TestPruneRoots(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const pqRoot = "/p/q"
|
||||
|
||||
// /p/q is on another filesystem than /p; /p/q/r is on the same one as
|
||||
// /p/q, /p/s as /p, and /p/t has no known device.
|
||||
devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1}
|
||||
device := func(path string) (uint64, bool) {
|
||||
dev, ok := devices[path]
|
||||
|
||||
return dev, ok
|
||||
}
|
||||
|
||||
// Duplicates and operands under other operands are dropped; /cc is
|
||||
// not under /c (sibling with a shared prefix).
|
||||
got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
|
||||
got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"},
|
||||
false, device)
|
||||
|
||||
want := []string{"/a", "/c", "/cc"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("pruneRoots = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"}
|
||||
|
||||
got = pruneRoots(nested, false, device)
|
||||
|
||||
want = []string{"/p"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("pruneRoots without -x = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// With -x, /p/q, on another filesystem than /p, is kept as well.
|
||||
got = pruneRoots(nested, true, device)
|
||||
|
||||
want = []string{"/p", pqRoot}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("pruneRoots with -x = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportsNeverTouchFilesystem(t *testing.T) {
|
||||
|
||||
+34
-11
@@ -2,10 +2,12 @@
|
||||
# script/bootstrap: install all dependencies needed to build and develop
|
||||
# this repo. Idempotent: every install is guarded by a check so already
|
||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||
# or apk (detected in that order); assumes nothing is present. Node is
|
||||
# used directly if installed; otherwise it is installed at a pinned
|
||||
# version via nvm (installing nvm itself first, from a hash-verified
|
||||
# release archive, never curl | sh).
|
||||
# or apk (detected in that order); assumes nothing is present. The
|
||||
# installed node is used only when its major version is the pinned one;
|
||||
# otherwise (node missing, or another major) the pinned version is
|
||||
# installed via nvm (installing nvm itself first, from a hash-verified
|
||||
# release archive, never curl | sh), and yarn and the packages are
|
||||
# installed under it.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
@@ -100,31 +102,52 @@ ensure_nvm() {
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
# True when the node on PATH has the pinned major version. Only the
|
||||
# major is compared: the Dockerfile stages start from a node 22 alpine
|
||||
# image whose exact version is not the pin, and nvm cannot install a
|
||||
# prebuilt node on alpine. Another major is not used: the pinned yarn 1
|
||||
# prints a deprecation warning under node 24. script/fmt and
|
||||
# script/fmt-check carry this function unchanged, to run the yarn
|
||||
# installed here.
|
||||
node_is_pinned_major() {
|
||||
if ! command -v node >/dev/null 2>&1; then return 1; fi
|
||||
installed="$(node --version)"
|
||||
installed="${installed#v}"
|
||||
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
|
||||
}
|
||||
|
||||
ensure_node() {
|
||||
if ! missing node; then return 0; fi
|
||||
if node_is_pinned_major; then
|
||||
echo "bootstrap: using node $(node --version)"
|
||||
return 0
|
||||
fi
|
||||
ensure_nvm
|
||||
nvm_sh "nvm install $NODE_VERSION"
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && \
|
||||
echo \"bootstrap: using node \$(node --version)\""
|
||||
}
|
||||
|
||||
ensure_yarn() {
|
||||
if ! node_is_pinned_major; then
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
||||
corepack prepare yarn@$YARN_VERSION --activate"
|
||||
return 0
|
||||
fi
|
||||
if ! missing yarn; then return 0; fi
|
||||
if ! missing corepack; then
|
||||
corepack enable
|
||||
corepack prepare "yarn@$YARN_VERSION" --activate
|
||||
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
||||
corepack prepare yarn@$YARN_VERSION --activate"
|
||||
else
|
||||
npm install -g "yarn@$YARN_VERSION"
|
||||
fi
|
||||
}
|
||||
|
||||
install_js_deps() {
|
||||
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
if node_is_pinned_major; then
|
||||
yarn install --frozen-lockfile
|
||||
else
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
|
||||
yarn install --frozen-lockfile"
|
||||
else
|
||||
yarn install --frozen-lockfile
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
+6
-4
@@ -2,9 +2,11 @@
|
||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||
# the formatter on the host, which a pristine checkout cannot do.
|
||||
# --no-cache for the same reason as script/docker: the gate phases the
|
||||
# final stage depends on are RUN steps, and a cached one is a check that
|
||||
# did not run.
|
||||
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
||||
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
||||
# first; a stage without that line is served from cache on an unchanged
|
||||
# tree, with no warning.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -22,7 +24,7 @@ main() {
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
tag="$("$SCRIPT_DIR/projectname")"
|
||||
docker build --no-cache \
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$tag" .
|
||||
}
|
||||
|
||||
+6
-3
@@ -1,8 +1,11 @@
|
||||
#!/bin/sh
|
||||
# script/docker: build the Docker image tagged with the project name.
|
||||
# Identical in all repos; the tag comes from script/projectname.
|
||||
# --no-cache because the gate phases the final stage depends on are RUN
|
||||
# steps, and a cached one is a check that did not run.
|
||||
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
||||
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
||||
# first; a stage without that line is served from cache on an unchanged
|
||||
# tree, with no warning.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -18,7 +21,7 @@ main() {
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
tag="$("$SCRIPT_DIR/projectname")"
|
||||
docker build --no-cache \
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$tag" .
|
||||
}
|
||||
|
||||
+16
-6
@@ -7,15 +7,25 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
# True when the node on PATH has the pinned major version: the same
|
||||
# function as in script/bootstrap, which uses it to decide where it
|
||||
# installs yarn.
|
||||
node_is_pinned_major() {
|
||||
if ! command -v node >/dev/null 2>&1; then return 1; fi
|
||||
installed="$(node --version)"
|
||||
installed="${installed#v}"
|
||||
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
|
||||
}
|
||||
|
||||
# Run the yarn script/bootstrap installed: the yarn on PATH when the
|
||||
# node on PATH has the pinned major version, and otherwise yarn under
|
||||
# the pinned node in nvm, which bootstrap leaves off the PATH of the
|
||||
# shell that called it. nvm is a bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
+16
-6
@@ -7,15 +7,25 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
# True when the node on PATH has the pinned major version: the same
|
||||
# function as in script/bootstrap, which uses it to decide where it
|
||||
# installs yarn.
|
||||
node_is_pinned_major() {
|
||||
if ! command -v node >/dev/null 2>&1; then return 1; fi
|
||||
installed="$(node --version)"
|
||||
installed="${installed#v}"
|
||||
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
|
||||
}
|
||||
|
||||
# Run the yarn script/bootstrap installed: the yarn on PATH when the
|
||||
# node on PATH has the pinned major version, and otherwise yarn under
|
||||
# the pinned node in nvm, which bootstrap leaves off the PATH of the
|
||||
# shell that called it. nvm is a bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt-check: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
+7
-4
@@ -5,9 +5,12 @@
|
||||
# make its answer untrustworthy.
|
||||
#
|
||||
# The phase is not the last stage in the file, so it is built only when
|
||||
# --target names it. --no-cache because a cached lint layer is a lint
|
||||
# that did not run. --output type=cacheonly writes no image, since
|
||||
# nothing uses one.
|
||||
# --target names it. A new CACHEBUST value reruns every RUN step, since
|
||||
# a cached lint layer is a lint that did not run; unlike --no-cache, it
|
||||
# keeps Go's build cache. This relies on every stage of the Dockerfile
|
||||
# declaring ARG CACHEBUST first; a stage without that line is served
|
||||
# from cache on an unchanged tree, with no warning.
|
||||
# --output type=cacheonly writes no image, since nothing uses one.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -15,7 +18,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
docker build --no-cache \
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--target lint \
|
||||
--output type=cacheonly .
|
||||
}
|
||||
|
||||
+7
-3
@@ -2,8 +2,12 @@
|
||||
# script/test: run the test suite. Testing is a phase of the Dockerfile
|
||||
# and this builds that phase alone, on the same terms as script/lint:
|
||||
# --target because a phase that is not the last stage is built only when
|
||||
# named, and --no-cache because a cached test layer is a test that did
|
||||
# not run. --output type=cacheonly writes no image, since nothing uses one.
|
||||
# named, and a new CACHEBUST value because a cached test layer is a test
|
||||
# that did not run; unlike --no-cache, it keeps Go's build cache. This
|
||||
# relies on every stage of the Dockerfile declaring ARG CACHEBUST first;
|
||||
# a stage without that line is served from cache on an unchanged tree,
|
||||
# with no warning. --output type=cacheonly writes no image, since
|
||||
# nothing uses one.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -11,7 +15,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
docker build --no-cache \
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--target test \
|
||||
--output type=cacheonly .
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user