Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c64bbbb78e |
+5
-20
@@ -1,13 +1,8 @@
|
|||||||
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
|
||||||
# on every build, which reruns every RUN step after it. The steps that
|
|
||||||
# compile mount Go's build cache, which survives from build to build.
|
|
||||||
|
|
||||||
# Lint phase, built alone by script/lint. The tools are invoked directly
|
# Lint phase, built alone by script/lint. The tools are invoked directly
|
||||||
# rather than through `make lint`, which runs docker itself and so cannot
|
# rather than through `make lint`, which runs docker itself and so cannot
|
||||||
# run inside a build step.
|
# run inside a build step.
|
||||||
# golangci/golangci-lint:v2.14.0, 2026-10-07
|
# golangci/golangci-lint:v2.14.0, 2026-10-07
|
||||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||||
ARG CACHEBUST
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
@@ -23,39 +18,31 @@ RUN files="$(gofmt -s -l .)" && \
|
|||||||
|
|
||||||
# Validates .golangci.yml against the schema the pinned binary embeds.
|
# Validates .golangci.yml against the schema the pinned binary embeds.
|
||||||
RUN golangci-lint config verify --config .golangci.yml
|
RUN golangci-lint config verify --config .golangci.yml
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN golangci-lint run --config .golangci.yml ./...
|
||||||
golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
|
||||||
# Test phase, built alone by script/test. -race needs cgo and so a C
|
# Test phase, built alone by script/test. -race needs cgo and so a C
|
||||||
# compiler, which the Debian Go image ships and the alpine one does not.
|
# compiler, which the Debian Go image ships and the alpine one does not.
|
||||||
# -count=1 because the mounted cache also holds test results, which go
|
|
||||||
# test would otherwise replay.
|
|
||||||
#
|
#
|
||||||
# The tests run as nobody: several of them make a file unreadable and
|
# The tests run as nobody: several of them make a file unreadable and
|
||||||
# expect reading it to fail, and root reads it anyway. nobody has no home
|
# expect reading it to fail, and root reads it anyway. nobody has no home
|
||||||
# directory, so HOME is /tmp, and Go keeps its build cache in
|
# directory, so HOME is /tmp, where Go puts its build cache.
|
||||||
# /tmp/.cache/go-build. That is where the cache is mounted, owned by
|
|
||||||
# nobody (65534) so the tests can write to it.
|
|
||||||
# golang:1.25-trixie, 2026-10-04
|
# golang:1.25-trixie, 2026-10-04
|
||||||
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
|
FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
|
||||||
ARG CACHEBUST
|
|
||||||
USER nobody
|
USER nobody
|
||||||
ENV HOME=/tmp
|
ENV HOME=/tmp
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
|
RUN go test -timeout 90s -race -cover ./... || \
|
||||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies are
|
# Build stage. Nothing is wanted from either phase above; the copies are
|
||||||
# what make BuildKit build them first, so this stage cannot run unless
|
# what make BuildKit build them first, so this stage cannot run unless
|
||||||
# lint and test passed.
|
# lint and test passed.
|
||||||
# golang:1.25-alpine, 2026-07-23
|
# golang:1.25-alpine, 2026-07-23
|
||||||
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
|
FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
COPY --from=test /src/go.sum /dev/null
|
||||||
RUN apk add --no-cache git make
|
RUN apk add --no-cache git make
|
||||||
@@ -73,8 +60,7 @@ COPY . .
|
|||||||
# "dev". `make build` rather than `go build`, so the image and a host
|
# "dev". `make build` rather than `go build`, so the image and a host
|
||||||
# build share one compile recipe, cgo disabled included.
|
# build share one compile recipe, cgo disabled included.
|
||||||
ARG VERSION
|
ARG VERSION
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
|
||||||
version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
|
|
||||||
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
|
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
|
||||||
[ "$version" = unknown ]; }; then \
|
[ "$version" = unknown ]; }; then \
|
||||||
echo "no version could be derived although the build context carries .git" >&2; \
|
echo "no version could be derived although the build context carries .git" >&2; \
|
||||||
@@ -86,7 +72,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
|||||||
# stage's chain and nothing else.
|
# stage's chain and nothing else.
|
||||||
# alpine:3.22, 2026-07-23
|
# alpine:3.22, 2026-07-23
|
||||||
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
|
FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
|
||||||
ARG CACHEBUST
|
|
||||||
|
|
||||||
COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
|
COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
|
||||||
|
|
||||||
|
|||||||
@@ -271,21 +271,16 @@ All three subcommands operate on a single SQLite database file:
|
|||||||
error (exit 1) telling the user to run `scan` first.
|
error (exit 1) telling the user to run `scan` first.
|
||||||
- Only one `scan` runs against a database at a time. For its whole run, `scan`
|
- Only one `scan` runs against a database at a time. For its whole run, `scan`
|
||||||
holds an exclusive `flock(2)` lock on a lock file beside the database, named
|
holds an exclusive `flock(2)` lock on a lock file beside the database, named
|
||||||
by appending `.lock` to the database's path with symlinks resolved
|
by appending `.lock` to the database path (`/var/lib/sfdupes/db.sqlite.lock`
|
||||||
(`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the
|
by default), taken before it walks the filesystem or opens the database. A
|
||||||
filesystem or opens the database. Every path to one database, through a
|
second `scan` against the same database does not wait: it fails at once with a
|
||||||
symlink to the file or to a directory above it, names the same lock file. Just
|
one-line error naming the lock file and exits 1, without walking anything or
|
||||||
before it takes the lock, `scan` creates the database file, empty, if it is
|
opening the database, and the running scan carries on. The lock file is
|
||||||
missing (through a symlink at the database path, it creates the symlink's
|
created on first use, open to its owner only, and left in place: a leftover
|
||||||
target), so the lock file is named the same way before the first scan sets up
|
file blocks nothing, because the lock ends with the process holding it however
|
||||||
the database as after. A second `scan` against the same database does not
|
it ends, a fatal error or an interrupt included, and deleting the file while a
|
||||||
wait: it fails at once with a one-line error naming the lock file and exits 1,
|
scan runs would let a second scan start. `report` and `trees` never take the
|
||||||
without walking anything or opening the database, and the running scan carries
|
lock, so they run during a scan.
|
||||||
on. The lock file is created on first use, open to its owner only, and left in
|
|
||||||
place: a leftover file blocks nothing, because the lock ends with the process
|
|
||||||
holding it however it ends, a fatal error or an interrupt included, and
|
|
||||||
deleting the file while a scan runs would let a second scan start. `report`
|
|
||||||
and `trees` never take the lock, so they run during a scan.
|
|
||||||
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so
|
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so
|
||||||
running a report while a cron `scan` is in progress is safe. The filesystem is
|
running a report while a cron `scan` is in progress is safe. The filesystem is
|
||||||
authoritative; the database is an eventually-consistent reflection of it.
|
authoritative; the database is an eventually-consistent reflection of it.
|
||||||
@@ -398,11 +393,7 @@ results do not depend on the working directory. All operands belong to a single
|
|||||||
scan and are enumerated concurrently: every operand seeds the shared walk worker
|
scan and are enumerated concurrently: every operand seeds the shared walk worker
|
||||||
pool. Overlapping operands are harmless — an operand that duplicates another or
|
pool. Overlapping operands are harmless — an operand that duplicates another or
|
||||||
lies under another is dropped before walking, so every file is reached exactly
|
lies under another is dropped before walking, so every file is reached exactly
|
||||||
once and produces one database record. With `-x`, though, an operand under
|
once and produces one database record.
|
||||||
another one is kept and walked on its own when any directory from it (from its
|
|
||||||
directory, for a regular file) up to but not including the operand covering it
|
|
||||||
is on a different filesystem than that operand, because that operand's walk does
|
|
||||||
not enter such a directory and so does not reach it.
|
|
||||||
|
|
||||||
An operand that is a symlink (never followed, not even as an operand), socket,
|
An operand that is a symlink (never followed, not even as an operand), socket,
|
||||||
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
|
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
|
||||||
@@ -411,8 +402,9 @@ drops it from the scanned operands before reading the database. Another operand
|
|||||||
beneath it is still scanned. The records stored beneath it are not deleted: they
|
beneath it is still scanned. The records stored beneath it are not deleted: they
|
||||||
are treated like any other record outside the scanned operands, including the
|
are treated like any other record outside the scanned operands, including the
|
||||||
content-phase exception below. If it lies under another operand, they are under
|
content-phase exception below. If it lies under another operand, they are under
|
||||||
that operand instead, and are treated like any other record there. This is not
|
that operand instead, and are deleted like any other record there that this scan
|
||||||
an error: a scan whose every operand is dropped walks nothing and exits 0.
|
did not verify. This is not an error: a scan whose every operand is dropped
|
||||||
|
walks nothing and exits 0.
|
||||||
|
|
||||||
`scan` synchronizes the database with the filesystem state under the scanned
|
`scan` synchronizes the database with the filesystem state under the scanned
|
||||||
operands:
|
operands:
|
||||||
@@ -450,12 +442,6 @@ operands:
|
|||||||
a `content` hash is read for one, wherever it lies, once its size, `head`, and
|
a `content` hash is read for one, wherever it lies, once its size, `head`, and
|
||||||
`tail` match another record's. If that file is gone or has changed since its
|
`tail` match another record's. If that file is gone or has changed since its
|
||||||
record was written, the record is left as it is.
|
record was written, the record is left as it is.
|
||||||
- With `-x`, a record below a directory the walk did not enter because that
|
|
||||||
directory is on another filesystem than its operand counts as outside the
|
|
||||||
scanned operands: it is not deleted, and its size joins the size census. A
|
|
||||||
filesystem mounted under an operand can therefore be scanned on its own
|
|
||||||
schedule into the same database. Records below an operand at or inside such a
|
|
||||||
directory are not kept this way, since that operand's own walk verifies them.
|
|
||||||
|
|
||||||
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
|
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
|
||||||
inside each phase; batched database writes begin during the hash phase:
|
inside each phase; batched database writes begin during the hash phase:
|
||||||
@@ -522,19 +508,15 @@ Rules for the walk:
|
|||||||
- Filesystem boundaries are crossed by default. With `-x` (long form
|
- Filesystem boundaries are crossed by default. With `-x` (long form
|
||||||
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
||||||
into a directory on a different filesystem than its `PATH` operand; each
|
into a directory on a different filesystem than its `PATH` operand; each
|
||||||
operand is bounded by its own filesystem. The records below such a directory,
|
operand is bounded by its own filesystem.
|
||||||
unless they lie below an operand at or inside that directory, count as outside
|
|
||||||
the scanned operands, as described in "`scan` mode" above: they are not
|
|
||||||
deleted, and their sizes join the size census.
|
|
||||||
- On any per-path error (permission denied, file vanished between passes,
|
- On any per-path error (permission denied, file vanished between passes,
|
||||||
unreadable): print a one-line warning to stderr, skip the path, and continue.
|
unreadable): print a one-line warning to stderr, skip the path, and continue.
|
||||||
Per-file errors never abort the run; the final summary reports how many were
|
Per-file errors never abort the run; the final summary reports how many were
|
||||||
skipped. As specified above, a skipped path that has a database record from an
|
skipped. As specified above, a skipped path that has a database record from an
|
||||||
earlier scan loses that record, unless it failed only in the content phase, or
|
earlier scan loses that record, unless it failed only in the content phase, or
|
||||||
is an operand dropped before the database was read whose record counts as
|
is an operand dropped before the database was read that lies under no other
|
||||||
outside the scanned operands; an unreadable directory subtree likewise loses
|
operand; an unreadable directory subtree likewise loses its records (accepted:
|
||||||
its records (accepted: the database mirrors what the latest scan could
|
the database mirrors what the latest scan could actually verify).
|
||||||
actually verify).
|
|
||||||
|
|
||||||
Concurrency: the walk phase (which also stats files), the hash phase, and the
|
Concurrency: the walk phase (which also stats files), the hash phase, and the
|
||||||
content phase each use a worker pool of `--workers` workers (default
|
content phase each use a worker pool of `--workers` workers (default
|
||||||
@@ -764,16 +746,14 @@ entrypoints are:
|
|||||||
- `script/bootstrap` — install everything needed to build and develop this
|
- `script/bootstrap` — install everything needed to build and develop this
|
||||||
repository, idempotently, assuming nothing is present. `git`, `make`, and `go`
|
repository, idempotently, assuming nothing is present. `git`, `make`, and `go`
|
||||||
come from the first of nix, apt, brew, or apk found on the host, and are
|
come from the first of nix, apt, brew, or apk found on the host, and are
|
||||||
presence-checked only. An installed node is used when its major version is 22;
|
presence-checked only. An installed node is used as it is; otherwise node
|
||||||
otherwise node 22.17.0 is installed through nvm, which comes from a release
|
22.17.0 is installed through nvm, which comes from a release archive whose
|
||||||
archive whose sha256 the script checks, and yarn 1.22.22 is activated under it
|
sha256 the script checks. A yarn already on `PATH` is used as it is; otherwise
|
||||||
through corepack. With an installed node 22, a yarn already on `PATH` is used
|
yarn 1.22.22 is activated through corepack, or installed with `npm` when there
|
||||||
as it is; otherwise yarn 1.22.22 is activated through corepack, or installed
|
is no corepack. `yarn install --frozen-lockfile` then installs the prettier
|
||||||
with `npm` when there is no corepack. `yarn install --frozen-lockfile` then
|
that `package.json` and `yarn.lock` pin. `golangci-lint` is never installed:
|
||||||
installs the prettier that `package.json` and `yarn.lock` pin. `golangci-lint`
|
it runs in Docker (see `script/lint`). `docker` is not installed either;
|
||||||
is never installed: it runs in Docker (see `script/lint`). `docker` is not
|
testing, linting and the image build need it. Ends with `go mod download`.
|
||||||
installed either; testing, linting and the image build need it. Ends with
|
|
||||||
`go mod download`.
|
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
- `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`.
|
`script/bootstrap`, then `script/install-precommit`.
|
||||||
- `script/projectname` — print this project's name (`sfdupes`). Scripts that
|
- `script/projectname` — print this project's name (`sfdupes`). Scripts that
|
||||||
@@ -794,8 +774,8 @@ entrypoints are:
|
|||||||
- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every
|
- `script/fmt` — format in place: the Go sources with `gofmt -s -w`, and every
|
||||||
Markdown file with prettier, at the settings in `.prettierrc` (4-space
|
Markdown file with prettier, at the settings in `.prettierrc` (4-space
|
||||||
indents, prose wrapped at 80 columns). Both run on the host, prettier through
|
indents, prose wrapped at 80 columns). Both run on the host, prettier through
|
||||||
yarn at the version `yarn.lock` pins. When the node on `PATH` is not node 22,
|
yarn at the version `yarn.lock` pins. When yarn is not on `PATH`, this loads
|
||||||
this loads the node 22.17.0 that `script/bootstrap` installed through nvm.
|
the node 22.17.0 that `script/bootstrap` installed through nvm.
|
||||||
- `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any
|
- `script/fmt-check` — the read-only counterpart of `script/fmt`: prints any
|
||||||
unformatted file and exits non-zero instead of writing. gofmt and prettier
|
unformatted file and exits non-zero instead of writing. gofmt and prettier
|
||||||
both run every time, and each names itself when it fails. The `Dockerfile`'s
|
both run every time, and each names itself when it fails. The `Dockerfile`'s
|
||||||
@@ -818,13 +798,10 @@ entrypoints are:
|
|||||||
- `script/install-precommit` — install the git pre-commit hook, as
|
- `script/install-precommit` — install the git pre-commit hook, as
|
||||||
`.git/hooks/pre-commit`, that runs `script/precommit`.
|
`.git/hooks/pre-commit`, that runs `script/precommit`.
|
||||||
|
|
||||||
Every `docker build` in `script/` passes a new `CACHEBUST` build argument, which
|
Every `docker build` in `script/` passes `--no-cache`. On an unchanged tree
|
||||||
every stage of the `Dockerfile` declares first, so every `RUN` step runs again
|
Docker would serve the gate steps from its cache, and the build would pass
|
||||||
each time. On an unchanged tree Docker would otherwise serve the gate steps from
|
having run no test and no linter. Every run therefore downloads the Go modules
|
||||||
its cache, and the build would pass having run no test and no linter. Go's build
|
again and needs the network.
|
||||||
cache is kept between builds on the steps that compile, so only what changed is
|
|
||||||
compiled again; the tests run with `-count=1`, so none reports a stored result.
|
|
||||||
Every run downloads the Go modules again and needs the network.
|
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
|
|||||||
+73
-117
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-10-15
|
last_modified: 2026-10-07
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -54,39 +54,34 @@ style conventions are in separate documents:
|
|||||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
||||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
||||||
(detected in that order; apt runs noninteractive). For node it uses the
|
(detected in that order; apt runs noninteractive). For node it uses the
|
||||||
installed node only when its major version is the pinned one; otherwise (node
|
installed node if present; otherwise it installs a PINNED node version via
|
||||||
missing, or another major, such as the node 24 the CI runner image ships) it
|
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
||||||
installs the PINNED node version via nvm and installs yarn and the packages
|
release archive (never `curl | sh`), with bash installed as an explicit
|
||||||
under it. nvm itself is installed first if missing, from a hash-verified
|
prerequisite since nvm requires bash. yarn is then pinned via
|
||||||
GitHub release archive (never `curl | sh`), with bash installed as an explicit
|
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||||
prerequisite since nvm requires bash. Only the major version is compared
|
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||||
because the `Dockerfile` stages start from a node image whose exact version is
|
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
||||||
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
|
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||||
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
|
|
||||||
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
|
|
||||||
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
|
|
||||||
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
|
||||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
||||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
||||||
pristine checkout with nothing installed the run dies there, after the
|
pristine checkout with nothing installed the run dies there, after the
|
||||||
containerised gates have passed. **The bootstrap alone is not enough**: when
|
containerised gates have passed. **The bootstrap alone is not enough**:
|
||||||
`script/bootstrap` installs node and yarn under nvm it leaves neither on the
|
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
||||||
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
|
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
||||||
under another node. The host entrypoints that need yarn — `script/fmt` and
|
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
||||||
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when
|
source nvm for the pinned node version before invoking it, exactly as
|
||||||
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
|
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
||||||
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
|
docker and git then gets through `script/check`. Four further scripts are our
|
||||||
pinned node. A runner carrying nothing but docker and git then gets through
|
own extensions to the standard: `script/check` runs `script/test`,
|
||||||
`script/check`. Four further scripts are our own extensions to the standard:
|
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
||||||
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
|
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
||||||
`script/precommit` is what the git pre-commit hook runs, and it calls
|
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
||||||
`script/check`; `script/install-precommit` installs the git pre-commit hook
|
`script/projectname` (literally that filename) simply outputs the project's
|
||||||
(the `make hooks` target shims to it); and `script/projectname` (literally
|
name. Scripts that need the name call `script/projectname` — e.g.
|
||||||
that filename) simply outputs the project's name. Scripts that need the name
|
`script/docker` assembles its image tag from it — so those scripts stay
|
||||||
call `script/projectname` — e.g. `script/docker` assembles its image tag from
|
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
||||||
it — so those scripts stay byte-identical across all repos. Repo-type-specific
|
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
||||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
the hook itself. Model scripts are at
|
||||||
`script/precommit`, not in the hook itself. Model scripts are at
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||||
must document the provided scripts in an **Entrypoints** section (see the
|
must document the provided scripts in an **Entrypoints** section (see the
|
||||||
README requirements below).
|
README requirements below).
|
||||||
@@ -123,10 +118,8 @@ style conventions are in separate documents:
|
|||||||
and nothing else:
|
and nothing else:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
docker build --no-cache --target lint --output type=cacheonly .
|
||||||
--target lint --output type=cacheonly .
|
docker build --no-cache --target test --output type=cacheonly .
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
|
||||||
--target test --output type=cacheonly .
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**A stage that is not the last one in the file is built only when the final
|
**A stage that is not the last one in the file is built only when the final
|
||||||
@@ -139,12 +132,12 @@ style conventions are in separate documents:
|
|||||||
**The gate builds write no image.** With `--output type=cacheonly` the phase
|
**The gate builds write no image.** With `--output type=cacheonly` the phase
|
||||||
runs and a failing step fails the build, but the result is not exported.
|
runs and a failing step fails the build, but the result is not exported.
|
||||||
Nothing uses those images, and writing one out is slow: a Go test phase's
|
Nothing uses those images, and writing one out is slow: a Go test phase's
|
||||||
image holds the toolchain. A build given neither `--output` nor `-t` writes
|
image holds the toolchain and every compiled package. A build given neither
|
||||||
an untagged image and leaves it dangling, on every developer host and every
|
`--output` nor `-t` writes an untagged image and leaves it dangling, on
|
||||||
CI runner. `script/cibuild` and `script/docker` build the image that ships
|
every developer host and every CI runner. `script/cibuild` and
|
||||||
and tag it, so each build replaces the previous image; each assigns the tag
|
`script/docker` build the image that ships and tag it, so each build
|
||||||
on its own line before the build, so `set -e` stops it where
|
replaces the previous image; each assigns the tag on its own line before the
|
||||||
`script/projectname` fails.
|
build, so `set -e` stops it where `script/projectname` fails.
|
||||||
|
|
||||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
||||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
||||||
@@ -159,33 +152,22 @@ style conventions are in separate documents:
|
|||||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||||
org, in both directions. A container has its own golangci-lint cache, its
|
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
||||||
own `TMPDIR` and a digest-pinned binary, so neither is reachable.
|
a digest-pinned binary, so neither is reachable.
|
||||||
|
|
||||||
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
||||||
invalidates a `COPY` layer only when the copied content changes, so on an
|
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
||||||
unchanged tree the check `RUN` is served from cache, nothing executes, and the
|
the check `RUN` is served from cache, nothing executes, and the build still
|
||||||
build still exits 0. Every `docker build` in `script/` therefore passes
|
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
||||||
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
|
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
||||||
id, so no two builds share a value, not even the three in one
|
four, and there is no fifth — `script/check` runs the two gate phases and
|
||||||
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
||||||
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
not evidence that anything ran: a sub-second build reporting success is a
|
||||||
two gate phases and `script/fmt-check`, and builds no image of its own. Every
|
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
||||||
stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after
|
and friends destroy a build cache shared with every other build on the host.
|
||||||
`FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps
|
When a check is added or changed, prove it works by planting a defect it must
|
||||||
are compared by content and need nothing. A stage without the `ARG` line is
|
catch and watching the run fail on it, then revert the defect. A green run
|
||||||
served from cache on an unchanged tree, so a repository re-vendors the scripts
|
alone shows neither that the check ran nor that it covers what it should.
|
||||||
and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`:
|
|
||||||
it reruns every step too, but it also empties every cache mount, and Go's
|
|
||||||
build cache, which the canonical Go `Dockerfile` below mounts so that compiled
|
|
||||||
packages survive between check builds, would start empty every time. A bare
|
|
||||||
`docker build .` is not evidence that anything ran: a sub-second build
|
|
||||||
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
|
||||||
`docker builder prune` and friends destroy a build cache shared with every
|
|
||||||
other build on the host. When a check is added or changed, prove it works by
|
|
||||||
planting a defect it must catch and watching the run fail on it, then revert
|
|
||||||
the defect. A green run alone shows neither that the check ran nor that it
|
|
||||||
covers what it should.
|
|
||||||
|
|
||||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||||
@@ -194,42 +176,32 @@ style conventions are in separate documents:
|
|||||||
`Dockerfile`:
|
`Dockerfile`:
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
# Every stage declares ARG CACHEBUST first: script/ passes a new value
|
|
||||||
# on every build, which reruns every RUN step after it. The steps that
|
|
||||||
# compile mount Go's build cache, which survives from build to build.
|
|
||||||
|
|
||||||
# Lint phase
|
# Lint phase
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
FROM golangci/golangci-lint@sha256:... AS lint
|
||||||
ARG CACHEBUST
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN golangci-lint run --config .golangci.yml ./...
|
||||||
golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
|
||||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||||
# image ships and the alpine one does not. -count=1 because the mounted
|
# image ships and the alpine one does not.
|
||||||
# cache also holds test results, which go test would otherwise replay.
|
|
||||||
# golang:1.x, YYYY-MM-DD
|
# golang:1.x, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS test
|
FROM golang@sha256:... AS test
|
||||||
ARG CACHEBUST
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN go test -timeout 90s -race -cover ./... || \
|
||||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
# Build stage. Nothing is wanted from either phase above; the copies
|
||||||
# are what make BuildKit build them first, so this stage cannot run
|
# are what make BuildKit build them first, so this stage cannot run
|
||||||
# unless lint and test passed.
|
# unless lint and test passed.
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
# golang:1.x-alpine, YYYY-MM-DD
|
||||||
FROM golang@sha256:... AS builder
|
FROM golang@sha256:... AS builder
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=test /src/go.sum /dev/null
|
COPY --from=test /src/go.sum /dev/null
|
||||||
RUN apk add --no-cache git
|
RUN apk add --no-cache git
|
||||||
@@ -245,8 +217,7 @@ style conventions are in separate documents:
|
|||||||
# .git present, a version that is still empty, dev or unknown fails the
|
# .git present, a version that is still empty, dev or unknown fails the
|
||||||
# build: git is missing or could not read the checkout.
|
# build: git is missing or could not read the checkout.
|
||||||
ARG VERSION
|
ARG VERSION
|
||||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||||
VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
|
||||||
if [ -e .git ]; then \
|
if [ -e .git ]; then \
|
||||||
case "$VERSION" in ""|dev|unknown) \
|
case "$VERSION" in ""|dev|unknown) \
|
||||||
echo "version is '$VERSION' although .git is present" >&2; \
|
echo "version is '$VERSION' although .git is present" >&2; \
|
||||||
@@ -259,7 +230,6 @@ style conventions are in separate documents:
|
|||||||
|
|
||||||
# Runtime stage, and the last one
|
# Runtime stage, and the last one
|
||||||
FROM alpine@sha256:...
|
FROM alpine@sha256:...
|
||||||
ARG CACHEBUST
|
|
||||||
COPY --from=builder /app /usr/local/bin/app
|
COPY --from=builder /app /usr/local/bin/app
|
||||||
ENTRYPOINT ["app"]
|
ENTRYPOINT ["app"]
|
||||||
```
|
```
|
||||||
@@ -274,16 +244,6 @@ style conventions are in separate documents:
|
|||||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||||
same two copies. A plain `docker build .` builds the last stage's chain
|
same two copies. A plain `docker build .` builds the last stage's chain
|
||||||
and nothing else.
|
and nothing else.
|
||||||
- Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps
|
|
||||||
it for root), and only on the steps that compile: `golangci-lint run`,
|
|
||||||
both `go test` runs and `go build`. It keeps compiled packages between
|
|
||||||
check builds, and Go reuses one only when all its inputs are unchanged, so
|
|
||||||
it saves the compile without skipping a check. Nothing else is kept:
|
|
||||||
golangci-lint's own cache would replay stored findings in place of a lint
|
|
||||||
run, and Go builds from whatever files a kept module cache holds without
|
|
||||||
checking them again (only `go mod verify` does), so a file changed there
|
|
||||||
would go unnoticed. Go's build cache also holds test results, which is why
|
|
||||||
both `go test` runs carry `-count=1`.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
- If the project uses `//go:embed` directives that reference build artifacts
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||||
create placeholder files so the embed directives resolve. Example:
|
create placeholder files so the embed directives resolve. Example:
|
||||||
@@ -344,18 +304,18 @@ style conventions are in separate documents:
|
|||||||
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
|
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
|
||||||
so a successful run means every check passed; a bare `docker build .` does not
|
so a successful run means every check passed; a bare `docker build .` does not
|
||||||
carry the same guarantee, because its gate phases may come from the cache. The
|
carry the same guarantee, because its gate phases may come from the cache. The
|
||||||
image build gets a `CACHEBUST` value of its own and so runs the gate phases a
|
image build is uncached and so runs the gate phases a second time. That is the
|
||||||
second time. That is the price of the rule above, and it is worth paying: the
|
price of the rule above, and it is worth paying: the image that ships is built
|
||||||
image that ships is built from a run of its own gates rather than from a cache
|
from a run of its own gates rather than from a cache entry. The `check` job
|
||||||
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
|
sets `timeout-minutes: 20`, so a hung build frees the shared runner after 20
|
||||||
shared runner after 20 minutes. That allows for the three Docker builds
|
minutes. That allows for the three Docker builds described above (the test
|
||||||
described above (the test phase, the lint phase, then the image), each held to
|
phase, the lint phase, then the image), each held to the 5-minute Docker build
|
||||||
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
|
limit below, plus the bootstrap. A separate workflow limited to `main` by a
|
||||||
limited to `main` by a `branches` list under `on: push` cannot be checked by
|
`branches` list under `on: push` cannot be checked by review: to try a change
|
||||||
review: to try a change to it, add the feature branch to that list and push,
|
to it, add the feature branch to that list and push, then remove the branch
|
||||||
then remove the branch from the list again before merging. Keep any job in it
|
from the list again before merging. Keep any job in it that publishes behind
|
||||||
that publishes behind `if: github.ref_name == 'main'`, so the run from the
|
`if: github.ref_name == 'main'`, so the run from the feature branch publishes
|
||||||
feature branch publishes nothing.
|
nothing.
|
||||||
|
|
||||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||||
@@ -415,11 +375,12 @@ style conventions are in separate documents:
|
|||||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
||||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
passing result in its cache directory (`GOCACHE`), and when the same tests
|
||||||
run again on unchanged code it prints that result, marked `(cached)`,
|
run again on unchanged code it prints that result, marked `(cached)`,
|
||||||
without running them. The directory lasts from one run to the next on a
|
without running them. That matters on a developer's machine, where this
|
||||||
developer's machine, where this target runs, and in the `test` phase of the
|
target runs and the directory lasts from one run to the next. The `test`
|
||||||
`Dockerfile`, which mounts it to keep compiled packages between check
|
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
||||||
builds, so the phase's `go test` runs carry `-count=1` too. A new
|
result for this repo's tests and nothing before its `go test` step runs a
|
||||||
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
|
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
||||||
|
step run on an unchanged tree.
|
||||||
|
|
||||||
Python example:
|
Python example:
|
||||||
|
|
||||||
@@ -506,7 +467,7 @@ style conventions are in separate documents:
|
|||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||||
[ -n "$version" ] || version="unknown"
|
[ -n "$version" ] || version="unknown"
|
||||||
tag="$(script/projectname)"
|
tag="$(script/projectname)"
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
docker build --no-cache \
|
||||||
--build-arg VERSION="$version" \
|
--build-arg VERSION="$version" \
|
||||||
-t "$tag" .
|
-t "$tag" .
|
||||||
```
|
```
|
||||||
@@ -588,11 +549,6 @@ style conventions are in separate documents:
|
|||||||
function defined and never invoked has the same exit status and the same
|
function defined and never invoked has the same exit status and the same
|
||||||
empty output as one that worked.
|
empty output as one that worked.
|
||||||
|
|
||||||
Node is the exception to the whole-token comparison: the canonical
|
|
||||||
`script/bootstrap` compares only its major version, because the node 22
|
|
||||||
alpine image the `Dockerfile` stages start from is not the exact pin, and
|
|
||||||
nvm cannot install a prebuilt node on alpine.
|
|
||||||
|
|
||||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||||
|
|
||||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
A Go tool a repo needs on the host is installed with `go install` pinned to
|
||||||
|
|||||||
@@ -30,43 +30,15 @@
|
|||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
|
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
|
||||||
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
`c55a0cb`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
||||||
that write no image, and `make test` runs the suite under the race detector,
|
that write no image, and `make test` runs the suite under the race detector,
|
||||||
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
|
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
|
||||||
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
|
gone; every `docker build` in `script/` passes `--no-cache`; prettier runs on
|
||||||
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
|
the host, from the node and yarn `script/bootstrap` installs, so the
|
||||||
every build, while the steps that compile keep Go's build cache and the tests
|
`prettier` and `markdown` stages are gone and the build stage installs `git`
|
||||||
run with `-count=1`; prettier runs on the host, from the node and yarn
|
and `make` itself; a new push cancels the workflow's older run on the same
|
||||||
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
|
branch, and a run stops after 20 minutes; `.claude/settings.json` is deleted
|
||||||
and the build stage installs `git` and `make` itself; a new push cancels the
|
(2026-10-08, https://git.eeqj.de/sneak/sfdupes/issues/95)
|
||||||
workflow's older run on the same branch, and a run stops after 20 minutes;
|
|
||||||
`.claude/settings.json` is deleted (2026-10-15,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95)
|
|
||||||
|
|
||||||
- with `-x`, an operand under another operand is walked on its own instead of
|
|
||||||
being dropped when a directory from it (from its directory, for a regular
|
|
||||||
file) up to but not including that operand is on a different filesystem than
|
|
||||||
that operand, whose walk then cannot reach it, and the records of files gone
|
|
||||||
from below it are deleted (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
|
||||||
|
|
||||||
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
|
||||||
under its operand, and counts their sizes in the size census, instead of
|
|
||||||
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
|
||||||
|
|
||||||
- the hash and content phases open each file without blocking and check that it
|
|
||||||
is still a regular file, so one replaced by a FIFO or device node after it was
|
|
||||||
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
|
||||||
|
|
||||||
- the scan lock file is named from the database's path with symlinks resolved,
|
|
||||||
after `scan` creates the database file if it is missing, so a scan through a
|
|
||||||
symlink to the database fails while another scan holds the lock (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/101)
|
|
||||||
|
|
||||||
- on a terminal, a warning for the last file of a phase no longer erases the
|
|
||||||
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
|
||||||
|
|
||||||
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
||||||
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
||||||
@@ -90,7 +62,7 @@
|
|||||||
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
|
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
||||||
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
|
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
|
||||||
`script/` pass a new `CACHEBUST` value, so theirs always run
|
`script/` pass `--no-cache`, so theirs always run
|
||||||
|
|
||||||
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
||||||
it; all Markdown reformatted (2026-10-04,
|
it; all Markdown reformatted (2026-10-04,
|
||||||
@@ -241,8 +213,7 @@
|
|||||||
served them from cache and the build exited 0 having run nothing. The fix was
|
served them from cache and the build exited 0 having run nothing. The fix was
|
||||||
a `CHECK_EPOCH` build argument; since
|
a `CHECK_EPOCH` build argument; since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
||||||
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
passes `--no-cache` instead. Run as root, the tests fail
|
||||||
declares first. Run as root, the tests fail
|
|
||||||
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
|
`TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
|
||||||
the test relies on, so the `test` phase runs them as `nobody`
|
the test relies on, so the `test` phase runs them as `nobody`
|
||||||
- check the installed golangci-lint version in `script/bootstrap` instead of
|
- check the installed golangci-lint version in `script/bootstrap` instead of
|
||||||
|
|||||||
@@ -35,10 +35,6 @@ const schemaVersion = 1
|
|||||||
// scan.
|
// scan.
|
||||||
const dbDirPerm = 0o755
|
const dbDirPerm = 0o755
|
||||||
|
|
||||||
// dbFilePerm is the mode for a database file created by scan, the mode
|
|
||||||
// SQLite gives a database file it creates.
|
|
||||||
const dbFilePerm = 0o644
|
|
||||||
|
|
||||||
// lockFilePerm is the mode for the scan lock file. Anyone who can open
|
// lockFilePerm is the mode for the scan lock file. Anyone who can open
|
||||||
// the file can hold the lock and keep every scan from running, so it
|
// the file can hold the lock and keep every scan from running, so it
|
||||||
// is open to its owner only.
|
// is open to its owner only.
|
||||||
@@ -144,37 +140,19 @@ func openDB(path, params string) (*sql.DB, error) {
|
|||||||
|
|
||||||
// lockScanDatabase takes the lock that keeps a second scan off the
|
// lockScanDatabase takes the lock that keeps a second scan off the
|
||||||
// database at path: an exclusive flock(2) on the file beside it named
|
// database at path: an exclusive flock(2) on the file beside it named
|
||||||
// by the database's path with symlinks resolved and ".lock" appended,
|
// path with ".lock" appended, created along with the database's parent
|
||||||
// so every path to one database names the same lock file. If missing,
|
// directory if missing. A lock held by another scan fails at once
|
||||||
// the database's parent directory and an empty database file are
|
// instead of waiting. The lock lasts until the returned file is closed
|
||||||
// created first, so the lock file is named the same way before the
|
// or the process ends. The file is never deleted: a scan that deleted
|
||||||
// database is set up as after. A lock held by another scan fails at
|
// it would let the next scan lock a new file while another still holds
|
||||||
// once instead of waiting. The lock lasts until the returned file is
|
// the old one.
|
||||||
// closed or the process ends. The lock file is never deleted: a scan
|
|
||||||
// that deleted it would let the next scan lock a new file while
|
|
||||||
// another still holds the old one.
|
|
||||||
func lockScanDatabase(path string) (*os.File, error) {
|
func lockScanDatabase(path string) (*os.File, error) {
|
||||||
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create database directory: %w", err)
|
return nil, fmt.Errorf("create database directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Like SQLite, this follows a symlink at path and creates its
|
lockPath := path + ".lock"
|
||||||
// target. SQLite opens an empty file as a new database.
|
|
||||||
//nolint:gosec // the operator chooses the database path
|
|
||||||
dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("create database: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = dbFile.Close()
|
|
||||||
|
|
||||||
resolved, err := filepath.EvalSymlinks(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("resolve database path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
lockPath := resolved + ".lock"
|
|
||||||
|
|
||||||
//nolint:gosec // the operator chooses the database path
|
//nolint:gosec // the operator chooses the database path
|
||||||
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm)
|
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm)
|
||||||
|
|||||||
+2
-10
@@ -13,19 +13,11 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// testDBPath returns a database path inside a fresh temp dir, with
|
// testDBPath returns a database path inside a fresh temp dir.
|
||||||
// symlinks resolved as the scan lock resolves them, so the lock file
|
|
||||||
// is this path plus ".lock" even where the temp dir is reached through
|
|
||||||
// a symlink.
|
|
||||||
func testDBPath(t *testing.T) string {
|
func testDBPath(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
return filepath.Join(t.TempDir(), "db.sqlite")
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return filepath.Join(dir, "db.sqlite")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// openTestDB creates a fresh scan database in a temp dir.
|
// openTestDB creates a fresh scan database in a temp dir.
|
||||||
|
|||||||
+4
-93
@@ -822,7 +822,7 @@ func holdScanLock(t *testing.T, path string) {
|
|||||||
|
|
||||||
func TestRunSecondScanFails(t *testing.T) {
|
func TestRunSecondScanFails(t *testing.T) {
|
||||||
// README §Database: while one scan holds the lock, a second scan
|
// README §Database: while one scan holds the lock, a second scan
|
||||||
// fails at once, naming the lock file, without opening the
|
// fails at once, naming the lock file, without creating the
|
||||||
// database.
|
// database.
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
@@ -848,99 +848,10 @@ func TestRunSecondScanFails(t *testing.T) {
|
|||||||
t.Errorf("stdout = %q, want nothing (data only)", got)
|
t.Errorf("stdout = %q, want nothing (data only)", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Taking the lock created the database file, empty; opening it as
|
_, err := os.Stat(path)
|
||||||
// a database would have written to it.
|
if !errors.Is(err, fs.ErrNotExist) {
|
||||||
info, err := os.Stat(path)
|
t.Errorf("stat %s = %v, want the database not created", path, err)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.Size() != 0 {
|
|
||||||
t.Errorf("database size = %d, want 0 (not opened)", info.Size())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertSecondScanFails runs a scan against the database
|
|
||||||
// SFDUPES_DATABASE names while the test holds the lock on it. It fails
|
|
||||||
// the test unless that scan fails at once, naming the lock file beside
|
|
||||||
// path, the database's real path.
|
|
||||||
func assertSecondScanFails(t *testing.T, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
|
|
||||||
if code != exitFatal {
|
|
||||||
t.Errorf("run(scan) = %d, want %d", code, exitFatal)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := "sfdupes: another scan is running (lock held on " +
|
|
||||||
path + ".lock)\n"
|
|
||||||
if got := stderr.String(); got != want {
|
|
||||||
t.Errorf("stderr = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
|
||||||
// README §Database: every path to one database names the same lock
|
|
||||||
// file, so while a scan holds it through one path, a scan through
|
|
||||||
// another fails, naming the lock file beside the database's real
|
|
||||||
// path.
|
|
||||||
t.Run("symlink to the database", func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
scanFixture(t)
|
|
||||||
|
|
||||||
link := filepath.Join(t.TempDir(), "link.sqlite")
|
|
||||||
|
|
||||||
err := os.Symlink(path, link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
holdScanLock(t, path)
|
|
||||||
t.Setenv(databaseEnv, link)
|
|
||||||
assertSecondScanFails(t, path)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("symlink to its directory, no database yet", func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
link := filepath.Join(t.TempDir(), "link")
|
|
||||||
|
|
||||||
err := os.Symlink(filepath.Dir(path), link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
holdScanLock(t, path)
|
|
||||||
t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
|
|
||||||
assertSecondScanFails(t, path)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("symlink whose target does not exist yet", func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
link := filepath.Join(t.TempDir(), "link.sqlite")
|
|
||||||
|
|
||||||
err := os.Symlink(path, link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The first scan takes the lock through the link before the
|
|
||||||
// database exists, then creates the database through it.
|
|
||||||
holdScanLock(t, link)
|
|
||||||
|
|
||||||
db, err := openScanDatabase(t.Context(), link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
closeScanDatabase(t.Context(), db, link)
|
|
||||||
|
|
||||||
t.Setenv(databaseEnv, link)
|
|
||||||
assertSecondScanFails(t, path)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanReleasesLock(t *testing.T) {
|
func TestRunScanReleasesLock(t *testing.T) {
|
||||||
|
|||||||
@@ -139,13 +139,6 @@ func (p *progress) warnf(format string, args ...any) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fmt.Fprintln(os.Stderr, msg)
|
fmt.Fprintln(os.Stderr, msg)
|
||||||
|
|
||||||
// The library never draws a finished bar again, so the warning for
|
|
||||||
// a pass's last item would erase it for good: draw its last frame
|
|
||||||
// again below the warning.
|
|
||||||
if p.bar != nil && p.bar.IsFinished() {
|
|
||||||
fmt.Fprint(os.Stderr, p.bar.String())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// finish terminates the pass's display. A bar whose pass stopped short
|
// finish terminates the pass's display. A bar whose pass stopped short
|
||||||
|
|||||||
@@ -187,25 +187,3 @@ func TestBarStoppedShortKeepsCount(t *testing.T) {
|
|||||||
t.Errorf("terminal shows %q, want a count of 2 of 10", shown)
|
t.Errorf("terminal shows %q, want a count of 2 of 10", shown)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWarningAfterLastItemKeepsBar checks that a warning printed after
|
|
||||||
// a pass's last item, as for a failed read of the last file, leaves the
|
|
||||||
// pass's finished bar on the terminal below the warning.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestWarningAfterLastItemKeepsBar(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
p := &progress{
|
|
||||||
label: "hash", total: 2, start: time.Now(),
|
|
||||||
bar: newBar("hash", 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
p.increment()
|
|
||||||
p.increment()
|
|
||||||
p.warnf("warning")
|
|
||||||
p.finish()
|
|
||||||
|
|
||||||
if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/2,") {
|
|
||||||
t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -203,16 +203,10 @@ func resolveRoots(roots []string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// pruneRoots drops operands already covered by another operand:
|
// pruneRoots drops operands already covered by another operand:
|
||||||
// duplicates and any operand lying under another one. With -x, an
|
// duplicates and any operand lying under another one. Every remaining
|
||||||
// operand under another one is kept when the walk of the deepest kept
|
// file is then reachable through exactly one root, so walked paths are
|
||||||
// operand above it cannot reach it: when device puts any directory from
|
// unique without keeping a scan-wide set of every path seen.
|
||||||
// it (from its directory, for a regular file) up to but not including
|
func pruneRoots(roots []string) []string {
|
||||||
// that operand on a different filesystem than that operand. Every
|
|
||||||
// remaining file is then reachable through exactly one root, so walked
|
|
||||||
// paths are unique without keeping a scan-wide set of every path seen.
|
|
||||||
func pruneRoots(roots []string, oneFS bool,
|
|
||||||
device func(path string) (uint64, bool),
|
|
||||||
) []string {
|
|
||||||
// Shorter-first ordering guarantees an ancestor is kept before any
|
// Shorter-first ordering guarantees an ancestor is kept before any
|
||||||
// operand under it is considered.
|
// operand under it is considered.
|
||||||
sorted := slices.Clone(roots)
|
sorted := slices.Clone(roots)
|
||||||
@@ -227,58 +221,14 @@ func pruneRoots(roots []string, oneFS bool,
|
|||||||
kept := make([]string, 0, len(sorted))
|
kept := make([]string, 0, len(sorted))
|
||||||
|
|
||||||
for _, r := range sorted {
|
for _, r := range sorted {
|
||||||
above := deepestRoot(r, kept)
|
if !underAnyRoot(r, kept) {
|
||||||
if above == "" {
|
|
||||||
kept = append(kept, r)
|
kept = append(kept, r)
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Without -x, and always for a duplicate, r is dropped.
|
|
||||||
if !oneFS || above == r {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
aboveDev, aboveOK := device(above)
|
|
||||||
if !aboveOK {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// The walk of above enters a directory only when it is on
|
|
||||||
// above's filesystem, and then reaches every regular file in it
|
|
||||||
// whatever device the file reports, so for a regular-file
|
|
||||||
// operand the directories checked start at its directory.
|
|
||||||
dir := r
|
|
||||||
|
|
||||||
fi, err := os.Lstat(r)
|
|
||||||
if err == nil && fi.Mode().IsRegular() {
|
|
||||||
dir = filepath.Dir(r)
|
|
||||||
}
|
|
||||||
|
|
||||||
for ; dir != above; dir = filepath.Dir(dir) {
|
|
||||||
dev, ok := device(dir)
|
|
||||||
if ok && dev != aboveDev {
|
|
||||||
kept = append(kept, r)
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return kept
|
return kept
|
||||||
}
|
}
|
||||||
|
|
||||||
// deviceOfPath returns the filesystem device ID of the file at path,
|
|
||||||
// when lstat succeeds and the platform exposes one.
|
|
||||||
func deviceOfPath(path string) (uint64, bool) {
|
|
||||||
fi, err := os.Lstat(path)
|
|
||||||
if err != nil {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return deviceOfInfo(fi)
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanStats summarizes one scan's database synchronization for the
|
// scanStats summarizes one scan's database synchronization for the
|
||||||
// final stderr summary, or for the line an interrupted scan prints.
|
// final stderr summary, or for the line an interrupted scan prints.
|
||||||
type scanStats struct {
|
type scanStats struct {
|
||||||
@@ -338,22 +288,21 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
|
|||||||
// size, head, and tail match another record's). Records outside the
|
// size, head, and tail match another record's). Records outside the
|
||||||
// roots are never touched, except that the content phase fills in
|
// roots are never touched, except that the content phase fills in
|
||||||
// their content hash. Operands the walk cannot start from are dropped
|
// their content hash. Operands the walk cannot start from are dropped
|
||||||
// first, so the records beneath them are treated as if those operands
|
// first, so the records beneath them count as outside the roots unless
|
||||||
// had not been given.
|
// they lie under another root.
|
||||||
func (s *scanState) runPhases(ctx context.Context, roots []string,
|
func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||||
workers int, oneFS bool,
|
workers int, oneFS bool,
|
||||||
) error {
|
) error {
|
||||||
// Types are checked before pruning so that an operand under a
|
// Types are checked before pruning so that an operand under a
|
||||||
// dropped one is still scanned, not dropped as lying under it.
|
// dropped one is still scanned, not dropped as lying under it.
|
||||||
roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath)
|
roots = pruneRoots(s.walkableRoots(roots))
|
||||||
|
|
||||||
err := s.loadIndex(ctx, roots)
|
err := s.loadIndex(ctx, roots)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
changed, unhashed := s.walkPhase(
|
changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers))
|
||||||
startWalk(ctx, roots, oneFS, workers), roots)
|
|
||||||
|
|
||||||
// A cancelled walk stops early, so its size census covers only part
|
// A cancelled walk stops early, so its size census covers only part
|
||||||
// of the roots, and every file it never reached would look vanished
|
// of the roots, and every file it never reached would look vanished
|
||||||
@@ -381,10 +330,10 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
// walkableRoots returns the operands the walk can start from: regular
|
// walkableRoots returns the operands the walk can start from: regular
|
||||||
// files, and directories not named .zfs. Every other operand is warned
|
// files, and directories not named .zfs. Every other operand is warned
|
||||||
// about, counted as skipped, and dropped. A dropped operand is no
|
// about, counted as skipped, and dropped. A dropped operand is no
|
||||||
// longer a root, so the records stored beneath it are treated as if it
|
// longer a root, so the records stored beneath it count as outside the
|
||||||
// had not been given: those under no other root count as outside the
|
// roots and are not deleted as unverified, unless it lies under another
|
||||||
// roots and are not deleted as unverified. An operand that fails lstat
|
// root. An operand that fails lstat here is kept, and the walk warns
|
||||||
// here is kept, and the walk warns about it.
|
// about it.
|
||||||
func (s *scanState) walkableRoots(roots []string) []string {
|
func (s *scanState) walkableRoots(roots []string) []string {
|
||||||
kept := make([]string, 0, len(roots))
|
kept := make([]string, 0, len(roots))
|
||||||
|
|
||||||
@@ -442,27 +391,15 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
|||||||
// (the content phase may still fill in its content hash). It returns
|
// (the content phase may still fill in its content hash). It returns
|
||||||
// the new-or-changed files and the unchanged files whose records lack
|
// the new-or-changed files and the unchanged files whose records lack
|
||||||
// hashes; both remain candidates until the census decides whether
|
// hashes; both remain candidates until the census decides whether
|
||||||
// their sizes are shared. A record below a directory -x did not enter
|
// their sizes are shared.
|
||||||
// counts as outside the roots: it is not deleted, and its size joins
|
|
||||||
// the census. That does not hold below a root at or inside that
|
|
||||||
// directory, since that root's own walk verifies the records below it.
|
|
||||||
func (s *scanState) walkPhase(
|
func (s *scanState) walkPhase(
|
||||||
events <-chan walkEvent, roots []string,
|
events <-chan walkEvent,
|
||||||
) ([]fileRec, []fileRec) {
|
) ([]fileRec, []fileRec) {
|
||||||
prog := newProgress("walk", -1)
|
prog := newProgress("walk", -1)
|
||||||
|
|
||||||
var (
|
var changed, unhashed []fileRec
|
||||||
changed, unhashed []fileRec
|
|
||||||
otherFSDirs []string
|
|
||||||
)
|
|
||||||
|
|
||||||
for ev := range events {
|
for ev := range events {
|
||||||
if ev.otherFSDir != "" {
|
|
||||||
otherFSDirs = append(otherFSDirs, ev.otherFSDir)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if ev.fail {
|
if ev.fail {
|
||||||
s.st.skipped++
|
s.st.skipped++
|
||||||
|
|
||||||
@@ -492,19 +429,6 @@ func (s *scanState) walkPhase(
|
|||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
for path, old := range s.existing {
|
|
||||||
// Kept only when a directory -x did not enter lies below the
|
|
||||||
// deepest root above the record. A root at or below that
|
|
||||||
// directory was walked on its own and did not find the file.
|
|
||||||
// Both contain the record, so the longer path is the deeper.
|
|
||||||
otherFSDir := deepestRoot(path, otherFSDirs)
|
|
||||||
if len(otherFSDir) > len(deepestRoot(path, roots)) {
|
|
||||||
delete(s.existing, path)
|
|
||||||
|
|
||||||
s.sizes = append(s.sizes, old.size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return changed, unhashed
|
return changed, unhashed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -932,20 +856,6 @@ func underRoot(path, root string) bool {
|
|||||||
return strings.HasPrefix(path, prefix)
|
return strings.HasPrefix(path, prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
// deepestRoot returns the longest of the roots that path is or lies
|
|
||||||
// under, or "" when it lies under none.
|
|
||||||
func deepestRoot(path string, roots []string) string {
|
|
||||||
deepest := ""
|
|
||||||
|
|
||||||
for _, root := range roots {
|
|
||||||
if underRoot(path, root) && len(root) > len(deepest) {
|
|
||||||
deepest = root
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return deepest
|
|
||||||
}
|
|
||||||
|
|
||||||
// dirJob is one directory awaiting traversal by the walk workers. It
|
// dirJob is one directory awaiting traversal by the walk workers. It
|
||||||
// carries its operand's filesystem device so -x can stop at
|
// carries its operand's filesystem device so -x can stop at
|
||||||
// filesystem boundaries.
|
// filesystem boundaries.
|
||||||
@@ -956,23 +866,19 @@ type dirJob struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// walkEvent is one walk result delivered to the walk phase: a regular
|
// walkEvent is one walk result delivered to the walk phase: a regular
|
||||||
// file's statted record, a warning when fail is set, or, when
|
// file's statted record, or a warning when fail is set.
|
||||||
// otherFSDir is set, a directory -x did not enter because it is on
|
|
||||||
// another filesystem than its operand.
|
|
||||||
type walkEvent struct {
|
type walkEvent struct {
|
||||||
rec fileRec
|
rec fileRec
|
||||||
warn string
|
warn string
|
||||||
fail bool
|
fail bool
|
||||||
otherFSDir string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// startWalk seeds every root into the shared walk worker pool and
|
// startWalk seeds every root into the shared walk worker pool and
|
||||||
// returns the event stream: one record per regular file, one warning
|
// returns the event stream: one record per regular file, one warning
|
||||||
// event per per-path error, and one event per directory -x did not
|
// event per per-path error. The channel is closed when the walk
|
||||||
// enter. The channel is closed when the walk completes, and also when
|
// completes, and also when ctx is cancelled — every goroutine in the
|
||||||
// ctx is cancelled — every goroutine in the pool abandons its blocking
|
// pool abandons its blocking send in that case, so the consumer sees a
|
||||||
// send in that case, so the consumer sees a truncated but properly
|
// truncated but properly terminated stream instead of a stalled one.
|
||||||
// terminated stream instead of a stalled one.
|
|
||||||
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
||||||
workers int,
|
workers int,
|
||||||
) <-chan walkEvent {
|
) <-chan walkEvent {
|
||||||
@@ -1039,8 +945,7 @@ func operandWarning(root string, fi fs.FileInfo) string {
|
|||||||
// operand becomes an initial job. walkableRoots has already dropped
|
// operand becomes an initial job. walkableRoots has already dropped
|
||||||
// every other operand. One that has changed into something else since
|
// every other operand. One that has changed into something else since
|
||||||
// is warned about and skipped here; it is still a root, so the records
|
// is warned about and skipped here; it is still a root, so the records
|
||||||
// stored beneath it, other than those under another root, are deleted
|
// stored beneath it are deleted as unverified.
|
||||||
// as unverified.
|
|
||||||
func seedRoot(ctx context.Context, root string,
|
func seedRoot(ctx context.Context, root string,
|
||||||
events chan<- walkEvent,
|
events chan<- walkEvent,
|
||||||
) []dirJob {
|
) []dirJob {
|
||||||
@@ -1239,9 +1144,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
// subdirJob applies the descent rules to directory p: never enter
|
// subdirJob applies the descent rules to directory p: never enter
|
||||||
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
|
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
|
||||||
// snapshot), and with -x never enter a directory on a different
|
// snapshot), and with -x never enter a directory on a different
|
||||||
// filesystem than its operand, sending an event that names it so the
|
// filesystem than its operand.
|
||||||
// scan keeps the records below it, except those below an operand at or
|
|
||||||
// inside it.
|
|
||||||
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||||
parent dirJob, oneFS bool, events chan<- walkEvent,
|
parent dirJob, oneFS bool, events chan<- walkEvent,
|
||||||
) (dirJob, bool) {
|
) (dirJob, bool) {
|
||||||
@@ -1265,8 +1168,6 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
|
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
|
||||||
sendEvent(ctx, events, walkEvent{otherFSDir: p})
|
|
||||||
|
|
||||||
return dirJob{}, false
|
return dirJob{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1408,38 +1309,6 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// errNotRegular reports a path that is no longer a regular file when it
|
|
||||||
// is opened to be read.
|
|
||||||
var errNotRegular = errors.New("not a regular file")
|
|
||||||
|
|
||||||
// openRegularFile opens path for reading and returns it only if it is
|
|
||||||
// still a regular file, since the path may have been replaced after it
|
|
||||||
// was statted. O_NONBLOCK keeps the open of a FIFO or a device node
|
|
||||||
// from waiting forever, and fstat then rejects it. Reads of a regular
|
|
||||||
// file ignore O_NONBLOCK.
|
|
||||||
func openRegularFile(path string) (*os.File, error) {
|
|
||||||
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
|
||||||
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
fi, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
_ = f.Close()
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !fi.Mode().IsRegular() {
|
|
||||||
_ = f.Close()
|
|
||||||
|
|
||||||
return nil, errNotRegular
|
|
||||||
}
|
|
||||||
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
|
// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
|
||||||
// tail, and content hash of every zero-length file.
|
// tail, and content hash of every zero-length file.
|
||||||
const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
|
const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
|
||||||
@@ -1462,7 +1331,8 @@ func hashSignature(path string, size int64) (string, string, string, error) {
|
|||||||
return emptyHash, emptyHash, emptyHash, nil
|
return emptyHash, emptyHash, emptyHash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := openRegularFile(path)
|
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||||
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", "", err
|
return "", "", "", err
|
||||||
}
|
}
|
||||||
@@ -1493,7 +1363,8 @@ func hashSignature(path string, size int64) (string, string, string, error) {
|
|||||||
// tail are returned empty, because the content phase keeps the ones its
|
// tail are returned empty, because the content phase keeps the ones its
|
||||||
// records already hold.
|
// records already hold.
|
||||||
func hashContentOnly(path string, size int64) (string, string, string, error) {
|
func hashContentOnly(path string, size int64) (string, string, string, error) {
|
||||||
f, err := openRegularFile(path)
|
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||||
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", "", err
|
return "", "", "", err
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-378
@@ -997,7 +997,7 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// callSubdirJob runs subdirJob against e under parent, collecting any
|
// callSubdirJob runs subdirJob against e under parent, collecting any
|
||||||
// events it emits (subdirJob emits at most one).
|
// warning events it emits (subdirJob emits at most one).
|
||||||
func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
|
func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
|
||||||
parent dirJob, oneFS bool,
|
parent dirJob, oneFS bool,
|
||||||
) (dirJob, bool, []walkEvent) {
|
) (dirJob, bool, []walkEvent) {
|
||||||
@@ -1075,15 +1075,8 @@ func TestSubdirJobOneFilesystem(t *testing.T) {
|
|||||||
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
|
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A refused descent names the directory, so the scan keeps
|
if len(evs) != 0 {
|
||||||
// the records below it.
|
t.Fatalf("unexpected events: %+v", evs)
|
||||||
var wantEvs []walkEvent
|
|
||||||
if !ok {
|
|
||||||
wantEvs = []walkEvent{{otherFSDir: sub}}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(evs, wantEvs) {
|
|
||||||
t.Fatalf("events = %+v, want %+v", evs, wantEvs)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The accepted job must carry the operand's device down, or -x
|
// The accepted job must carry the operand's device down, or -x
|
||||||
@@ -1135,296 +1128,6 @@ func TestSubdirJobStatError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// syncOneFilesystem scans roots into db with -x and returns the scan
|
|
||||||
// stats. Every device pruneRoots looks up, and the device each
|
|
||||||
// operand's walk compares its subdirectories with, comes from devices
|
|
||||||
// instead of from lstat, so a test can put a path on a filesystem that
|
|
||||||
// no directory here is on. Unlike runPhases, it skips walkableRoots,
|
|
||||||
// the check for a cancelled walk, and the content phase.
|
|
||||||
func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
|
|
||||||
devices map[string]uint64,
|
|
||||||
) scanStats {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
device := func(path string) (uint64, bool) {
|
|
||||||
dev, ok := devices[path]
|
|
||||||
|
|
||||||
return dev, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
roots = pruneRoots(roots, true, device)
|
|
||||||
|
|
||||||
ctx := t.Context()
|
|
||||||
s := &scanState{db: db}
|
|
||||||
|
|
||||||
err := s.loadIndex(ctx, roots)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
|
|
||||||
|
|
||||||
// seedRoot emits a regular-file operand itself and returns a job
|
|
||||||
// only for a directory operand.
|
|
||||||
var initial []dirJob
|
|
||||||
|
|
||||||
for _, root := range roots {
|
|
||||||
for _, job := range seedRoot(ctx, root, events) {
|
|
||||||
job.rootDev = devices[root]
|
|
||||||
initial = append(initial, job)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
dispatchDirs(ctx, initial, jobs, subdirs)
|
|
||||||
|
|
||||||
changed, unhashed := s.walkPhase(events, roots)
|
|
||||||
s.partition(changed, unhashed)
|
|
||||||
|
|
||||||
err = s.hashPhase(ctx, 2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = s.updatePhase(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.st
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
|
|
||||||
// record below a directory the walk does not enter, because it is on
|
|
||||||
// another filesystem than its operand, counts as outside the operands:
|
|
||||||
// it is not deleted, and its size joins the size census. The record of
|
|
||||||
// a gone file that is not below that directory is still deleted, even
|
|
||||||
// when its path begins with the directory's name. The operand is given
|
|
||||||
// a device that no directory here is on, so no second real filesystem
|
|
||||||
// is needed.
|
|
||||||
func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
// A scan without -x records the file below mnt, and mnt2, which is
|
|
||||||
// beside mnt, not below it.
|
|
||||||
mounted := writeFile(t, dir, "mnt/f", pattern(1, 100))
|
|
||||||
gone := writeFile(t, dir, "mnt2", pattern(3, 50))
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
err := os.Remove(gone)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same size as mnt/f, so it is hashed only if mnt/f's size is
|
|
||||||
// in the census.
|
|
||||||
added := writeFile(t, dir, "added", pattern(2, 100))
|
|
||||||
|
|
||||||
info, err := os.Lstat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ok := deviceOfInfo(info)
|
|
||||||
if !ok {
|
|
||||||
t.Skip("platform exposes no device id")
|
|
||||||
}
|
|
||||||
|
|
||||||
syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1})
|
|
||||||
|
|
||||||
recs := dbRecords(t, db)
|
|
||||||
recordByPath(t, recs, mounted)
|
|
||||||
|
|
||||||
if recordByPath(t, recs, added).head == "" {
|
|
||||||
t.Errorf("%s not hashed: the census lacks the size of %s",
|
|
||||||
added, mounted)
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(recordPaths(recs), gone) {
|
|
||||||
t.Errorf("record for %s kept: the file is gone and not below mnt",
|
|
||||||
gone)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanOneFilesystemWalksNestedOperand checks that with -x an
|
|
||||||
// operand under another one, on a different filesystem than it, is
|
|
||||||
// walked on its own: a file added below it is recorded, and the record
|
|
||||||
// of a file gone from below it is deleted. A record below a directory
|
|
||||||
// that its own walk does not enter is still kept. Each operand is given
|
|
||||||
// a device that no directory here is on, so neither walk enters a
|
|
||||||
// subdirectory.
|
|
||||||
func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
// A scan without -x records both files below mnt.
|
|
||||||
gone := writeFile(t, dir, "mnt/gone", pattern(1, 10))
|
|
||||||
kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20))
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
err := os.Remove(gone)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
added := writeFile(t, dir, "mnt/added", pattern(3, 30))
|
|
||||||
|
|
||||||
info, err := os.Lstat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ok := deviceOfInfo(info)
|
|
||||||
if !ok {
|
|
||||||
t.Skip("platform exposes no device id")
|
|
||||||
}
|
|
||||||
|
|
||||||
mnt := filepath.Join(dir, "mnt")
|
|
||||||
syncOneFilesystem(t, db, []string{dir, mnt},
|
|
||||||
map[string]uint64{dir: dev + 1, mnt: dev + 2})
|
|
||||||
|
|
||||||
paths := recordPaths(dbRecords(t, db))
|
|
||||||
|
|
||||||
if !slices.Contains(paths, added) {
|
|
||||||
t.Errorf("no record for %s: the walk did not reach %s", added, mnt)
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(paths, gone) {
|
|
||||||
t.Errorf("record for %s kept: the file is gone", gone)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Contains(paths, kept) {
|
|
||||||
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
|
|
||||||
kept, mnt, filepath.Dir(kept))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that
|
|
||||||
// with -x an operand on the same filesystem as the operand covering it
|
|
||||||
// is walked on its own when a directory between them is on another
|
|
||||||
// filesystem, since the walk of the covering operand does not enter
|
|
||||||
// that directory: a file added below the nested operand is recorded,
|
|
||||||
// and the record of a file gone from below it is deleted. A record
|
|
||||||
// below that directory but outside the nested operand is still kept.
|
|
||||||
// Both operands are given one device that no directory here is on,
|
|
||||||
// while mnt keeps its own.
|
|
||||||
func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
// A scan without -x records both files below mnt.
|
|
||||||
gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10))
|
|
||||||
kept := writeFile(t, dir, "mnt/kept", pattern(2, 20))
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
err := os.Remove(gone)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
added := writeFile(t, dir, "mnt/back/added", pattern(3, 30))
|
|
||||||
|
|
||||||
info, err := os.Lstat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ok := deviceOfInfo(info)
|
|
||||||
if !ok {
|
|
||||||
t.Skip("platform exposes no device id")
|
|
||||||
}
|
|
||||||
|
|
||||||
mnt := filepath.Join(dir, "mnt")
|
|
||||||
back := filepath.Join(mnt, "back")
|
|
||||||
syncOneFilesystem(t, db, []string{dir, back},
|
|
||||||
map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1})
|
|
||||||
|
|
||||||
paths := recordPaths(dbRecords(t, db))
|
|
||||||
|
|
||||||
if !slices.Contains(paths, added) {
|
|
||||||
t.Errorf("no record for %s: the walk did not reach %s", added, back)
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(paths, gone) {
|
|
||||||
t.Errorf("record for %s kept: the file is gone", gone)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Contains(paths, kept) {
|
|
||||||
t.Errorf("record for %s deleted: the walk of %s did not enter %s",
|
|
||||||
kept, dir, mnt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
|
|
||||||
// regular-file operand under another operand is dropped when the file
|
|
||||||
// reports a different device than that operand but its directory does
|
|
||||||
// not, as a file bind-mounted into the tree does: the walk of the other
|
|
||||||
// operand enters the directory and reaches the file, so the file is
|
|
||||||
// recorded once.
|
|
||||||
func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
file := writeFile(t, dir, "sub/f", pattern(1, 10))
|
|
||||||
|
|
||||||
info, err := os.Lstat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ok := deviceOfInfo(info)
|
|
||||||
if !ok {
|
|
||||||
t.Skip("platform exposes no device id")
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
|
|
||||||
dir: dev, filepath.Dir(file): dev, file: dev + 1,
|
|
||||||
})
|
|
||||||
|
|
||||||
if st.added != 1 {
|
|
||||||
t.Errorf("%s recorded %d times, want once", file, st.added)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a
|
|
||||||
// regular-file operand under another operand is kept when its directory
|
|
||||||
// is on a different filesystem than that operand: the walk of the other
|
|
||||||
// operand does not enter the directory, so the file is recorded once,
|
|
||||||
// as its own operand. The other operand is given a device that no
|
|
||||||
// directory here is on.
|
|
||||||
func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
file := writeFile(t, dir, "sub/f", pattern(1, 10))
|
|
||||||
|
|
||||||
info, err := os.Lstat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dev, ok := deviceOfInfo(info)
|
|
||||||
if !ok {
|
|
||||||
t.Skip("platform exposes no device id")
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
|
|
||||||
dir: dev + 1, filepath.Dir(file): dev,
|
|
||||||
})
|
|
||||||
|
|
||||||
if st.added != 1 {
|
|
||||||
t.Errorf("%s recorded %d times, want once", file, st.added)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||||
// with deterministic content and returns the tree root.
|
// with deterministic content and returns the tree root.
|
||||||
func buildSmokeTree(t *testing.T) string {
|
func buildSmokeTree(t *testing.T) string {
|
||||||
@@ -2094,54 +1797,6 @@ func TestScanHardlinkRunFailsTogether(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file
|
|
||||||
// with a FIFO before the hash phase reads it. Opening a FIFO that has
|
|
||||||
// no writer waits for one, so a blocking open would hang the phase. The
|
|
||||||
// path must be warned about once and skipped, with no record written.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
dir := t.TempDir()
|
|
||||||
p := writeFile(t, dir, "a.bin", pattern(1, 300))
|
|
||||||
|
|
||||||
walked, _ := collectWalk(t, []string{dir}, false, 1)
|
|
||||||
|
|
||||||
err := os.Remove(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = unix.Mkfifo(p, 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &scanState{db: openTestDB(t), toHash: walked}
|
|
||||||
|
|
||||||
err = s.hashPhase(t.Context(), 1)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hashPhase = %v, want nil", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.st.skipped != 1 || len(s.batch) != 0 {
|
|
||||||
t.Errorf("skipped = %d, records = %d, want 1 skipped and none",
|
|
||||||
s.st.skipped, len(s.batch))
|
|
||||||
}
|
|
||||||
|
|
||||||
warning := "hash " + p + ": not a regular file\n"
|
|
||||||
if got := stderr(); strings.Count(got, warning) != 1 {
|
|
||||||
t.Errorf("stderr = %q, want %q once", got, warning)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The content phase opens files the same way.
|
|
||||||
_, _, content, err := hashContentOnly(p, headTailMin)
|
|
||||||
if content != "" || !errors.Is(err, errNotRegular) {
|
|
||||||
t.Errorf("hashContentOnly on a FIFO = %q, %v; want no hash and %v",
|
|
||||||
content, err, errNotRegular)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// injectedWriteFailure is the message the injected database trigger
|
// injectedWriteFailure is the message the injected database trigger
|
||||||
// aborts with, so the test can recognize its own failure in the error
|
// aborts with, so the test can recognize its own failure in the error
|
||||||
// the scan reports.
|
// the scan reports.
|
||||||
@@ -2335,43 +1990,14 @@ func TestHashRuns(t *testing.T) {
|
|||||||
func TestPruneRoots(t *testing.T) {
|
func TestPruneRoots(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const pqRoot = "/p/q"
|
|
||||||
|
|
||||||
// /p/q is on another filesystem than /p; /p/q/r is on the same one as
|
|
||||||
// /p/q, /p/s as /p, and /p/t has no known device.
|
|
||||||
devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1}
|
|
||||||
device := func(path string) (uint64, bool) {
|
|
||||||
dev, ok := devices[path]
|
|
||||||
|
|
||||||
return dev, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// Duplicates and operands under other operands are dropped; /cc is
|
// Duplicates and operands under other operands are dropped; /cc is
|
||||||
// not under /c (sibling with a shared prefix).
|
// not under /c (sibling with a shared prefix).
|
||||||
got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"},
|
got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
|
||||||
false, device)
|
|
||||||
|
|
||||||
want := []string{"/a", "/c", "/cc"}
|
want := []string{"/a", "/c", "/cc"}
|
||||||
if !slices.Equal(got, want) {
|
if !slices.Equal(got, want) {
|
||||||
t.Fatalf("pruneRoots = %q, want %q", got, want)
|
t.Fatalf("pruneRoots = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"}
|
|
||||||
|
|
||||||
got = pruneRoots(nested, false, device)
|
|
||||||
|
|
||||||
want = []string{"/p"}
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("pruneRoots without -x = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// With -x, /p/q, on another filesystem than /p, is kept as well.
|
|
||||||
got = pruneRoots(nested, true, device)
|
|
||||||
|
|
||||||
want = []string{"/p", pqRoot}
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("pruneRoots with -x = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportsNeverTouchFilesystem(t *testing.T) {
|
func TestReportsNeverTouchFilesystem(t *testing.T) {
|
||||||
|
|||||||
+11
-34
@@ -2,12 +2,10 @@
|
|||||||
# script/bootstrap: install all dependencies needed to build and develop
|
# script/bootstrap: install all dependencies needed to build and develop
|
||||||
# this repo. Idempotent: every install is guarded by a check so already
|
# this repo. Idempotent: every install is guarded by a check so already
|
||||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||||
# or apk (detected in that order); assumes nothing is present. The
|
# or apk (detected in that order); assumes nothing is present. Node is
|
||||||
# installed node is used only when its major version is the pinned one;
|
# used directly if installed; otherwise it is installed at a pinned
|
||||||
# otherwise (node missing, or another major) the pinned version is
|
# version via nvm (installing nvm itself first, from a hash-verified
|
||||||
# installed via nvm (installing nvm itself first, from a hash-verified
|
# release archive, never curl | sh).
|
||||||
# release archive, never curl | sh), and yarn and the packages are
|
|
||||||
# installed under it.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -102,52 +100,31 @@ ensure_nvm() {
|
|||||||
rm -rf "$tmp"
|
rm -rf "$tmp"
|
||||||
}
|
}
|
||||||
|
|
||||||
# True when the node on PATH has the pinned major version. Only the
|
|
||||||
# major is compared: the Dockerfile stages start from a node 22 alpine
|
|
||||||
# image whose exact version is not the pin, and nvm cannot install a
|
|
||||||
# prebuilt node on alpine. Another major is not used: the pinned yarn 1
|
|
||||||
# prints a deprecation warning under node 24. script/fmt and
|
|
||||||
# script/fmt-check carry this function unchanged, to run the yarn
|
|
||||||
# installed here.
|
|
||||||
node_is_pinned_major() {
|
|
||||||
if ! command -v node >/dev/null 2>&1; then return 1; fi
|
|
||||||
installed="$(node --version)"
|
|
||||||
installed="${installed#v}"
|
|
||||||
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_node() {
|
ensure_node() {
|
||||||
if node_is_pinned_major; then
|
if ! missing node; then return 0; fi
|
||||||
echo "bootstrap: using node $(node --version)"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
ensure_nvm
|
ensure_nvm
|
||||||
nvm_sh "nvm install $NODE_VERSION"
|
nvm_sh "nvm install $NODE_VERSION"
|
||||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && \
|
|
||||||
echo \"bootstrap: using node \$(node --version)\""
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ensure_yarn() {
|
ensure_yarn() {
|
||||||
if ! node_is_pinned_major; then
|
|
||||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
|
||||||
corepack prepare yarn@$YARN_VERSION --activate"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if ! missing yarn; then return 0; fi
|
if ! missing yarn; then return 0; fi
|
||||||
if ! missing corepack; then
|
if ! missing corepack; then
|
||||||
corepack enable
|
corepack enable
|
||||||
corepack prepare "yarn@$YARN_VERSION" --activate
|
corepack prepare "yarn@$YARN_VERSION" --activate
|
||||||
|
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||||
|
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
||||||
|
corepack prepare yarn@$YARN_VERSION --activate"
|
||||||
else
|
else
|
||||||
npm install -g "yarn@$YARN_VERSION"
|
npm install -g "yarn@$YARN_VERSION"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
install_js_deps() {
|
install_js_deps() {
|
||||||
if node_is_pinned_major; then
|
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||||
yarn install --frozen-lockfile
|
|
||||||
else
|
|
||||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
|
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
|
||||||
yarn install --frozen-lockfile"
|
yarn install --frozen-lockfile"
|
||||||
|
else
|
||||||
|
yarn install --frozen-lockfile
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+4
-6
@@ -2,11 +2,9 @@
|
|||||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||||
# the formatter on the host, which a pristine checkout cannot do.
|
# the formatter on the host, which a pristine checkout cannot do.
|
||||||
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
# --no-cache for the same reason as script/docker: the gate phases the
|
||||||
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
# final stage depends on are RUN steps, and a cached one is a check that
|
||||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
# did not run.
|
||||||
# first; a stage without that line is served from cache on an unchanged
|
|
||||||
# tree, with no warning.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
@@ -24,7 +22,7 @@ main() {
|
|||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||||
[ -n "$version" ] || version="unknown"
|
[ -n "$version" ] || version="unknown"
|
||||||
tag="$("$SCRIPT_DIR/projectname")"
|
tag="$("$SCRIPT_DIR/projectname")"
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
docker build --no-cache \
|
||||||
--build-arg VERSION="$version" \
|
--build-arg VERSION="$version" \
|
||||||
-t "$tag" .
|
-t "$tag" .
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-6
@@ -1,11 +1,8 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/docker: build the Docker image tagged with the project name.
|
# script/docker: build the Docker image tagged with the project name.
|
||||||
# Identical in all repos; the tag comes from script/projectname.
|
# Identical in all repos; the tag comes from script/projectname.
|
||||||
# A new CACHEBUST value reruns every RUN step, since a cached gate phase
|
# --no-cache because the gate phases the final stage depends on are RUN
|
||||||
# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
|
# steps, and a cached one is a check that did not run.
|
||||||
# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
|
|
||||||
# first; a stage without that line is served from cache on an unchanged
|
|
||||||
# tree, with no warning.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
@@ -21,7 +18,7 @@ main() {
|
|||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||||
[ -n "$version" ] || version="unknown"
|
[ -n "$version" ] || version="unknown"
|
||||||
tag="$("$SCRIPT_DIR/projectname")"
|
tag="$("$SCRIPT_DIR/projectname")"
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
docker build --no-cache \
|
||||||
--build-arg VERSION="$version" \
|
--build-arg VERSION="$version" \
|
||||||
-t "$tag" .
|
-t "$tag" .
|
||||||
}
|
}
|
||||||
|
|||||||
+6
-16
@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|||||||
# Must match the pin in script/bootstrap.
|
# Must match the pin in script/bootstrap.
|
||||||
NODE_VERSION="22.17.0"
|
NODE_VERSION="22.17.0"
|
||||||
|
|
||||||
# True when the node on PATH has the pinned major version: the same
|
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||||
# function as in script/bootstrap, which uses it to decide where it
|
# on the PATH of the shell that called it, so resolve the pinned
|
||||||
# installs yarn.
|
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||||
node_is_pinned_major() {
|
# bash script, hence the subshell.
|
||||||
if ! command -v node >/dev/null 2>&1; then return 1; fi
|
|
||||||
installed="$(node --version)"
|
|
||||||
installed="${installed#v}"
|
|
||||||
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run the yarn script/bootstrap installed: the yarn on PATH when the
|
|
||||||
# node on PATH has the pinned major version, and otherwise yarn under
|
|
||||||
# the pinned node in nvm, which bootstrap leaves off the PATH of the
|
|
||||||
# shell that called it. nvm is a bash script, hence the subshell.
|
|
||||||
run_yarn() {
|
run_yarn() {
|
||||||
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
|
if command -v yarn >/dev/null 2>&1; then
|
||||||
exec yarn "$@"
|
exec yarn "$@"
|
||||||
fi
|
fi
|
||||||
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||||
echo "fmt: no yarn; run script/bootstrap first" >&2
|
echo "fmt: no yarn; run script/bootstrap first" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
+6
-16
@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|||||||
# Must match the pin in script/bootstrap.
|
# Must match the pin in script/bootstrap.
|
||||||
NODE_VERSION="22.17.0"
|
NODE_VERSION="22.17.0"
|
||||||
|
|
||||||
# True when the node on PATH has the pinned major version: the same
|
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||||
# function as in script/bootstrap, which uses it to decide where it
|
# on the PATH of the shell that called it, so resolve the pinned
|
||||||
# installs yarn.
|
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||||
node_is_pinned_major() {
|
# bash script, hence the subshell.
|
||||||
if ! command -v node >/dev/null 2>&1; then return 1; fi
|
|
||||||
installed="$(node --version)"
|
|
||||||
installed="${installed#v}"
|
|
||||||
[ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run the yarn script/bootstrap installed: the yarn on PATH when the
|
|
||||||
# node on PATH has the pinned major version, and otherwise yarn under
|
|
||||||
# the pinned node in nvm, which bootstrap leaves off the PATH of the
|
|
||||||
# shell that called it. nvm is a bash script, hence the subshell.
|
|
||||||
run_yarn() {
|
run_yarn() {
|
||||||
if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
|
if command -v yarn >/dev/null 2>&1; then
|
||||||
exec yarn "$@"
|
exec yarn "$@"
|
||||||
fi
|
fi
|
||||||
if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||||
echo "fmt-check: no yarn; run script/bootstrap first" >&2
|
echo "fmt-check: no yarn; run script/bootstrap first" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
+4
-7
@@ -5,12 +5,9 @@
|
|||||||
# make its answer untrustworthy.
|
# make its answer untrustworthy.
|
||||||
#
|
#
|
||||||
# The phase is not the last stage in the file, so it is built only when
|
# The phase is not the last stage in the file, so it is built only when
|
||||||
# --target names it. A new CACHEBUST value reruns every RUN step, since
|
# --target names it. --no-cache because a cached lint layer is a lint
|
||||||
# a cached lint layer is a lint that did not run; unlike --no-cache, it
|
# that did not run. --output type=cacheonly writes no image, since
|
||||||
# keeps Go's build cache. This relies on every stage of the Dockerfile
|
# nothing uses one.
|
||||||
# declaring ARG CACHEBUST first; a stage without that line is served
|
|
||||||
# from cache on an unchanged tree, with no warning.
|
|
||||||
# --output type=cacheonly writes no image, since nothing uses one.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
@@ -18,7 +15,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
docker build --no-cache \
|
||||||
--target lint \
|
--target lint \
|
||||||
--output type=cacheonly .
|
--output type=cacheonly .
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-7
@@ -2,12 +2,8 @@
|
|||||||
# script/test: run the test suite. Testing is a phase of the Dockerfile
|
# script/test: run the test suite. Testing is a phase of the Dockerfile
|
||||||
# and this builds that phase alone, on the same terms as script/lint:
|
# and this builds that phase alone, on the same terms as script/lint:
|
||||||
# --target because a phase that is not the last stage is built only when
|
# --target because a phase that is not the last stage is built only when
|
||||||
# named, and a new CACHEBUST value because a cached test layer is a test
|
# named, and --no-cache because a cached test layer is a test that did
|
||||||
# that did not run; unlike --no-cache, it keeps Go's build cache. This
|
# not run. --output type=cacheonly writes no image, since nothing uses one.
|
||||||
# relies on every stage of the Dockerfile declaring ARG CACHEBUST first;
|
|
||||||
# a stage without that line is served from cache on an unchanged tree,
|
|
||||||
# with no warning. --output type=cacheonly writes no image, since
|
|
||||||
# nothing uses one.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
@@ -15,7 +11,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
docker build --no-cache \
|
||||||
--target test \
|
--target test \
|
||||||
--output type=cacheonly .
|
--output type=cacheonly .
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user