Re-vendor the canonical files from sneak/prompts at c55a0cb (closes #95)

Every vendored file, REPO_POLICIES.md and every model script is the
copy at sneak/prompts c55a0cb, with this repository's own entries kept
after the canonical content. Lint and test are phases of the Dockerfile
that write no image, built uncached. make test runs the suite under the
race detector as nobody, because root reads the files the tests make
unreadable. Dockerfile.lint, script/verify-lint-image-pin and
make test-race are gone. Prettier runs on the host, from the node and
yarn that script/bootstrap installs. golangci-lint v2.14.0 raises no
findings. .claude/settings.json is deleted.

Deviation: the set comes from c55a0cb on next rather than dd4027b, as
the instructions on sneak/prompts#78 allow.

Model: opus-5-5

Dockerfile

@@ -1,14 +1,3 @@
-# In every stage ARG CACHEBUST comes right after the steps that install
-# system packages and download Go's modules, or right after FROM in a
-# stage with neither, and every step that lints, tests or compiles
-# comes after it. script/ passes a new value on every build, which
-# reruns every RUN step after it. Docker keeps the layers above it
-# until the base image, their own lines or go.mod and go.sum change.
-# The steps that compile mount Go's build cache, and golangci-lint run
-# also mounts golangci-lint's; both survive from build to build. The
-# test phase runs as nobody and so mounts Go's build cache at
-# /tmp/.cache/go-build; its own comment says why.
-
 # Lint phase, built alone by script/lint. The tools are invoked directly
 # rather than through `make lint`, which runs docker itself and so cannot
 # run inside a build step.
@@ -17,7 +6,6 @@ FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a924
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
-ARG CACHEBUST
 COPY . .
 
 # The gofmt half of `make fmt-check`. gofmt's output is assigned to a
@@ -30,20 +18,14 @@ RUN files="$(gofmt -s -l .)" && \
 
 # Validates .golangci.yml against the schema the pinned binary embeds.
 RUN golangci-lint config verify --config .golangci.yml
-RUN --mount=type=cache,target=/root/.cache/go-build \
+RUN golangci-lint run --config .golangci.yml ./...
-    --mount=type=cache,target=/root/.cache/golangci-lint \
-    golangci-lint run --config .golangci.yml ./...
 
 # Test phase, built alone by script/test. -race needs cgo and so a C
 # compiler, which the Debian Go image ships and the alpine one does not.
-# -count=1 because the mounted cache also holds test results, which go
-# test would otherwise replay.
 #
 # The tests run as nobody: several of them make a file unreadable and
 # expect reading it to fail, and root reads it anyway. nobody has no home
-# directory, so HOME is /tmp, and Go keeps its build cache in
+# directory, so HOME is /tmp, where Go puts its build cache.
-# /tmp/.cache/go-build. That is where the cache is mounted, owned by
-# nobody (65534) so the tests can write to it.
 # golang:1.25-trixie, 2026-10-04
 FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
 USER nobody
@@ -51,27 +33,24 @@ ENV HOME=/tmp
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
-ARG CACHEBUST
 COPY . .
-RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
+RUN go test -timeout 90s -race -cover ./... || \
-    go test -count=1 -timeout 90s -race -cover ./... || \
     { echo "--- Rerunning with -v for details ---"; \
-      go test -count=1 -timeout 90s -race -v ./...; exit 1; }
+      go test -timeout 90s -race -v ./...; exit 1; }
 
 # Build stage. Nothing is wanted from either phase above; the copies are
-# what make BuildKit build them first, so nothing after them can run
+# what make BuildKit build them first, so this stage cannot run unless
-# unless lint and test passed.
+# lint and test passed.
 # golang:1.25-alpine, 2026-07-23
 FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
+COPY --from=lint /src/go.sum /dev/null
+COPY --from=test /src/go.sum /dev/null
 RUN apk add --no-cache git make
+# A tar-stream context keeps the sender's file owners, which git refuses.
+RUN git config --system --add safe.directory /src
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
-ARG CACHEBUST
-COPY --from=lint /src/go.sum /dev/null
-COPY --from=test /src/go.sum /dev/null
-# A tar-stream context keeps the sender's file owners, which git refuses.
-RUN git config --system --add safe.directory /src
 COPY . .
 
 # The version stamped into the binary: the VERSION build argument when
@@ -81,8 +60,7 @@ COPY . .
 # "dev". `make build` rather than `go build`, so the image and a host
 # build share one compile recipe, cgo disabled included.
 ARG VERSION
-RUN --mount=type=cache,target=/root/.cache/go-build \
+RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
-    version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
     if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
         [ "$version" = unknown ]; }; then \
         echo "no version could be derived although the build context carries .git" >&2; \
@@ -94,7 +72,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
 # stage's chain and nothing else.
 # alpine:3.22, 2026-07-23
 FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
-ARG CACHEBUST
 
 COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
 

REPO_POLICIES.md

@@ -1,6 +1,6 @@
 ---
 title: Repository Policies
-last_modified: 2026-10-18
+last_modified: 2026-10-07
 ---
 
 This document covers repository structure, tooling, and workflow standards. Code
@@ -54,39 +54,34 @@ style conventions are in separate documents:
   `cibuild`. `script/bootstrap` installs all dependencies idempotently and
   assumes nothing is present: base tools come from nix, apt, brew, or apk
   (detected in that order; apt runs noninteractive). For node it uses the
-  installed node only when its major version is the pinned one; otherwise (node
+  installed node if present; otherwise it installs a PINNED node version via
-  missing, or another major, such as the node 24 the CI runner image ships) it
+  nvm, first installing nvm itself if missing — from a hash-verified GitHub
-  installs the PINNED node version via nvm and installs yarn and the packages
+  release archive (never `curl | sh`), with bash installed as an explicit
-  under it. nvm itself is installed first if missing, from a hash-verified
+  prerequisite since nvm requires bash. yarn is then pinned via
-  GitHub release archive (never `curl | sh`), with bash installed as an explicit
+  `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
-  prerequisite since nvm requires bash. Only the major version is compared
+  always exact versions. `script/cibuild` runs the CI build: it changes to the
-  because the `Dockerfile` stages start from a node image whose exact version is
+  repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
-  not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
+  with the version; the Gitea workflow calls it. **`script/cibuild` runs
-  via `corepack prepare yarn@<version> --activate`. Never install "latest" or
-  "lts"; always exact versions. `script/cibuild` runs the CI build: it changes
-  to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
-  image with the version; the Gitea workflow calls it. **`script/cibuild` runs
   `script/bootstrap` first**, because the workflow checks out the repo and runs
   nothing else, while `script/fmt-check` runs the formatter on the host: on a
   pristine checkout with nothing installed the run dies there, after the
-  containerised gates have passed. **The bootstrap alone is not enough**: when
+  containerised gates have passed. **The bootstrap alone is not enough**:
-  `script/bootstrap` installs node and yarn under nvm it leaves neither on the
+  `script/bootstrap` installs node and yarn under nvm and leaves neither on the
-  `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
+  `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
-  under another node. The host entrypoints that need yarn — `script/fmt` and
+  entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
-  `script/fmt-check` — therefore use the same test as `script/bootstrap`: when
+  source nvm for the pinned node version before invoking it, exactly as
-  the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
+  `script/bootstrap`'s own install step does. A runner carrying nothing but
-  and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
+  docker and git then gets through `script/check`. Four further scripts are our
-  pinned node. A runner carrying nothing but docker and git then gets through
+  own extensions to the standard: `script/check` runs `script/test`,
-  `script/check`. Four further scripts are our own extensions to the standard:
+  `script/lint` and `script/fmt-check`; `script/precommit` is what the git
-  `script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
+  pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
-  `script/precommit` is what the git pre-commit hook runs, and it calls
+  installs the git pre-commit hook (the `make hooks` target shims to it); and
-  `script/check`; `script/install-precommit` installs the git pre-commit hook
+  `script/projectname` (literally that filename) simply outputs the project's
-  (the `make hooks` target shims to it); and `script/projectname` (literally
+  name. Scripts that need the name call `script/projectname` — e.g.
-  that filename) simply outputs the project's name. Scripts that need the name
+  `script/docker` assembles its image tag from it — so those scripts stay
-  call `script/projectname` — e.g. `script/docker` assembles its image tag from
+  byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
-  it — so those scripts stay byte-identical across all repos. Repo-type-specific
+  `go mod tidy` verification in Go repos) belong in `script/precommit`, not in
-  pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
+  the hook itself. Model scripts are at
-  `script/precommit`, not in the hook itself. Model scripts are at
   `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
   must document the provided scripts in an **Entrypoints** section (see the
   README requirements below).
@@ -123,10 +118,8 @@ style conventions are in separate documents:
   and nothing else:
 
     ```sh
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+    docker build --no-cache --target lint --output type=cacheonly .
-        --target lint --output type=cacheonly .
+    docker build --no-cache --target test --output type=cacheonly .
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
-        --target test --output type=cacheonly .
     ```
 
     **A stage that is not the last one in the file is built only when the final
@@ -139,12 +132,12 @@ style conventions are in separate documents:
     **The gate builds write no image.** With `--output type=cacheonly` the phase
     runs and a failing step fails the build, but the result is not exported.
     Nothing uses those images, and writing one out is slow: a Go test phase's
-    image holds the toolchain. A build given neither `--output` nor `-t` writes
+    image holds the toolchain and every compiled package. A build given neither
-    an untagged image and leaves it dangling, on every developer host and every
+    `--output` nor `-t` writes an untagged image and leaves it dangling, on
-    CI runner. `script/cibuild` and `script/docker` build the image that ships
+    every developer host and every CI runner. `script/cibuild` and
-    and tag it, so each build replaces the previous image; each assigns the tag
+    `script/docker` build the image that ships and tag it, so each build
-    on its own line before the build, so `set -e` stops it where
+    replaces the previous image; each assigns the tag on its own line before the
-    `script/projectname` fails.
+    build, so `set -e` stops it where `script/projectname` fails.
 
     Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
     `eslint`, `prettier` — never through `make lint` or `script/test`, which are
@@ -159,42 +152,22 @@ style conventions are in separate documents:
     one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
     exit non-zero with `parallel golangci-lint is running` — a status a caller
     cannot tell from real findings. Both have produced wrong verdicts in this
-    org, in both directions. A container has its own `TMPDIR`, so the lock is
+    org, in both directions. A container has its own cache, its own `TMPDIR` and
-    out of reach. The lint phase does keep golangci-lint's cache between check
+    a digest-pinned binary, so neither is reachable.
-    builds (below), but it lints every checkout at `/src`, and a finding is
-    taken from that cache only for the same files, settings and golangci-lint
-    version.
 
-- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
+- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
-  invalidates a `COPY` layer only when the copied content changes, so on an
+  a `COPY` layer only when the copied content changes, so on an unchanged tree
-  unchanged tree the check `RUN` is served from cache, nothing executes, and the
+  the check `RUN` is served from cache, nothing executes, and the build still
-  build still exits 0. Every `docker build` in `script/` therefore passes
+  exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
-  `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
+  `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
-  id, so no two builds share a value, not even the three in one
+  four, and there is no fifth — `script/check` runs the two gate phases and
-  `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
+  `script/fmt-check`, and builds no image of its own. A bare `docker build .` is
-  `script/docker` are the four, and there is no fifth — `script/check` runs the
+  not evidence that anything ran: a sub-second build reporting success is a
-  two gate phases and `script/fmt-check`, and builds no image of its own. In
+  cache hit, not a result. Never invalidate by pruning — `docker builder prune`
-  every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
+  and friends destroy a build cache shared with every other build on the host.
-  that install system packages and download Go's modules, or right after `FROM`
+  When a check is added or changed, prove it works by planting a defect it must
-  in a stage with neither, and every step that lints, tests or compiles comes
+  catch and watching the run fail on it, then revert the defect. A green run
-  after it. A step that installs anything else as well, such as a
+  alone shows neither that the check ran nor that it covers what it should.
-  `script/bootstrap` that also installs node packages, comes after it too. A new
-  value reruns every `RUN` step after that line; `COPY` steps are compared by
-  content and need nothing. Docker keeps the layers above the line until the
-  base image, their own lines or the files they copy change. A stage without the
-  `ARG` line is served from cache on an unchanged tree, so a repository
-  re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
-  not pass `--no-cache`: it reruns every step too, but it also builds the layers
-  above the line again and empties every cache mount, so Go's build cache and
-  golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
-  survive between check builds, would start empty every time. A bare
-  `docker build .` is not evidence that anything ran: a sub-second build
-  reporting success is a cache hit, not a result. Never invalidate by pruning —
-  `docker builder prune` and friends destroy a build cache shared with every
-  other build on the host. When a check is added or changed, prove it works by
-  planting a defect it must catch and watching the run fail on it, then revert
-  the defect. A green run alone shows neither that the check ran nor that it
-  covers what it should.
 
 - **The gate phases are separate stages, and the build stage depends on both.**
   The lint phase is based on the `golangci/golangci-lint` image (pinned by
@@ -203,56 +176,40 @@ style conventions are in separate documents:
   `Dockerfile`:
 
     ```dockerfile
-    # In every stage ARG CACHEBUST comes right after the steps that install
-    # system packages and download Go's modules, or right after FROM in a
-    # stage with neither, and every step that lints, tests or compiles
-    # comes after it. script/ passes a new value on every build, which
-    # reruns every RUN step after it. Docker keeps the layers above it
-    # until the base image, their own lines or go.mod and go.sum change.
-    # The steps that compile mount Go's build cache, and golangci-lint run
-    # also mounts golangci-lint's; both survive from build to build.
-
     # Lint phase
     # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
     FROM golangci/golangci-lint@sha256:... AS lint
     WORKDIR /src
     COPY go.mod go.sum ./
     RUN go mod download
-    ARG CACHEBUST
     COPY . .
-    RUN --mount=type=cache,target=/root/.cache/go-build \
+    RUN golangci-lint run --config .golangci.yml ./...
-        --mount=type=cache,target=/root/.cache/golangci-lint \
-        golangci-lint run --config .golangci.yml ./...
 
     # Test phase. -race needs cgo and so a C compiler, which the Debian Go
-    # image ships and the alpine one does not. -count=1 because the mounted
+    # image ships and the alpine one does not.
-    # cache also holds test results, which go test would otherwise replay.
     # golang:1.x, YYYY-MM-DD
     FROM golang@sha256:... AS test
     WORKDIR /src
     COPY go.mod go.sum ./
     RUN go mod download
-    ARG CACHEBUST
     COPY . .
-    RUN --mount=type=cache,target=/root/.cache/go-build \
+    RUN go test -timeout 90s -race -cover ./... || \
-        go test -count=1 -timeout 90s -race -cover ./... || \
         { echo "--- Rerunning with -v for details ---"; \
-          go test -count=1 -timeout 90s -race -v ./...; exit 1; }
+          go test -timeout 90s -race -v ./...; exit 1; }
 
     # Build stage. Nothing is wanted from either phase above; the copies
-    # are what make BuildKit build them first, so nothing after them can
+    # are what make BuildKit build them first, so this stage cannot run
-    # run unless lint and test passed.
+    # unless lint and test passed.
     # golang:1.x-alpine, YYYY-MM-DD
     FROM golang@sha256:... AS builder
+    COPY --from=lint /src/go.sum /dev/null
+    COPY --from=test /src/go.sum /dev/null
     RUN apk add --no-cache git
+    # A tar-stream context keeps the sender's file owners, which git refuses.
+    RUN git config --system --add safe.directory /src
     WORKDIR /src
     COPY go.mod go.sum ./
     RUN go mod download
-    ARG CACHEBUST
-    COPY --from=lint /src/go.sum /dev/null
-    COPY --from=test /src/go.sum /dev/null
-    # A tar-stream context keeps the sender's file owners, which git refuses.
-    RUN git config --system --add safe.directory /src
     COPY . .
 
     # The VERSION build arg when one is given, otherwise
@@ -260,8 +217,7 @@ style conventions are in separate documents:
     # .git present, a version that is still empty, dev or unknown fails the
     # build: git is missing or could not read the checkout.
     ARG VERSION
-    RUN --mount=type=cache,target=/root/.cache/go-build \
+    RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
-        VERSION="${VERSION:-$(git describe --tags --always)}"; \
         if [ -e .git ]; then \
             case "$VERSION" in ""|dev|unknown) \
                 echo "version is '$VERSION' although .git is present" >&2; \
@@ -274,7 +230,6 @@ style conventions are in separate documents:
 
     # Runtime stage, and the last one
     FROM alpine@sha256:...
-    ARG CACHEBUST
     COPY --from=builder /app /usr/local/bin/app
     ENTRYPOINT ["app"]
     ```
@@ -289,26 +244,6 @@ style conventions are in separate documents:
     - Keep the runtime stage last, and if you add a stage after it, give it the
       same two copies. A plain `docker build .` builds the last stage's chain
       and nothing else.
-    - Check builds keep these, and none of them can hand back a stale result:
-        - The layers above `ARG CACHEBUST`: the system packages and Go's
-          modules. Docker installs the packages again when the base image or the
-          install line changes, and downloads the modules again when `go.mod` or
-          `go.sum` changes. Go checks every module against `go.sum` as it
-          downloads it, and no later build can change a kept layer. The modules
-          are kept in a layer rather than a cache mount because a cache mount is
-          shared by every build on the host, and Go builds from whatever files
-          it holds without checking them again (only `go mod verify` does), so a
-          file changed there would go unnoticed.
-        - Go's build cache (`/root/.cache/go-build`, where Go keeps it for
-          root), mounted on the steps that compile: `golangci-lint run`, both
-          `go test` runs and `go build`. Go reuses a compiled package only when
-          all its inputs are unchanged. The cache also holds test results, which
-          is why both `go test` runs carry `-count=1`.
-        - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
-          image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
-          a package's findings only when the package's files and those of every
-          package it imports, `go.mod`, the linters and their settings in
-          `.golangci.yml`, and the golangci-lint version are all unchanged.
     - If the project uses `//go:embed` directives that reference build artifacts
       (e.g. a web frontend compiled in a separate stage), the lint phase must
       create placeholder files so the embed directives resolve. Example:
@@ -317,20 +252,12 @@ style conventions are in separate documents:
       in the lint phase. The `golangci/golangci-lint` image is Debian-based and
       has no `apk`, so install with `apt-get` under the Debian package name
       (`libvips-dev`, where alpine says `vips-dev`), and delete the package
-      lists in the same `RUN`, so the layer does not keep them. The test phase,
+      lists in the same `RUN`, so the layer does not keep them:
-      on the Debian Go image, installs the libraries its tests need the same
-      way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
-      Docker keeps it until the base image or the install line changes:
 
         ```dockerfile
-        FROM golangci/golangci-lint@sha256:... AS lint
         RUN apt-get update \
             && apt-get install -y --no-install-recommends libvips-dev \
             && rm -rf /var/lib/apt/lists/*
-        WORKDIR /src
-        COPY go.mod go.sum ./
-        RUN go mod download
-        ARG CACHEBUST
         ```
 
     - `.dockerignore` lets `.git` into the build context. It keeps out every git
@@ -377,18 +304,18 @@ style conventions are in separate documents:
   `script/cibuild` bootstraps, runs the gate phases, and then builds the image,
   so a successful run means every check passed; a bare `docker build .` does not
   carry the same guarantee, because its gate phases may come from the cache. The
-  image build gets a `CACHEBUST` value of its own and so runs the gate phases a
+  image build is uncached and so runs the gate phases a second time. That is the
-  second time. That is the price of the rule above, and it is worth paying: the
+  price of the rule above, and it is worth paying: the image that ships is built
-  image that ships is built from a run of its own gates rather than from a cache
+  from a run of its own gates rather than from a cache entry. The `check` job
-  entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
+  sets `timeout-minutes: 20`, so a hung build frees the shared runner after 20
-  shared runner after 20 minutes. That allows for the three Docker builds
+  minutes. That allows for the three Docker builds described above (the test
-  described above (the test phase, the lint phase, then the image), each held to
+  phase, the lint phase, then the image), each held to the 5-minute Docker build
-  the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
+  limit below, plus the bootstrap. A separate workflow limited to `main` by a
-  limited to `main` by a `branches` list under `on: push` cannot be checked by
+  `branches` list under `on: push` cannot be checked by review: to try a change
-  review: to try a change to it, add the feature branch to that list and push,
+  to it, add the feature branch to that list and push, then remove the branch
-  then remove the branch from the list again before merging. Keep any job in it
+  from the list again before merging. Keep any job in it that publishes behind
-  that publishes behind `if: github.ref_name == 'main'`, so the run from the
+  `if: github.ref_name == 'main'`, so the run from the feature branch publishes
-  feature branch publishes nothing.
+  nothing.
 
 - Use platform-standard formatters: `black` for Python, `prettier` for
   JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -448,11 +375,12 @@ style conventions are in separate documents:
     That cache is Go's own, separate from Docker's layer cache. Go stores a
     passing result in its cache directory (`GOCACHE`), and when the same tests
     run again on unchanged code it prints that result, marked `(cached)`,
-    without running them. The directory lasts from one run to the next on a
+    without running them. That matters on a developer's machine, where this
-    developer's machine, where this target runs, and in the `test` phase of the
+    target runs and the directory lasts from one run to the next. The `test`
-    `Dockerfile`, which mounts it to keep compiled packages between check
+    phase of the `Dockerfile` needs no `-count=1`: its base image holds no
-    builds, so the phase's `go test` runs carry `-count=1` too. A new
+    result for this repo's tests and nothing before its `go test` step runs a
-    `CACHEBUST` value (above) is what makes that step run on an unchanged tree.
+    test, so there is nothing to replay. `--no-cache` (above) is what makes that
+    step run on an unchanged tree.
 
     Python example:
 
@@ -539,7 +467,7 @@ style conventions are in separate documents:
     version="$(git describe --tags --always --dirty 2>/dev/null || true)"
     [ -n "$version" ] || version="unknown"
     tag="$(script/projectname)"
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+    docker build --no-cache \
         --build-arg VERSION="$version" \
         -t "$tag" .
     ```
@@ -621,11 +549,6 @@ style conventions are in separate documents:
       function defined and never invoked has the same exit status and the same
       empty output as one that worked.
 
-    Node is the exception to the whole-token comparison: the canonical
-    `script/bootstrap` compares only its major version, because the node 22
-    alpine image the `Dockerfile` stages start from is not the exact pin, and
-    nvm cannot install a prebuilt node on alpine.
-
     Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
 
     A Go tool a repo needs on the host is installed with `go install` pinned to

TODO.md

@@ -29,208 +29,16 @@
 
 # Completed Steps
 
-- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
-  it. It leads with what the program is and how to use it. A reference section
-  covers every setting and every subcommand. Development and testing come last.
-  Every statement was checked against the code on `next`. Five comments in
-  `main_test.go` cite "Settings", where their rules now live. One comment each
-  in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
-  now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
-
-- an operand at or inside a directory named `.zfs` is never scanned, whether
-  given alone or under another operand. `scan` warns about it, counts it as
-  skipped and does not walk it (2026-10-26,
-  https://git.eeqj.de/sneak/sfdupes/issues/142)
-
-- a file or directory removed while the scan runs is not an error, in the walk
-  or in the hash phase. The scan prints no line for it and does not count it as
-  skipped. When the first of several hard links to a file is gone, the hash
-  phase reads the next one instead. On a terminal every progress display starts
-  with the spinner, the hash phase's included. A phase with an exact total ends
-  its line with its percent and a 10-column bar at the terminal's right edge,
-  which no longer moves as the line's length changes. A line too wide for the
-  terminal now loses its rate before its bar. The time remaining is labelled
-  `remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
-
-- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
-  bytes of a SHA-256, half the size of the hex text they held. The hashes stay
-  raw from the hash phase through the database to `report` and `trees`. Reading
-  a record whose hash is not 32 bytes is an error (2026-10-26,
-  https://git.eeqj.de/sneak/sfdupes/issues/134)
-
-- the hash phase gives every file the walk records three hashes: `head` and
-  `tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
-  of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
-  The 10 MiB and 50 MiB thresholds, the size census, the records stored without
-  hashes and the content phase are gone, so a scan has four phases and a rescan
-  reads only new and changed files. `info` no longer counts the records with
-  hashes, since every record has them (2026-10-25,
-  https://git.eeqj.de/sneak/sfdupes/issues/132)
-
-- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
-  `sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
-  stages of the `Dockerfile` the module download, and in `builder` the
-  system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
-  until the base image, the install line, `go.mod` or `go.sum` changes, and
-  every step that lints, tests or compiles still comes after it;
-  `golangci-lint run` also keeps golangci-lint's cache. README says the same
-  (2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
-
-- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
-  `go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
-  passed the build; README "Install" says so, says what `@main` and `v0.0.1`
-  give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
-  early build without a database. When the linker set no version,
-  `sfdupes --version` prints the main module's version that Go recorded in the
-  binary, the tag or the pseudo-version of a `go install` build, and `dev` only
-  when neither is known (2026-10-17,
-  https://git.eeqj.de/sneak/sfdupes/issues/116)
-
-- `scan` creates the database's directory where the database path leads: the
-  path with its last element cut off as text, never cleaned, so a `..` after a
-  symlinked directory is followed through the symlink, as when the file is
-  opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
-  symlink to `/data/x`, it creates `/data/new` and the database in it, where it
-  used to create `/srv/new` and then fail (2026-10-17,
-  https://git.eeqj.de/sneak/sfdupes/issues/119)
-
-- `scan` never records its own database: once the database and its `.lock` file
-  are open, it notes the device and inode of the database file, its `-wal` and
-  `-shm` files and its `.lock` file, and the walk passes over a file with one of
-  them wherever the operands reach it, through a symlinked directory included,
-  printing no line and counting nothing. A record an earlier scan left for one
-  of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
-
-- `scan` never records or reads anything that is not on a persistent filesystem
-  (`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
-  matched by the device ID of each such filesystem, listed as the scan starts
-  from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
-  however the walk or the content phase reaches it; no flag turns that off.
-  Below an operand, its walk does not enter the default excludes either, so
-  `sfdupes scan /` skips the kernel's view of the running system and the
-  temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
-  `/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
-  `/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
-  Windows list, not used yet. An operand at or inside a default exclude is still
-  walked. The scan prints a line for each path it leaves out, without counting
-  it as skipped, and the records there count as outside the scanned operands, as
-  for `-x`. `--no-default-excludes` turns off the default excludes only, and
-  `scan --help` lists them (2026-10-17,
-  https://git.eeqj.de/sneak/sfdupes/issues/124)
-
-- the rate a progress display shows covers the items completed over the last 5
-  seconds, or since the line before when stderr is not a terminal, so it falls
-  to 0 while nothing completes, and a burst after a wait no longer reads as a
-  steady rate of over 1,000 files a second; the ETA still comes from the phase's
-  average. Once no item has completed for 10 seconds, the word `stalled` and
-  where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
-  `db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
-  either, each phase giving its word where it is defined. A line too wide for
-  the terminal loses its bar first, then its rate or those words, so the phase,
-  progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
-  like those of README's example, the content phase's checks and reads lose
-  those words at most times in minutes, and every other phase keeps them
-  whenever the elapsed time and the ETA are both in minutes. The content phase's
-  last partial batch is now written under a display of its own (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/125)
-
-- new subcommand `info` prints the database's details on stdout, one per line as
-  a name, a tab and the value: its path and where it came from, the sizes of the
-  database file and of its `-wal` file when there is one, when the database file
-  was last changed, the schema version, the number of records, how many have
-  head and tail hashes and how many a content hash, and the sizes of all records
-  added up. It opens the database as `report` does, and shows two phases on
-  stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
-  from one read-only transaction, the counts from one query (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/117)
-
-- `report` shows its work on stderr in two phases, `[1/2] open` and
-  `[2/2] read`, with the progress display `scan` and `trees` use, from the
-  moment it starts until the summary line: `open` says it is opening the
-  database, also while it waits on a busy one; `read` counts the records, then
-  reads them against that count in signature order, which SQLite takes from the
-  `files_signature` index without sorting, so the first record comes at once.
-  `report` forms the duplicate groups itself, holding only those of the size it
-  is reading, and writes them ordered by first path before it reads the next
-  size. stdout and the summary line are unchanged, and SQLite no longer needs a
-  temporary file to sort in (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/114)
-
-- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
-  `[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
-  from the moment it starts until the summary line: `open` says it is opening
-  the database, also while it waits on a busy one; `read` counts the records,
-  then reads them against that count; `group` counts the directories it groups,
-  and `write` the groups it writes. A display's rate names what its phase
-  counts: files, records, directories or groups. stdout and the summary line are
-  unchanged. When stdout and stderr are the same file, as one terminal is, none
-  of stdout's lines shares a line with the display; when they are not, the
-  display keeps being drawn while stdout waits on a slow reader (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/113)
-
-- each `scan` phase's progress display names the phase and its place among the
-  five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
-  time and ETA: `load` counts the records first, the walk's total is estimated
-  from the records under the operands, and the content phase counts the records
-  it checks from SQLite's grouping alone, then says it is searching the database
-  until the first one is returned. On a terminal a phase with an exact total
-  draws its line followed by the library's bar, and any other a spinner followed
-  by the line; every display is drawn again several times a second. No line
-  reaches the terminal's last column, however narrow, also after the terminal is
-  made narrower during a phase: a line too wide leaves out its rate, then its
-  bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
-  only when those alone do not fit, as below about 70 columns late in a large
-  scan. In plain mode the line is printed every 5 seconds whether or not an item
-  completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
-
-- `scan`, `report` and `trees` first print a line on stderr naming the database
-  they use, as an absolute path, and whether that path is the default or comes
-  from `SFDUPES_DATABASE`; `scan` says when the database is new. README
-  "Database" says when `scan` writes each file (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/111)
-
 - re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
-  `e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
+  `c55a0cb`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
   that write no image, and `make test` runs the suite under the race detector,
   so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
-  gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
+  gone; every `docker build` in `script/` passes `--no-cache`; prettier runs on
-  every stage of the `Dockerfile` declares first, so every `RUN` step runs on
+  the host, from the node and yarn `script/bootstrap` installs, so the
-  every build, while the steps that compile keep Go's build cache and the tests
+  `prettier` and `markdown` stages are gone and the build stage installs `git`
-  run with `-count=1`; prettier runs on the host, from the node and yarn
+  and `make` itself; a new push cancels the workflow's older run on the same
-  `script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
+  branch, and a run stops after 20 minutes; `.claude/settings.json` is deleted
-  and the build stage installs `git` and `make` itself; a new push cancels the
+  (2026-10-08, https://git.eeqj.de/sneak/sfdupes/issues/95)
-  workflow's older run on the same branch, and a run stops after 20 minutes;
-  `.claude/settings.json` is deleted (2026-10-15,
-  https://git.eeqj.de/sneak/sfdupes/issues/95). Since
-  https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
-  `test` and the build stage, and the install of `git` and `make` in the build
-  stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
-  every step that lints, tests or compiles still runs on every build
-
-- with `-x`, an operand under another operand is walked on its own instead of
-  being dropped when a directory from it (from its directory, for a regular
-  file) up to but not including that operand is on a different filesystem than
-  that operand, whose walk then cannot reach it, and the records of files gone
-  from below it are deleted (2026-10-14,
-  https://git.eeqj.de/sneak/sfdupes/issues/106)
-
-- with `-x`, `scan` keeps the records below a directory on another filesystem
-  under its operand, and counts their sizes in the size census, instead of
-  deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
-
-- the hash and content phases open each file without blocking and check that it
-  is still a regular file, so one replaced by a FIFO or device node after it was
-  statted is warned about and skipped instead of hanging the scan (2026-10-14,
-  https://git.eeqj.de/sneak/sfdupes/issues/100)
-
-- the scan lock file is named from the database's path with symlinks resolved,
-  after `scan` creates the database file if it is missing, so a scan through a
-  symlink to the database fails while another scan holds the lock (2026-10-14,
-  https://git.eeqj.de/sneak/sfdupes/issues/101)
-
-- on a terminal, a warning for the last file of a phase no longer erases the
-  phase's finished bar: the bar is drawn again below the warning (2026-10-14,
-  https://git.eeqj.de/sneak/sfdupes/issues/102)
 
 - `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
   `mtime_nsec`, and compares it at that resolution, so a same-size rewrite
@@ -238,9 +46,8 @@
   https://git.eeqj.de/sneak/sfdupes/issues/12)
 
 - cut the narration from `TODO.md` Completed Steps and from the comments in
-  `script/`, `Dockerfile` and `Dockerfile.lint` (gone since
+  `script/` and both Dockerfiles; §Workflow now branches from and merges to
-  https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and
+  `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
-  merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
 
 - `make test-race` ran the test suite under the race detector in a cgo-enabled
   container, outside `make check` (2026-10-04,
@@ -254,7 +61,7 @@
   https://git.eeqj.de/sneak/sfdupes/issues/39). Since
   https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
   `REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
-  `script/` pass a new `CACHEBUST` value, so theirs always run
+  `script/` pass `--no-cache`, so theirs always run
 
 - `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
   it; all Markdown reformatted (2026-10-04,
@@ -369,46 +176,82 @@
 
 - fix the lint-image pin comments and `FROM` form in `Dockerfile` and
   `Dockerfile.lint` (2026-08-10, branch `next`, closes
-  https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy
+  https://git.eeqj.de/sneak/sfdupes/issues/25): both pins are now the policy
   `# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
-  without the false `(Debian-based)` note or the tag. Since
+  without the false `(Debian-based)` note or the tag; digest unchanged.
-  https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
+  `script/verify-lint-image-pin` still matches the tagless form, and a tag on
-  holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and
+  one side only is caught as a plain mismatch.
-  `script/verify-lint-image-pin`, which compared the two pins, are gone.
 
 - run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
   branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
-  owner ruling the linter is never installed on a host, and
+  owner ruling the linter is never installed on a host. `Dockerfile.lint` copies
-  `golangci-lint config verify` runs before `golangci-lint run`.
+  the repo into the digest-pinned `golangci/golangci-lint:v2.12.2` image and
-  `script/bootstrap` stopped installing or pinning the linter, and
+  runs `golangci-lint config verify` and `golangci-lint run` as build steps;
-  `script/verify-linter-pin` was retired. Since
+  `script/lint` builds it. `script/bootstrap` no longer installs or pins the
-  https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
+  linter, and warns rather than fails when `docker` is absent;
-  `Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build
+  `ENV PATH=/home/builder/go/bin:$PATH` went with its `go install`.
-  stage depends on through `COPY --from=lint /src/go.sum /dev/null`;
+  `script/verify-linter-pin` is retired; `script/verify-lint-image-pin`, a gate
-  `Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
+  in both files, compares their two `FROM` lines and restates neither pin.
-  an image build may run docker, so the phase calls `golangci-lint` directly.
+  Traps: nothing inside an image build may shell out to docker, so the
+  `Dockerfile` lint stage calls `golangci-lint` directly and the build stage
+  runs `make test` and `make fmt-check` instead of `make check`, through `make`
+  because the Makefile's `export CGO_ENABLED = 0` only reaches what it invokes.
+  `COPY --from=lint /src/go.sum /dev/null` replaces the copied linter binary as
+  the only edge making the build stage wait for lint; dropping it would end
+  fail-fast linting under a still-green build. `golangci-lint config verify`,
+  included per the ruling, validates from an embedded schema with no network
+  call, but `go mod download` above the gates still needs the network on a cold
+  cache. Verified: `make lint` green with no `golangci-lint` on `PATH`; two
+  back-to-back `script/lint` runs on an untouched tree both ran the linter
+  (27.7s and 28.7s in the lint step, `COPY . .` `CACHED` above); a planted
+  unused variable failed `script/lint`, and failed `make docker` at `[lint 9/9]`
+  with the build stage stopped at `[builder 3/12]`; the drift guard fails on a
+  tag-only, a digest-only and an unreadable reference, naming both sides; under
+  `--network none` config verify passes a valid config and rejects an invalid
+  one; `make docker` green in 5m35s with all six gates run (lint 37.6s, test
+  25.2s reporting `ok sneak.berlin/go/sfdupes 1.938s coverage: 88.5%`, not
+  `(cached)`); in the builder image with the Go test cache off, `--user 0:0`
+  still fails `TestScanHardlinkRunFailsTogether` where the unprivileged user
+  passes. Noted for follow-up, not fixed here: `golangci-lint` warns that
+  `gomodguard` is deprecated since v2.12.0 in favour of `gomodguard_v2`.
 
 - install the Docker build stage's prerequisites by running `script/bootstrap`
   instead of `apk add --no-cache make` inline (2026-08-09, branch
-  `dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42),
+  `dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42):
-  with `script/verify-linter-pin` failing the build unless the linter copied
+  the stage copies `script/` plus `go.mod`/`go.sum` and runs `script/bootstrap`,
-  from the lint stage was the version `script/bootstrap` pinned. Builds then
+  which ends in `go mod download`, so the separate call to it is gone.
-  took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as
+  `COPY --from=lint /usr/bin/golangci-lint` stays and moves above the bootstrap
-  https://git.eeqj.de/sneak/sfdupes/issues/43. Since
+  layer: it is the only edge making this stage depend on the lint stage, so
-  https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
+  deleting it would end fail-fast linting silently. A new
-  `make` with `apk add --no-cache` and only compiles; the `lint` and `test`
+  `script/verify-linter-pin`, run in the build stage before bootstrap, fails the
-  phases are the gates
+  build naming both versions unless that copied binary is the version
+  `script/bootstrap` pins; a pin it cannot read is a hard failure, not a skip.
+  `$GOPATH/bin` joins `PATH`, where bootstrap's `go install` lands. The `chown`
+  and `USER builder` still precede `make check`. Verified: the guard fails the
+  build with both versions named when the lint stage's linter is faked to
+  another version, and passes an unmodified build; bootstrap runs clean under
+  Alpine's `sh` and `apk`, finding the copied linter already at the pin; a
+  second build served the bootstrap and dependency layers `CACHED` while both
+  gates ran; a planted `unused` finding failed the build at the lint gate in
+  48.9s with the build stage's `make check` never starting; and the suite run in
+  the image as `--user 0:0` fails `TestScanHardlinkRunFailsTogether`, so the
+  drop to the unprivileged user is still needed. That last check needs the Go
+  test cache off: as root it first reported `ok ... (cached)`, reusing the
+  build-time result. Build times on a noisy shared host: 2m13s on an unchanged
+  tree, 2m17s and 4m29s after a source change, 5m14s cold, which breaches the
+  policy ceiling; `chown -R builder:builder /src /home/builder` walks the module
+  cache and alone varied from 77s to 210s across those builds, and `main`
+  measured 5m03s cold with a 209s `chown`. Filed as
+  https://git.eeqj.de/sneak/sfdupes/issues/43
 - bust the Docker layer cache for the gate steps, so `script/cibuild` and
   `script/docker` cannot report a green they did not earn (2026-08-09, branch
   `cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
   `Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
-  served them from cache and the build exited 0 having run nothing. The fix was
+  served them from cache and the build exited 0 having run nothing. Every
-  a `CHECK_EPOCH` build argument; since
+  `docker build` in `script/` now passes `--no-cache` instead
-  https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
+  (https://git.eeqj.de/sneak/sfdupes/issues/95). Run as root, the tests fail
-  passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
+  `TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
-  declares before every step that lints, tests or compiles. Run as root, the
+  the test relies on, so they run as an unprivileged user
-  tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
-  `chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
 - check the installed golangci-lint version in `script/bootstrap` instead of
   only its presence (2026-08-09, branch `bootstrap-version-check`, closes
   https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in

cancel_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -169,8 +168,8 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) {
 
 // TestSyncScanCancelledMidWalkKeepsRecords is the regression net under
 // the post-walk guard. The scan is cancelled part-way through the
-// walk, so it reaches the guard having walked a genuinely partial tree
+// walk, so it reaches the guard holding a genuinely partial size
-// and holding a still-populated index of records the walk never got to.
+// census and a still-populated index of records the walk never got to.
 // Every one of those records would look vanished to the update phase.
 // The guard is what stops the scan there, and this test is what
 // notices if it stops doing so: deleting the guard, or making it
@@ -197,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
 	base := baselineGoroutines(t)
 
 	st, err := syncScan(newWalkClock(walkCancelAtDone), db,
-		[]string{dir}, walkCancelWorkers, false, walkRules{})
+		[]string{dir}, walkCancelWorkers, false)
 
 	assertWalkGuardAborted(t, st, err)
 	assertRecordsIntact(t, db, before)
@@ -209,10 +208,10 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
 }
 
 // assertWalkGuardAborted checks that the scan stopped at the post-walk
-// guard: with a walk that reached some files (it really ran) but not
+// guard: with a census that is neither empty (the walk really ran)
-// all of them (it really was cut short), and with no record counted for
+// nor complete (it really was cut short), and with no record counted
-// removal. A removal count means the partial walk was carried past
+// for removal. A removal count means the partial census was carried
-// the guard into the update phase, which is the failure this test
+// past the guard into the update phase, which is the failure this test
 // exists to catch.
 func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
 	t.Helper()
@@ -223,30 +222,30 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
 	}
 
 	if st.unchanged == 0 {
-		t.Fatalf("stats = %+v: the walk reached no file, so it never "+
+		t.Fatalf("stats = %+v: the census is empty, so the walk never "+
 			"ran and the guard was reached for the wrong reason", st)
 	}
 
 	if st.unchanged >= walkCancelFiles {
-		t.Fatalf("stats = %+v: the walk reached the whole tree, so it "+
+		t.Fatalf("stats = %+v: the census covers the whole tree, so the "+
-			"was not cut short", st)
+			"walk was not cut short", st)
 	}
 
 	// The workers drop every directory still queued once the scan is
 	// cancelled, so only the files in the directories already in flight
-	// can still be reached after the fact. A walk beyond that bound
+	// can add to the census after the fact. A census beyond that bound
 	// would mean the cancellation was not observed where it should have
 	// been.
 	limit := walkCancelAtDone + walkCancelInFlightFiles
 	if st.unchanged > limit {
-		t.Errorf("the walk reached %d files, want at most %d: it kept "+
+		t.Errorf("census covers %d files, want at most %d: the walk kept "+
 			"taking directories off the queue after cancellation",
 			st.unchanged, limit)
 	}
 
 	if st.removed != 0 {
 		t.Errorf("stats = %+v: the scan counted records for removal from "+
-			"a partial walk", st)
+			"a partial census", st)
 	}
 }
 
@@ -269,8 +268,7 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	cancel()
 
-	st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false,
+	st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false)
-		walkRules{})
 	if !errors.Is(err, context.Canceled) {
 		t.Fatalf("syncScan on a cancelled context = %v, want %v",
 			err, context.Canceled)
@@ -303,7 +301,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) {
 	db := openTestDB(t)
 
 	st, err := syncScan(newWalkClock(hashCancelAtDone), db,
-		[]string{dir}, walkCancelWorkers, false, walkRules{})
+		[]string{dir}, walkCancelWorkers, false)
 	if !errors.Is(err, context.Canceled) {
 		t.Fatalf("syncScan cancelled mid-hash = %v, want %v",
 			err, context.Canceled)
@@ -346,10 +344,9 @@ func storedPaths(t *testing.T, path string) []string {
 
 // TestRunScanInterrupted calls the scan entrypoint with a context that
 // is already cancelled, as when a signal arrives at once. It must return
-// errInterrupted promptly, with its line on stderr after the one naming
+// errInterrupted promptly with its one line on stderr and nothing on
-// the database and nothing on stdout, leave the database valid and as
+// stdout, leave the database valid and as it was, and leave nothing in
-// it was, and leave nothing in the way of the next scan, which must
+// the way of the next scan, which must bring the database up to date.
-// bring the database up to date.
 func TestRunScanInterrupted(t *testing.T) {
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
@@ -358,8 +355,7 @@ func TestRunScanInterrupted(t *testing.T) {
 	stderr := captureStderr(t)
 	dir := buildSmokeTree(t)
 
-	err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
+	err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
-		false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -383,7 +379,7 @@ func TestRunScanInterrupted(t *testing.T) {
 		defer close(done)
 
 		err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
-			false, false)
+			false)
 	}()
 
 	awaitReturn(t, done, "runScan")
@@ -393,8 +389,7 @@ func TestRunScanInterrupted(t *testing.T) {
 			err, errInterrupted)
 	}
 
-	want := "scan: database " + path + " (from " + databaseEnv + ")\n" +
+	want := "scan: interrupted after 0 files\n"
-		"scan: interrupted after 0 files\n"
 	if got := stderr()[shown:]; got != want {
 		t.Errorf("stderr = %q, want %q", got, want)
 	}
@@ -410,8 +405,7 @@ func TestRunScanInterrupted(t *testing.T) {
 			got, before)
 	}
 
-	err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
+	err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
-		false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -437,7 +431,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
 	dir := buildWalkCancelTree(t)
 
 	err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
-		walkCancelWorkers, false, false)
+		walkCancelWorkers, false)
 	if !errors.Is(err, errInterrupted) {
 		t.Fatalf("runScan interrupted mid-hash = %v, want %v",
 			err, errInterrupted)
@@ -621,8 +615,7 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) {
 	// Unbuffered and unread: on a cancelled scan every send gives up.
 	events := make(chan walkEvent)
 
-	subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false,
+	subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events)
-		walkRules{}, events)
 	if len(subs) != 0 {
 		t.Errorf("cancelled walkOneDir returned %+v to descend into, "+
 			"want none", subs)
@@ -642,8 +635,7 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) {
 
 	missing := filepath.Join(t.TempDir(), "missing")
 
-	jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false,
+	jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false)
-		walkRules{})
 
 	for range 64 {
 		jobs <- dirJob{path: missing}
@@ -674,7 +666,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
-	jobs, subdirs, events := startWalkWorkers(ctx, 1, false, walkRules{})
+	jobs, subdirs, events := startWalkWorkers(ctx, 1, false)
 
 	// Fill the hand-back channel to its capacity — one slot per worker
 	// — so the worker's own hand-back is certain to block.
@@ -763,9 +755,7 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) {
 
 	var hashed atomic.Bool
 
-	hash := func(path string, size int64) (
+	hash := func(path string, size int64) (string, string, string, error) {
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-	) {
 		hashed.Store(true)
 
 		return hashSignature(path, size)
@@ -805,9 +795,7 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) {
 
 	// The scan is cancelled while the worker hashes, so the worker has
 	// already passed the check that drops queued runs.
-	hash := func(path string, size int64) (
+	hash := func(path string, size int64) (string, string, string, error) {
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-	) {
 		cancel()
 
 		return hashSignature(path, size)

db.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -12,7 +11,6 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
-	"strings"
 	"time"
 
 	"golang.org/x/sys/unix"
@@ -37,10 +35,6 @@ const schemaVersion = 1
 // scan.
 const dbDirPerm = 0o755
 
-// dbFilePerm is the mode for a database file created by scan, the mode
-// SQLite gives a database file it creates.
-const dbFilePerm = 0o644
-
 // lockFilePerm is the mode for the scan lock file. Anyone who can open
 // the file can hold the lock and keep every scan from running, so it
 // is open to its owner only.
@@ -49,22 +43,21 @@ const lockFilePerm = 0o600
 // createTableSQL is the schema applied to a fresh database. Paths are
 // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
 // holds whole Unix seconds and mtime_nsec the nanoseconds within that
-// second. head, tail and content each hold the raw 32 bytes of a
+// second.
-// SHA-256.
 const createTableSQL = `
 CREATE TABLE files (
     path       BLOB PRIMARY KEY,
     size       INTEGER NOT NULL,
     mtime      INTEGER NOT NULL,
     mtime_nsec INTEGER NOT NULL,
-    head       BLOB NOT NULL,
+    head       TEXT NOT NULL,
-    tail       BLOB NOT NULL,
+    tail       TEXT NOT NULL,
-    content    BLOB NOT NULL
+    content    TEXT NOT NULL
 ) WITHOUT ROWID
 `
 
-// createIndexSQL indexes the records by signature, so report can read
+// createIndexSQL indexes the records by signature, so report can have
-// them in signature order without sorting the table.
+// SQLite group them without sorting the whole table.
 const createIndexSQL = `
 CREATE INDEX files_signature ON files (size, head, tail, content)
 `
@@ -81,8 +74,7 @@ ON CONFLICT (path) DO UPDATE SET
     content = excluded.content
 `
 
-// errNoDatabase reports a missing database file for report, trees and
+// errNoDatabase reports a missing database file for report/trees.
-// info.
 var errNoDatabase = errors.New(
 	"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
 
@@ -94,57 +86,14 @@ var errSchemaVersion = errors.New("unsupported database schema version")
 // database.
 var errScanRunning = errors.New("another scan is running")
 
-// errHashSize reports a record whose head, tail or content is not the
-// 32 bytes of a SHA-256.
-var errHashSize = errors.New("hash is not 32 bytes")
-
 // databasePath resolves the database location: SFDUPES_DATABASE when
-// set and non-empty, the compiled-in default otherwise. It returns the
+// set and non-empty, the compiled-in default otherwise.
-// path, with the working directory, its symlinks resolved, put in front
+func databasePath() string {
-// of it when it is relative, and where the path came from, "default" or
-// "from SFDUPES_DATABASE", for the line each subcommand prints first on
-// stderr.
-func databasePath() (string, string, error) {
-	path, source := defaultDatabasePath, "default"
 	if p := os.Getenv(databaseEnv); p != "" {
-		path, source = p, "from "+databaseEnv
+		return p
 	}
 
-	if filepath.IsAbs(path) {
+	return defaultDatabasePath
-		return path, source, nil
-	}
-
-	// The working directory as the kernel reports it, with no symlink
-	// in it, as README "Database" says. os.Getwd may return PWD
-	// instead, which can name it through a symlink.
-	wd, err := unix.Getwd()
-	if err != nil {
-		return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
-	}
-
-	// Not filepath.Abs or filepath.Join: they clean the path, which
-	// drops each ".." together with the directory name before it. When
-	// that directory is a symlink, the cleaned path names another file.
-	// The working directory ends in "/" only when it is the root.
-	return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
-}
-
-// createDatabaseDirectory creates the directory the database at path
-// goes in, and any directory above it, if missing. path is absolute, as
-// databasePath returns it. The directory is path with its last element
-// cut off as text, by filepath.Split. filepath.Dir would clean it,
-// dropping each ".." together with the directory name before it; when
-// that name is a symlink, the cleaned path names another directory than
-// the one the system reaches when it opens path.
-func createDatabaseDirectory(path string) error {
-	dir, _ := filepath.Split(path)
-
-	err := os.MkdirAll(dir, dbDirPerm)
-	if err != nil {
-		return fmt.Errorf("create database directory: %w", err)
-	}
-
-	return nil
 }
 
 // scanParams are the connection parameters for scan: read-write, with
@@ -154,9 +103,9 @@ const scanParams = "_pragma=busy_timeout(10000)" +
 	"&_pragma=journal_mode(WAL)" +
 	"&_pragma=synchronous(NORMAL)"
 
-// reportParams are the connection parameters for report, trees and
+// reportParams are the connection parameters for report and trees:
-// info: read-only, with the same busy timeout. They set no journal
+// read-only, with the same busy timeout. They set no journal mode,
-// mode, because setting one is a write.
+// because setting one is a write.
 const reportParams = "mode=ro" +
 	"&_pragma=busy_timeout(10000)" +
 	"&_pragma=query_only(1)"
@@ -191,37 +140,19 @@ func openDB(path, params string) (*sql.DB, error) {
 
 // lockScanDatabase takes the lock that keeps a second scan off the
 // database at path: an exclusive flock(2) on the file beside it named
-// by the database's path with symlinks resolved and ".lock" appended,
+// path with ".lock" appended, created along with the database's parent
-// so every path to one database names the same lock file. If missing,
+// directory if missing. A lock held by another scan fails at once
-// the database's parent directory and an empty database file are
+// instead of waiting. The lock lasts until the returned file is closed
-// created first, so the lock file is named the same way before the
+// or the process ends. The file is never deleted: a scan that deleted
-// database is set up as after. A lock held by another scan fails at
+// it would let the next scan lock a new file while another still holds
-// once instead of waiting. The lock lasts until the returned file is
+// the old one.
-// closed or the process ends. The lock file is never deleted: a scan
-// that deleted it would let the next scan lock a new file while
-// another still holds the old one.
 func lockScanDatabase(path string) (*os.File, error) {
-	err := createDatabaseDirectory(path)
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create database directory: %w", err)
 	}
 
-	// Like SQLite, this follows a symlink at path and creates its
+	lockPath := path + ".lock"
-	// target. SQLite opens an empty file as a new database.
-	//nolint:gosec // the operator chooses the database path
-	dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm)
-	if err != nil {
-		return nil, fmt.Errorf("create database: %w", err)
-	}
-
-	_ = dbFile.Close()
-
-	resolved, err := filepath.EvalSymlinks(path)
-	if err != nil {
-		return nil, fmt.Errorf("resolve database path: %w", err)
-	}
-
-	lockPath := resolved + ".lock"
 
 	//nolint:gosec // the operator chooses the database path
 	f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, lockFilePerm)
@@ -247,9 +178,9 @@ func lockScanDatabase(path string) (*os.File, error) {
 // openScanDatabase opens the database for the scan subcommand, creating
 // the file, its parent directory, and the schema as needed.
 func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
-	err := createDatabaseDirectory(path)
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create database directory: %w", err)
 	}
 
 	db, err := openDB(path, scanParams)
@@ -267,42 +198,6 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
 	return db, nil
 }
 
-// databaseFiles returns, by device and inode, the files of the database
-// at path that the scan keeps open: the database file, its -wal and
-// -shm files and its .lock file, each named from the database's path
-// with symlinks resolved, as SQLite and lockScanDatabase name them. The
-// scan calls it once the database and the lock file are open, so that
-// its walk passes over them however it reaches them, through a
-// symlinked directory included. A file that does not exist then is left
-// out, since no walk can reach it. There is no -journal file while the
-// database is in WAL mode.
-func databaseFiles(path string) ([]fileRec, error) {
-	resolved, err := filepath.EvalSymlinks(path)
-	if err != nil {
-		return nil, fmt.Errorf("resolve database path: %w", err)
-	}
-
-	var files []fileRec
-
-	for _, p := range []string{
-		resolved, resolved + "-wal", resolved + "-shm", resolved + ".lock",
-	} {
-		fi, err := os.Lstat(p)
-		if errors.Is(err, fs.ErrNotExist) {
-			continue
-		}
-
-		if err != nil {
-			return nil, fmt.Errorf("database: %w", err)
-		}
-
-		dev, ino := inodeOfInfo(fi)
-		files = append(files, fileRec{path: p, dev: dev, ino: ino})
-	}
-
-	return files, nil
-}
-
 // closeScanDatabase switches the database at path from WAL back to
 // rollback-journal mode and closes it. Out of WAL mode the database
 // file alone holds the whole database, so a reader needs no -wal or
@@ -321,9 +216,9 @@ func closeScanDatabase(ctx context.Context, db *sql.DB, path string) {
 	_ = db.Close()
 }
 
-// openReportDatabase opens an existing database for the report, trees
+// openReportDatabase opens an existing database for the report and
-// and info subcommands. A missing database file is an error directing
+// trees subcommands. A missing database file is an error directing the
-// the user to run scan first; the schema version must match exactly.
+// user to run scan first; the schema version must match exactly.
 func openReportDatabase(ctx context.Context,
 	path string,
 ) (*sql.DB, error) {
@@ -462,32 +357,10 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
 	return v, nil
 }
 
-// loadFileRows gives total the number of records, then streams every
+// loadFileRows streams every record to fn in path order: byte order,
-// record to fn in path order: byte order, which is the order of the
+// which is the order of the primary key, so SQLite does not sort.
-// primary key, so SQLite does not sort. The count and the records are
+func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
-// read in one transaction, so they agree while a scan is committing.
+	rows, err := db.QueryContext(ctx,
-func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
-	fn func(r scanRec),
-) error {
-	// Everything goes through tx: db has one connection, so a query on
-	// db would wait for tx forever.
-	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
-	if err != nil {
-		return fmt.Errorf("read records: %w", err)
-	}
-
-	defer func() { _ = tx.Rollback() }()
-
-	var n int64
-
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
-	if err != nil {
-		return fmt.Errorf("count records: %w", err)
-	}
-
-	total(n)
-
-	rows, err := tx.QueryContext(ctx,
 		"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
 			"FROM files ORDER BY path")
 	if err != nil {
@@ -498,24 +371,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 
 	for rows.Next() {
 		var (
-			path, head, tail, content []byte
+			path      []byte
-			sec, nsec                 int64
+			sec, nsec int64
-			r                         scanRec
+			r         scanRec
 		)
 
-		err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
+			&r.content)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
 		r.path = string(path)
 		r.mtime = time.Unix(sec, nsec)
-
-		err = r.setHashes(head, tail, content)
-		if err != nil {
-			return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
-		}
-
 		fn(r)
 	}
 
@@ -527,166 +395,88 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 	return nil
 }
 
-// signatureRowsSQL selects every record's signature and path in
+// dupeRowsSQL selects every record in a duplicate group, with the
-// signature order, largest size first. That is the order of the
+// group's first path. A group is the records with a content hash that
-// files_signature index read backwards, so SQLite reads the index alone
+// share a size, head, tail, and content, when there are two or more of
-// and returns the first row at once, with nothing to sort.
+// them. The rows come in report order: groups by size descending, then
-const signatureRowsSQL = `
+// by first path, and each group's paths ascending.
-SELECT size, head, tail, content, path
+const dupeRowsSQL = `
-FROM files
+SELECT g.first, f.path, f.size
-ORDER BY size DESC, head DESC, tail DESC, content DESC
+FROM files AS f
+JOIN (
+    SELECT size, head, tail, content, MIN(path) AS first
+    FROM files
+    WHERE content <> ''
+    GROUP BY size, head, tail, content
+    HAVING COUNT(*) > 1
+) AS g USING (size, head, tail, content)
+ORDER BY f.size DESC, g.first, f.path
 `
 
-// loadSignatureRows gives total the number of records, then streams
+// loadDupeRows streams the rows of dupeRowsSQL to fn and returns the
-// every record's signature and path to fn, without its mtime, in the
+// number of records in the database. The count and the rows are read
-// order of signatureRowsSQL. The count and the records are read in one
+// in one transaction, so they agree while a scan is committing. An
-// transaction, so they agree while a scan is committing. An error from
+// error from fn stops the reading and is returned as it is.
-// fn stops the reading and is returned as it is.
+func loadDupeRows(ctx context.Context, db *sql.DB,
-func loadSignatureRows(ctx context.Context, db *sql.DB,
+	fn func(first, path string, size int64) error,
-	total func(n int64), fn func(r scanRec) error,
+) (int, error) {
-) error {
 	// Everything goes through tx: the report connection is the only
 	// one, so a query on db would wait for tx forever.
 	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
 	if err != nil {
-		return fmt.Errorf("read records: %w", err)
+		return 0, fmt.Errorf("read records: %w", err)
 	}
 
 	defer func() { _ = tx.Rollback() }()
 
-	var n int64
+	var records int
 
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records)
 	if err != nil {
-		return fmt.Errorf("count records: %w", err)
+		return 0, fmt.Errorf("read records: %w", err)
 	}
 
-	total(n)
+	rows, err := tx.QueryContext(ctx, dupeRowsSQL)
-
-	rows, err := tx.QueryContext(ctx, signatureRowsSQL)
 	if err != nil {
-		return fmt.Errorf("read records: %w", err)
+		return 0, fmt.Errorf("read records: %w", err)
 	}
 
 	defer func() { _ = rows.Close() }()
 
 	for rows.Next() {
 		var (
-			path, head, tail, content []byte
+			first, path []byte
-			r                         scanRec
+			size        int64
 		)
 
-		err = rows.Scan(&r.size, &head, &tail, &content, &path)
+		err = rows.Scan(&first, &path, &size)
 		if err != nil {
-			return fmt.Errorf("read record: %w", err)
+			return 0, fmt.Errorf("read record: %w", err)
 		}
 
-		r.path = string(path)
+		err = fn(string(first), string(path), size)
-
-		err = r.setHashes(head, tail, content)
 		if err != nil {
-			return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
+			return 0, err
-		}
-
-		err = fn(r)
-		if err != nil {
-			return err
 		}
 	}
 
 	err = rows.Err()
 	if err != nil {
-		return fmt.Errorf("read records: %w", err)
+		return 0, fmt.Errorf("read records: %w", err)
 	}
 
-	return nil
+	return records, nil
 }
 
-// setHashes sets r's head, tail and content to the values read from
+// loadFileMeta streams every record's path, size, mtime, and whether
-// those columns. Each must be the raw 32 bytes of a SHA-256.
+// it carries hashes to fn. Scan change detection needs no hash
-func (r *scanRec) setHashes(head, tail, content []byte) error {
+// values, and skipping the hash columns keeps the scan's in-memory
-	for _, b := range [][]byte{head, tail, content} {
+// index small on multi-million-file databases.
-		if len(b) != sha256.Size {
-			return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
-		}
-	}
-
-	r.head = [sha256.Size]byte(head)
-	r.tail = [sha256.Size]byte(tail)
-	r.content = [sha256.Size]byte(content)
-
-	return nil
-}
-
-// countRecords returns the number of records in the database.
-func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
-	var n int64
-
-	err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
-	if err != nil {
-		return 0, fmt.Errorf("count records: %w", err)
-	}
-
-	return n, nil
-}
-
-// databaseCountsSQL counts the records and adds up their sizes, in one
-// pass over the table. A SUM over no rows is NULL, so on an empty table
-// the SUM falls back to 0.
-const databaseCountsSQL = `
-SELECT COUNT(*), COALESCE(SUM(size), 0)
-FROM files
-`
-
-// databaseCounts is what info reads from the database: its schema
-// version and the counts of databaseCountsSQL.
-type databaseCounts struct {
-	schemaVersion int
-	records       int64 // every record
-	size          int64 // the sizes of all records added up
-}
-
-// readDatabaseCounts reads the schema version, then the counts of
-// databaseCountsSQL, in one read-only transaction, so both describe the
-// database at one moment while a scan is committing.
-func readDatabaseCounts(ctx context.Context,
-	db *sql.DB,
-) (databaseCounts, error) {
-	var c databaseCounts
-
-	// Everything goes through tx: the report connection is the only
-	// one, so a query on db would wait for tx forever.
-	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
-	if err != nil {
-		return c, fmt.Errorf("count records: %w", err)
-	}
-
-	defer func() { _ = tx.Rollback() }()
-
-	err = tx.QueryRowContext(ctx, "PRAGMA user_version").
-		Scan(&c.schemaVersion)
-	if err != nil {
-		return c, fmt.Errorf("read schema version: %w", err)
-	}
-
-	err = tx.QueryRowContext(ctx, databaseCountsSQL).
-		Scan(&c.records, &c.size)
-	if err != nil {
-		return c, fmt.Errorf("count records: %w", err)
-	}
-
-	return c, nil
-}
-
-// loadFileMeta streams every record's path, size and mtime to fn. Scan
-// change detection needs no hash values, and skipping the hash columns
-// keeps the scan's in-memory index small on multi-million-file
-// databases.
 func loadFileMeta(ctx context.Context, db *sql.DB,
-	fn func(path string, size int64, mtime time.Time),
+	fn func(path string, size int64, mtime time.Time, hashed bool),
 ) error {
 	rows, err := db.QueryContext(ctx,
-		"SELECT path, size, mtime, mtime_nsec FROM files")
+		"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -697,14 +487,74 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 		var (
 			path            []byte
 			size, sec, nsec int64
+			hashed          int64
 		)
 
-		err = rows.Scan(&path, &size, &sec, &nsec)
+		err = rows.Scan(&path, &size, &sec, &nsec, &hashed)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
-		fn(string(path), size, time.Unix(sec, nsec))
+		fn(string(path), size, time.Unix(sec, nsec), hashed != 0)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	return nil
+}
+
+// contentCandidatesSQL selects every record of at least headTailMin
+// bytes whose size, head, and tail equal another record's, in each
+// group (the records sharing a size, head, and tail) where at least one
+// record has no content hash, with whether each record has one. SQLite
+// does the grouping, so no other record's hashes are loaded into
+// memory; the rows come ordered by size, head, and tail, so each
+// group's rows arrive together.
+const contentCandidatesSQL = `
+SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
+FROM files AS f
+JOIN (
+    SELECT size, head, tail
+    FROM files
+    WHERE size >= ? AND head <> ''
+    GROUP BY size, head, tail
+    HAVING COUNT(*) > 1 AND SUM(content = '') > 0
+) AS g USING (size, head, tail)
+ORDER BY size, head, tail
+`
+
+// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
+// each record, without its content hash, and whether it has one.
+func loadContentCandidates(ctx context.Context, db *sql.DB,
+	fn func(r scanRec, hashed bool),
+) error {
+	rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var (
+			path      []byte
+			sec, nsec int64
+			r         scanRec
+			hashed    int64
+		)
+
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
+			&hashed)
+		if err != nil {
+			return fmt.Errorf("read record: %w", err)
+		}
+
+		r.path = string(path)
+		r.mtime = time.Unix(sec, nsec)
+		fn(r, hashed != 0)
 	}
 
 	err = rows.Err()
@@ -788,8 +638,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 
 	for _, r := range upserts {
 		_, err = st.ExecContext(ctx, []byte(r.path), r.size,
-			r.mtime.Unix(), r.mtime.Nanosecond(),
+			r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
-			r.head[:], r.tail[:], r.content[:])
 		if err != nil {
 			return fmt.Errorf("upsert %s: %w", r.path, err)
 		}

db_test.go

@@ -2,11 +2,9 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
-	"io/fs"
 	"os"
 	"path/filepath"
 	"slices"
@@ -15,19 +13,11 @@ import (
 	"time"
 )
 
-// testDBPath returns a database path inside a fresh temp dir, with
+// testDBPath returns a database path inside a fresh temp dir.
-// symlinks resolved as the scan lock resolves them, so the lock file
-// is this path plus ".lock" even where the temp dir is reached through
-// a symlink.
 func testDBPath(t *testing.T) string {
 	t.Helper()
 
-	dir, err := filepath.EvalSymlinks(t.TempDir())
+	return filepath.Join(t.TempDir(), "db.sqlite")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return filepath.Join(dir, "db.sqlite")
 }
 
 // openTestDB creates a fresh scan database in a temp dir.
@@ -45,47 +35,16 @@ func openTestDB(t *testing.T) *sql.DB {
 }
 
 func TestDatabasePath(t *testing.T) {
-	// README §Database: SFDUPES_DATABASE when set and non-empty, the
+	t.Setenv(databaseEnv, "")
-	// default otherwise, with the working directory, its symlinks
+
-	// resolved, put in front of a relative path. Nothing is cleaned, so
+	if got := databasePath(); got != defaultDatabasePath {
-	// a ".." stays.
+		t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
-	dir, err := filepath.EvalSymlinks(t.TempDir())
-	if err != nil {
-		t.Fatal(err)
 	}
 
-	t.Chdir(dir)
+	t.Setenv(databaseEnv, "/custom/place.sqlite")
 
-	fromEnv := "from " + databaseEnv
+	if got := databasePath(); got != "/custom/place.sqlite" {
-	cases := []struct {
+		t.Errorf("databasePath() = %q, want the env override", got)
-		env, path, source string
-	}{
-		{"", defaultDatabasePath, "default"},
-		{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
-		{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
-			fromEnv},
-		{"sub/place.sqlite", dir + "/sub/place.sqlite", fromEnv},
-		{"link/../place.sqlite", dir + "/link/../place.sqlite", fromEnv},
-	}
-
-	for _, tc := range cases {
-		t.Setenv(databaseEnv, tc.env)
-
-		path, source, err := databasePath()
-		if err != nil || path != tc.path || source != tc.source {
-			t.Errorf("databasePath() with %s=%q = %q, %q, %v; want %q, %q",
-				databaseEnv, tc.env, path, source, err, tc.path, tc.source)
-		}
-	}
-
-	// In the root directory the path gets one slash in front, not two.
-	t.Chdir("/")
-	t.Setenv(databaseEnv, "place.sqlite")
-
-	path, _, err := databasePath()
-	if err != nil || path != "/place.sqlite" {
-		t.Errorf("databasePath() in / = %q, %v; want %q",
-			path, err, "/place.sqlite")
 	}
 }
 
@@ -120,116 +79,6 @@ func TestOpenScanDatabaseCreates(t *testing.T) {
 	}
 }
 
-// TestScanCreatesDatabaseDirectoryAfterSymlink has a database path go
-// through a symlink to a directory, then "..", to a missing directory.
-// With srv/link a symlink to data/x, srv/link/../new/db.sqlite leads to
-// data/new/db.sqlite. lockScanDatabase and openScanDatabase, the two
-// places scan creates the database's directory, must each create the
-// database there, and nothing at srv/new, where the path leads once
-// cleaned as text.
-func TestScanCreatesDatabaseDirectoryAfterSymlink(t *testing.T) {
-	t.Parallel()
-
-	creates := map[string]func(ctx context.Context, path string) error{
-		"lockScanDatabase": func(_ context.Context, path string) error {
-			lock, err := lockScanDatabase(path)
-			if err == nil {
-				_ = lock.Close()
-			}
-
-			return err
-		},
-		"openScanDatabase": func(ctx context.Context, path string) error {
-			db, err := openScanDatabase(ctx, path)
-			if err == nil {
-				_ = db.Close()
-			}
-
-			return err
-		},
-	}
-
-	for name, create := range creates {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			dir, err := filepath.EvalSymlinks(t.TempDir())
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = os.MkdirAll(filepath.Join(dir, "data", "x"), 0o750)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = os.Mkdir(filepath.Join(dir, "srv"), 0o750)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = os.Symlink(filepath.Join(dir, "data", "x"),
-				filepath.Join(dir, "srv", "link"))
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = create(t.Context(), dir+"/srv/link/../new/db.sqlite")
-			if err != nil {
-				t.Fatalf("%s: %v", name, err)
-			}
-
-			_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
-			if err != nil {
-				t.Errorf("database the path leads to: %v", err)
-			}
-
-			_, err = os.Lstat(filepath.Join(dir, "srv", "new"))
-			if !errors.Is(err, fs.ErrNotExist) {
-				t.Errorf("srv/new: stat error %v, want it missing", err)
-			}
-		})
-	}
-}
-
-// TestDatabaseFiles opens a database through a symlink to its file:
-// databaseFiles finds the database file and its -wal and -shm files
-// beside the symlink's target, where SQLite keeps them, and leaves out
-// the .lock file, which only a scan creates.
-func TestDatabaseFiles(t *testing.T) {
-	t.Parallel()
-
-	path := testDBPath(t)
-	link := filepath.Join(t.TempDir(), "link.sqlite")
-
-	err := os.Symlink(path, link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db, err := openScanDatabase(t.Context(), link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = db.Close() }()
-
-	files, err := databaseFiles(link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	got := make([]string, 0, len(files))
-	for _, f := range files {
-		got = append(got, f.path)
-	}
-
-	want := []string{path, path + "-wal", path + "-shm"}
-	if !slices.Equal(got, want) {
-		t.Errorf("databaseFiles(%s) = %q, want %q", link, got, want)
-	}
-}
-
 func TestOpenDatabaseUnversionedForeign(t *testing.T) {
 	t.Parallel()
 
@@ -396,189 +245,12 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
 
 	defer func() { _ = reportDB.Close() }()
 
-	err = loadFileRows(t.Context(), reportDB, func(int64) {},
+	err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
-		func(scanRec) {})
 	if err != nil {
 		t.Fatalf("loadFileRows: %v", err)
 	}
 }
 
-// TestLoadRowsCountAgreeDuringScan has a scan commit records after
-// loadFileRows, as trees calls it, or loadSignatureRows, as report
-// calls it, has counted the records and before it reads them, as when
-// trees or report runs during a scan: each must read exactly as many
-// records as it counted.
-func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
-	t.Parallel()
-
-	loads := map[string]func(context.Context, *sql.DB, func(int64),
-		func(scanRec)) error{
-		"loadFileRows": loadFileRows,
-		"loadSignatureRows": func(ctx context.Context, db *sql.DB,
-			total func(int64), fn func(scanRec),
-		) error {
-			return loadSignatureRows(ctx, db, total, func(r scanRec) error {
-				fn(r)
-
-				return nil
-			})
-		},
-	}
-
-	for name, load := range loads {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			path := testDBPath(t)
-
-			scanDB, err := openScanDatabase(t.Context(), path)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			defer func() { _ = scanDB.Close() }()
-
-			recs := smokeTreeRecs()
-
-			err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			reportDB, err := openReportDatabase(t.Context(), path)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			defer func() { _ = reportDB.Close() }()
-
-			var counted, read int64
-
-			err = load(t.Context(), reportDB, func(n int64) {
-				counted = n
-
-				err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
-				if err != nil {
-					t.Error(err)
-				}
-			}, func(scanRec) { read++ })
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if counted != 3 || read != counted {
-				t.Errorf("counted %d records and read %d, want 3 and 3",
-					counted, read)
-			}
-		})
-	}
-}
-
-// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
-// content is not the 32 bytes of a SHA-256: either one byte, or the 64
-// bytes of a SHA-256 written as hex text. loadFileRows and
-// loadSignatureRows must each fail on it with an error naming its path.
-func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
-	t.Parallel()
-
-	hexText := strings.Repeat("ab", sha256.Size)
-
-	cases := []struct {
-		name   string
-		update string
-		value  any
-	}{
-		{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
-		{"hex-text head", "UPDATE files SET head = ?", hexText},
-		{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
-		{"hex-text tail", "UPDATE files SET tail = ?", hexText},
-		{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
-		{"hex-text content", "UPDATE files SET content = ?", hexText},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			t.Parallel()
-
-			db := openTestDB(t)
-
-			err := applyChanges(t.Context(), db,
-				[]scanRec{{path: "/a/f"}}, nil, nil)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			_, err = db.ExecContext(t.Context(), c.update, c.value)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			loads := map[string]func() error{
-				"loadFileRows": func() error {
-					return loadFileRows(t.Context(), db, func(int64) {},
-						func(scanRec) {})
-				},
-				"loadSignatureRows": func() error {
-					return loadSignatureRows(t.Context(), db, func(int64) {},
-						func(scanRec) error { return nil })
-				},
-			}
-
-			for name, load := range loads {
-				err := load()
-				if !errors.Is(err, errHashSize) ||
-					!strings.Contains(err.Error(), "/a/f") {
-					t.Errorf("%s: err = %v, want %v naming /a/f",
-						name, err, errHashSize)
-				}
-			}
-		})
-	}
-}
-
-// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
-// from the files_signature index alone, with nothing to sort: the
-// first record comes at once, so report's read phase counts each
-// record as it comes rather than waiting on SQLite for all of them.
-func TestSignatureRowsNeedNoSort(t *testing.T) {
-	t.Parallel()
-
-	db := openTestDB(t)
-
-	rows, err := db.QueryContext(t.Context(),
-		"EXPLAIN QUERY PLAN "+signatureRowsSQL)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = rows.Close() }()
-
-	var plan []string
-
-	for rows.Next() {
-		var (
-			id, parent, unused int
-			detail             string
-		)
-
-		err = rows.Scan(&id, &parent, &unused, &detail)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		plan = append(plan, detail)
-	}
-
-	err = rows.Err()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := []string{"SCAN files USING COVERING INDEX files_signature"}
-	if !slices.Equal(plan, want) {
-		t.Errorf("query plan = %q, want %q", plan, want)
-	}
-}
-
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()
 
@@ -589,20 +261,17 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// written.
 	recs := []scanRec{
 		{
-			size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
+			size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
-			tail: testHash("t2"), content: testHash("c2"),
+			content: "c2", path: "/a/tab\tnew\nline",
-			path: "/a/tab\tnew\nline",
 		},
 		{
-			size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
+			size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
-			tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
+			content: "c1", path: "/a/x",
 		},
 	}
 
-	prog := newProgress(progressParams{label: "update", total: 4})
+	err := applyChanges(t.Context(), db, recs, nil,
-	defer prog.finish()
+		newProgress("update", 2))
-
-	err := applyChanges(t.Context(), db, recs, nil, prog)
 	if err != nil {
 		t.Fatalf("applyChanges: %v", err)
 	}
@@ -616,12 +285,12 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// An upsert for an existing path updates in place; a delete
 	// removes exactly its path.
 	upd := scanRec{
-		size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
+		size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
-		tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
+		path: "/a/x",
 	}
 
 	err = applyChanges(t.Context(), db, []scanRec{upd},
-		[]string{"/a/tab\tnew\nline"}, prog)
+		[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
 	if err != nil {
 		t.Fatalf("applyChanges: %v", err)
 	}
@@ -644,15 +313,13 @@ func TestApplyChangesBatching(t *testing.T) {
 	recs := make([]scanRec, 0, n)
 	for i := range n {
 		recs = append(recs, scanRec{
-			size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
+			size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
-			tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
+			path: fmt.Sprintf("/batch/%07d", i),
 		})
 	}
 
-	prog := newProgress(progressParams{label: "update", total: int64(2 * n)})
+	err := applyChanges(t.Context(), db, recs, nil,
-	defer prog.finish()
+		newProgress("update", int64(n)))
-
-	err := applyChanges(t.Context(), db, recs, nil, prog)
 	if err != nil {
 		t.Fatalf("applyChanges: %v", err)
 	}
@@ -666,7 +333,8 @@ func TestApplyChangesBatching(t *testing.T) {
 		deletes = append(deletes, r.path)
 	}
 
-	err = applyChanges(t.Context(), db, nil, deletes, prog)
+	err = applyChanges(t.Context(), db, nil, deletes,
+		newProgress("update", int64(n)))
 	if err != nil {
 		t.Fatalf("applyChanges deletes: %v", err)
 	}

exclude.go

@@ -1,195 +0,0 @@
-package main
-
-// This file holds two rules for the walk (README "Rules for the walk"):
-// the default excludes, the paths a scan's walk does not enter below an
-// operand unless scan is given --no-default-excludes, and the
-// filesystems that are not persistent, on which the scan never records
-// or reads anything.
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"path/filepath"
-	"slices"
-	"strings"
-
-	"golang.org/x/sys/unix"
-)
-
-// The systems that have default excludes, as runtime.GOOS names them.
-const (
-	goosLinux   = "linux"
-	goosDarwin  = "darwin"
-	goosWindows = "windows"
-)
-
-// The fields of a line of mountInfoPath that readMountInfo reads,
-// counted from 0 (see proc(5)): the device ID, as major:minor, is the
-// third field, and after the sixth come optional fields, then a field
-// holding "-", then the filesystem type.
-const (
-	deviceField         = 2
-	optionalFieldsStart = 6
-)
-
-// typesPerHelpLine is how many filesystem types the scan command's help
-// lists on each line.
-const typesPerHelpLine = 6
-
-// errMountInfoLine reports a line of mountInfoPath without the fields
-// readMountInfo reads.
-var errMountInfoLine = errors.New("unexpected line")
-
-// walkRules holds the rules for the walk that a scan sets up as it
-// starts. excludes are the default excludes, or none with
-// --no-default-excludes. nonPersistent maps the device ID of every
-// mounted filesystem that is not persistent to its type; the scan
-// records and reads nothing on one of those, whatever the flags.
-// databaseFiles are the scan's own database files, which the walk
-// passes over (see databaseFiles in db.go).
-type walkRules struct {
-	excludes      []string
-	nonPersistent map[uint64]string
-	databaseFiles []fileRec
-}
-
-// defaultExcludes returns the default excludes of the system goos
-// names, as runtime.GOOS does: a fixed list on Linux, macOS and
-// Windows, and none on any other system.
-func defaultExcludes(goos string) []string {
-	switch goos {
-	case goosLinux:
-		return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
-	case goosDarwin:
-		// A walk of / reaches everything in /System/Volumes/Data through
-		// the firmlinks, so walking it as well would record each of
-		// those files twice.
-		return []string{
-			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
-			"/System/Volumes/VM", "/System/Volumes/Data",
-		}
-	case goosWindows:
-		// Relative to the root of the drive an operand lies on. The
-		// first three are the page, swap and hibernation files.
-		return []string{
-			"pagefile.sys", "swapfile.sys", "hiberfil.sys",
-			"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
-			`Users\*\AppData\Local\Temp`,
-		}
-	default:
-		return nil
-	}
-}
-
-// isDefaultExclude reports whether path is one of excludes, in which a
-// * matches one path component.
-func isDefaultExclude(path string, excludes []string) bool {
-	for _, pattern := range excludes {
-		// The patterns are fixed, so none is malformed.
-		if ok, _ := filepath.Match(pattern, path); ok {
-			return true
-		}
-	}
-
-	return false
-}
-
-// nonPersistentFilesystemTypes returns the filesystem types that are
-// not persistent: filesystems that the kernel makes up or that are kept
-// in memory. devfs is the macOS filesystem of /dev; the others are
-// Linux names.
-func nonPersistentFilesystemTypes() []string {
-	return []string{
-		"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs",
-		"cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore",
-		"bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs",
-		"binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", "devfs",
-	}
-}
-
-// nonPersistentLine returns the line the scan prints for path when fi,
-// its lstat, puts it on a filesystem in rules.nonPersistent, and ""
-// when it is on any other.
-func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string {
-	dev, ok := deviceOfInfo(fi)
-	if !ok {
-		return ""
-	}
-
-	fsType, found := rules.nonPersistent[dev]
-	if !found {
-		return ""
-	}
-
-	return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)",
-		path, fsType)
-}
-
-// readMountInfo reads mountinfo, laid out as mountInfoPath is, and
-// returns the device ID of every mounted filesystem that is not
-// persistent, with its type. The device ID is the one each line gives,
-// not that of the file now at its mount point, so a mount that a later
-// mount hides never adds the device of another filesystem.
-func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
-	types := nonPersistentFilesystemTypes()
-	devices := make(map[uint64]string)
-	lines := bufio.NewScanner(mountinfo)
-
-	for lines.Scan() {
-		fields := strings.Fields(lines.Text())
-
-		sep := slices.Index(fields, "-")
-		if sep < optionalFieldsStart || sep+1 >= len(fields) {
-			return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
-		}
-
-		var major, minor uint32
-
-		_, err := fmt.Sscanf(fields[deviceField], "%d:%d", &major, &minor)
-		if err != nil {
-			return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
-		}
-
-		fsType := fields[sep+1]
-		if slices.Contains(types, fsType) {
-			devices[unix.Mkdev(major, minor)] = fsType
-		}
-	}
-
-	return devices, lines.Err()
-}
-
-// defaultExcludesHelp describes, for the scan command's help, the
-// default excludes of the system goos names and, where the scan lists
-// the mounted filesystems, the types it never records anything on.
-func defaultExcludesHelp(goos string) string {
-	var help strings.Builder
-
-	dirs := defaultExcludes(goos)
-	if len(dirs) == 0 {
-		help.WriteString("This system has no default excludes.\n")
-	} else {
-		help.WriteString("Below an operand, the walk does not enter these " +
-			"default excludes, unless\n--no-default-excludes is given:\n\n")
-
-		for _, dir := range dirs {
-			fmt.Fprintf(&help, "  %s\n", dir)
-		}
-	}
-
-	if goos == goosLinux || goos == goosDarwin {
-		help.WriteString("\nWith or without --no-default-excludes, the scan " +
-			"records nothing on a\nfilesystem that is not persistent, " +
-			"one of these types:\n\n")
-
-		types := nonPersistentFilesystemTypes()
-		for line := range slices.Chunk(types, typesPerHelpLine) {
-			fmt.Fprintf(&help, "  %s\n", strings.Join(line, ", "))
-		}
-	}
-
-	return help.String()
-}

exclude_darwin.go

@@ -1,52 +0,0 @@
-//go:build darwin
-
-package main
-
-import (
-	"slices"
-
-	"golang.org/x/sys/unix"
-)
-
-// nonPersistentFilesystems returns the device ID of every mounted
-// filesystem that is not persistent, with its type, from the system's
-// list of mounted filesystems. The device ID is read with lstat on the
-// mount point. Where one filesystem is mounted over another on the same
-// mount point, the later one in the list counts. A mount point that
-// lstat cannot reach is left out.
-func nonPersistentFilesystems() (map[uint64]string, error) {
-	// The first call counts the filesystems, the second lists them.
-	n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT)
-	if err != nil {
-		return nil, err
-	}
-
-	list := make([]unix.Statfs_t, n)
-
-	n, err = unix.Getfsstat(list, unix.MNT_NOWAIT)
-	if err != nil {
-		return nil, err
-	}
-
-	mounts := make(map[string]string, n)
-
-	for _, st := range list[:n] {
-		mounts[unix.ByteSliceToString(st.Mntonname[:])] =
-			unix.ByteSliceToString(st.Fstypename[:])
-	}
-
-	types := nonPersistentFilesystemTypes()
-	devices := make(map[uint64]string)
-
-	for mountPoint, fsType := range mounts {
-		if !slices.Contains(types, fsType) {
-			continue
-		}
-
-		if dev, ok := deviceOfPath(mountPoint); ok {
-			devices[dev] = fsType
-		}
-	}
-
-	return devices, nil
-}

exclude_other.go

@@ -1,36 +0,0 @@
-//go:build !darwin
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"runtime"
-)
-
-// mountInfoPath is the file in which Linux lists the mounts the reading
-// process sees.
-const mountInfoPath = "/proc/self/mountinfo"
-
-// nonPersistentFilesystems returns the device ID of every mounted
-// filesystem that is not persistent, with its type: on Linux read from
-// mountInfoPath (see readMountInfo), and on any other system none.
-func nonPersistentFilesystems() (map[uint64]string, error) {
-	if runtime.GOOS != goosLinux {
-		return map[uint64]string{}, nil
-	}
-
-	f, err := os.Open(mountInfoPath)
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() { _ = f.Close() }()
-
-	devices, err := readMountInfo(f)
-	if err != nil {
-		return nil, fmt.Errorf("read %s: %w", mountInfoPath, err)
-	}
-
-	return devices, nil
-}

exclude_test.go

@@ -1,515 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"errors"
-	"maps"
-	"os"
-	"path/filepath"
-	"runtime"
-	"slices"
-	"strings"
-	"testing"
-
-	"golang.org/x/sys/unix"
-)
-
-func TestDefaultExcludes(t *testing.T) {
-	t.Parallel()
-
-	// README "Rules for the walk": the Linux and macOS lists, and none
-	// on any other system. The Windows list exists in the code only: the
-	// binary does not build for Windows, so the README leaves it out.
-	cases := map[string][]string{
-		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
-		goosDarwin: {
-			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
-			"/System/Volumes/VM", "/System/Volumes/Data",
-		},
-		goosWindows: {
-			"pagefile.sys", "swapfile.sys", "hiberfil.sys",
-			"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
-			`Users\*\AppData\Local\Temp`,
-		},
-		"freebsd": nil,
-	}
-
-	for goos, want := range cases {
-		if got := defaultExcludes(goos); !slices.Equal(got, want) {
-			t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want)
-		}
-	}
-}
-
-func TestIsDefaultExclude(t *testing.T) {
-	t.Parallel()
-
-	// A * matches one path component, no more and no fewer.
-	excludes := []string{"/scratch", "/home/*/cache"}
-
-	cases := map[string]bool{
-		"/scratch":        true,
-		"/scratch/x":      false,
-		"/home/u/cache":   true,
-		"/home/u/v/cache": false,
-		"/home/cache":     false,
-	}
-
-	for path, want := range cases {
-		if got := isDefaultExclude(path, excludes); got != want {
-			t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
-		}
-	}
-}
-
-func TestReadMountInfo(t *testing.T) {
-	t.Parallel()
-
-	// The second-to-last line has no optional fields. The tmpfs on
-	// /data/a/b is hidden by the root filesystem's /srv, mounted over
-	// /data/a after it: the tmpfs still gives its own device, and the
-	// root filesystem's device is not listed.
-	mountinfo := strings.Join([]string{
-		"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
-		"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
-		"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
-		"25 22 0:23 / /data/a/b rw shared:2 - tmpfs none rw",
-		"26 22 8:1 /srv /data/a rw - ext4 /dev/sda1 rw",
-		"27 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
-	}, "\n")
-
-	got, err := readMountInfo(strings.NewReader(mountinfo))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := map[uint64]string{
-		unix.Mkdev(0, 21): "proc", unix.Mkdev(0, 22): "sysfs",
-		unix.Mkdev(0, 23): "tmpfs", unix.Mkdev(0, 25): "autofs",
-	}
-	if !maps.Equal(got, want) {
-		t.Errorf("readMountInfo = %v, want %v", got, want)
-	}
-
-	for _, line := range []string{
-		"22 1 8:1 / / rw,relatime\n",
-		"22 1 sda / / rw,relatime shared:1 - ext4 /dev/sda1 rw\n",
-	} {
-		_, err = readMountInfo(strings.NewReader(line))
-		if !errors.Is(err, errMountInfoLine) {
-			t.Errorf("readMountInfo(%q) = %v, want %v", line, err,
-				errMountInfoLine)
-		}
-	}
-}
-
-func TestNonPersistentFilesystems(t *testing.T) {
-	t.Parallel()
-
-	if runtime.GOOS != goosLinux {
-		t.Skip("the test knows a mount only on Linux: /proc")
-	}
-
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dev, ok := deviceOfPath("/proc")
-	if !ok {
-		t.Fatal("no device for /proc")
-	}
-
-	if got := nonPersistent[dev]; got != "proc" {
-		t.Errorf("filesystem of /proc = %q, want %q", got, "proc")
-	}
-}
-
-func TestDefaultExcludesHelp(t *testing.T) {
-	t.Parallel()
-
-	for _, goos := range []string{goosLinux, goosDarwin, goosWindows} {
-		help := defaultExcludesHelp(goos)
-
-		for _, dir := range defaultExcludes(goos) {
-			if !strings.Contains(help, "\n  "+dir+"\n") {
-				t.Errorf("help for %s does not list %s:\n%s", goos, dir, help)
-			}
-		}
-
-		// Only Linux and macOS list their mounted filesystems.
-		for _, fsType := range nonPersistentFilesystemTypes() {
-			listed := strings.Contains(help, " "+fsType+",") ||
-				strings.Contains(help, " "+fsType+"\n")
-			if listed != (goos != goosWindows) {
-				t.Errorf("help for %s lists %s: %v", goos, fsType, listed)
-			}
-		}
-	}
-
-	want := "This system has no default excludes.\n"
-	if got := defaultExcludesHelp("freebsd"); got != want {
-		t.Errorf("help for freebsd = %q, want %q", got, want)
-	}
-
-	// scan's help shows those of the system it runs on.
-	var stdout, stderr bytes.Buffer
-
-	run([]string{cmdScan, "-h"}, &stdout, &stderr)
-
-	help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
-	if !strings.Contains(stderr.String(), help) {
-		t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help)
-	}
-}
-
-// TestScanDoesNotEnterDefaultExclude checks that the walk does not
-// enter a default exclude below an operand. It says so on a line of its
-// own and does not count it as skipped. The records below it count as
-// outside the operands: the record of a file gone from it is kept,
-// while a file added to it is not recorded.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	db := openTestDB(t)
-	excluded := filepath.Join(dir, "tmp")
-
-	// A scan with no default excludes records the file in excluded.
-	gone := writeFile(t, excluded, "gone", pattern(1, 100))
-	syncTree(t, db, dir)
-
-	err := os.Remove(gone)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	unseen := writeFile(t, excluded, "new", pattern(2, 10))
-
-	st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
-		walkRules{excludes: []string{excluded}})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.skipped != 0 {
-		t.Errorf("skipped = %d, want 0", st.skipped)
-	}
-
-	recs := dbRecords(t, db)
-	recordByPath(t, recs, gone)
-
-	if slices.Contains(recordPaths(recs), unseen) {
-		t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
-	}
-
-	line := "\nscan: skipped " + excluded + " (default exclude)\n"
-	if !strings.Contains(stderr(), line) {
-		t.Errorf("stderr = %q, want %q", stderr(), line)
-	}
-}
-
-// TestScanWalksOperandInDefaultExclude checks that an operand at or
-// inside a default exclude is walked, also when it lies under another
-// operand, whose walk does not enter the default exclude.
-func TestScanWalksOperandInDefaultExclude(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	excluded := filepath.Join(dir, "tmp")
-	f := writeFile(t, excluded, "sub/f", pattern(1, 10))
-	sub := filepath.Dir(f)
-
-	cases := map[string][]string{
-		"at it":                      {excluded},
-		"inside it":                  {sub},
-		"inside it under an operand": {dir, sub},
-	}
-
-	for name, roots := range cases {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			db := openTestDB(t)
-
-			st, err := syncScan(t.Context(), db, roots, 1, false,
-				walkRules{excludes: []string{excluded}})
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if st.added != 1 {
-				t.Errorf("added = %d, want %s recorded once", st.added, f)
-			}
-
-			recordByPath(t, dbRecords(t, db), f)
-		})
-	}
-}
-
-// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default
-// exclude given as an operand under another operand, which the walk of
-// that other operand does not enter, gets no line saying it was
-// skipped: it is walked as an operand of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	db := openTestDB(t)
-	excluded := filepath.Join(dir, "tmp")
-	f := writeFile(t, excluded, "f", pattern(1, 10))
-
-	st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
-		walkRules{excludes: []string{excluded}})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.added != 1 {
-		t.Errorf("added = %d, want %s recorded once", st.added, f)
-	}
-
-	recordByPath(t, dbRecords(t, db), f)
-
-	if strings.Contains(stderr(), "scan: skipped") {
-		t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
-	}
-}
-
-// TestRunScanTempDirOperand scans a tree in the tests' temporary
-// directory, which on Linux lies inside the default exclude /tmp unless
-// TMPDIR names another place, with the system's own default excludes:
-// the scan walks the whole tree and reports no default exclude.
-func TestRunScanTempDirOperand(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
-		t.Skip(dir + " is not inside a default exclude")
-	}
-
-	f := writeFile(t, dir, "sub/f", pattern(1, 10))
-
-	stderr := scanOK(t, dir)
-	if strings.Contains(stderr, "scan: skipped") {
-		t.Errorf("stderr = %q, want no line saying a path was skipped",
-			stderr)
-	}
-
-	assertRecorded(t, path, f, true)
-}
-
-// TestRunScanNoDefaultExcludes scans /var through the command. Its
-// /var/tmp, on a persistent filesystem, is a default exclude on Linux:
-// without --no-default-excludes the walk does not enter it and says so,
-// and with that flag the walk records a file the test puts there.
-func TestRunScanNoDefaultExcludes(t *testing.T) {
-	if runtime.GOOS != goosLinux {
-		t.Skip("only Linux has /var/tmp among its default excludes")
-	}
-
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	rules := walkRules{nonPersistent: nonPersistent}
-
-	for _, dir := range []string{"/var", "/var/tmp"} {
-		fi, lstatErr := os.Lstat(dir)
-		if lstatErr != nil || !fi.IsDir() ||
-			rules.nonPersistentLine(dir, fi) != "" {
-			t.Skip(dir + " is not a directory on a persistent filesystem here")
-		}
-	}
-
-	//nolint:usetesting // the directory must be inside /var/tmp
-	tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-")
-	if err != nil {
-		t.Skip("cannot write to /var/tmp here: " + err.Error())
-	}
-
-	t.Cleanup(func() { _ = os.RemoveAll(tmp) })
-
-	f := writeFile(t, tmp, "f", pattern(1, 10))
-
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	line := "scan: skipped /var/tmp (default exclude)\n"
-
-	if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
-		t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
-	}
-
-	assertRecorded(t, path, f, false)
-
-	stderr := scanOK(t, "--no-default-excludes", "/var")
-	if strings.Contains(stderr, line) {
-		t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q",
-			stderr, line)
-	}
-
-	assertRecorded(t, path, f, true)
-}
-
-// TestRunScanSkipsNonPersistentOperand scans, through the command, a
-// directory on a filesystem that is not persistent, with and without
-// --no-default-excludes: either way it is skipped, with a line saying
-// so, and nothing in it is recorded.
-func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
-	shm, fsType, _ := shmTempDir(t)
-	writeFile(t, shm, "f", pattern(1, 10))
-
-	t.Setenv(databaseEnv, testDBPath(t))
-
-	line := "scan: skipped " + shm + " (" + fsType +
-		", not a persistent filesystem)\n"
-	summary := "scan: 0 files seen"
-
-	for _, flags := range [][]string{nil, {"--no-default-excludes"}} {
-		stderr := scanOK(t, append(flags, shm)...)
-		if !strings.Contains(stderr, line) ||
-			!strings.Contains(stderr, summary) {
-			t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr,
-				line, summary)
-		}
-	}
-}
-
-// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
-// that is not persistent beside one on the tests' temporary directory.
-// The records an earlier scan left in it count as outside the operands:
-// the record of a file gone from it is kept, while a file added to it
-// is not recorded.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
-	shm, fsType, rules := shmTempDir(t)
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	db := openTestDB(t)
-
-	// A scan that, without the list of filesystems, records gone.
-	gone := writeFile(t, shm, "gone", pattern(1, 100))
-	syncTree(t, db, shm)
-
-	err := os.Remove(gone)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	unseen := writeFile(t, shm, "new", pattern(2, 10))
-
-	st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.skipped != 0 {
-		t.Errorf("skipped = %d, want 0", st.skipped)
-	}
-
-	recs := dbRecords(t, db)
-	recordByPath(t, recs, gone)
-
-	if slices.Contains(recordPaths(recs), unseen) {
-		t.Errorf("%s recorded", unseen)
-	}
-
-	line := "scan: skipped " + shm + " (" + fsType +
-		", not a persistent filesystem)\n"
-	if !strings.Contains(stderr(), line) {
-		t.Errorf("stderr = %q, want %q", stderr(), line)
-	}
-}
-
-// TestWalkLeavesOutNonPersistent starts the walk itself in a directory
-// on a filesystem that is not persistent, which a scan never does: it
-// drops such an operand first. Below a directory on a persistent
-// filesystem, one on another takes a mount, which a test cannot make.
-// The walk enters no directory and records no file there, and names
-// each.
-func TestWalkLeavesOutNonPersistent(t *testing.T) {
-	t.Parallel()
-
-	shm, fsType, rules := shmTempDir(t)
-	f := writeFile(t, shm, "f", pattern(1, 10))
-	sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10)))
-
-	var leftOut []string
-
-	for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) {
-		if ev.rec.path != "" {
-			t.Errorf("walk recorded %s", ev.rec.path)
-		}
-
-		if ev.leftOut == "" {
-			continue
-		}
-
-		leftOut = append(leftOut, ev.leftOut)
-
-		want := "scan: skipped " + ev.leftOut + " (" + fsType +
-			", not a persistent filesystem)"
-		if ev.warn != want {
-			t.Errorf("line = %q, want %q", ev.warn, want)
-		}
-	}
-
-	slices.Sort(leftOut)
-
-	if want := []string{f, sub}; !slices.Equal(leftOut, want) {
-		t.Errorf("walk left out %q, want %q", leftOut, want)
-	}
-}
-
-// shmTempDir returns a new directory in /dev/shm, removed when the test
-// ends, the type of its filesystem, and the rules holding the real
-// filesystems that are not persistent. It skips the test where /dev/shm
-// is on a persistent filesystem, or cannot be written to.
-func shmTempDir(t *testing.T) (string, string, walkRules) {
-	t.Helper()
-
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dev, ok := deviceOfPath("/dev/shm")
-
-	fsType, found := nonPersistent[dev]
-	if !ok || !found {
-		t.Skip("/dev/shm is not a memory-backed filesystem here")
-	}
-
-	//nolint:usetesting // the directory must be inside /dev/shm
-	dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-")
-	if err != nil {
-		t.Skip("cannot write to /dev/shm here: " + err.Error())
-	}
-
-	t.Cleanup(func() { _ = os.RemoveAll(dir) })
-
-	return dir, fsType, walkRules{nonPersistent: nonPersistent}
-}
-
-// assertRecorded checks that the database at dbPath holds a record for
-// path when want is set, and none when it is not.
-func assertRecorded(t *testing.T, dbPath, path string, want bool) {
-	t.Helper()
-
-	db, err := openDB(dbPath, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = db.Close() }()
-
-	if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want {
-		t.Errorf("%s recorded: %v, want %v", path, got, want)
-	}
-}

go.mod

@@ -24,5 +24,3 @@ require (
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 )
-
-retract v0.0.1 // early build without a database; does not work as the README describes

info.go

@@ -1,125 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-)
-
-// infoPhase names one phase of info with its place among all of them,
-// as "[2/2] count", for the phase's progress display. The list here is
-// the one place the phases are counted.
-func infoPhase(name string) string {
-	return phaseLabel([]string{openPhase, "count"}, name)
-}
-
-// runInfo implements the info subcommand: it prints the database's
-// details on stdout (see writeInfo). It opens the database as report
-// does, read-only and without the scan lock, and never touches the
-// scanned filesystem. It names the database on stderr before it opens
-// it. Any database problem, including a missing database, is fatal.
-//
-// The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and count (count the records,
-// with one query). Each display ends before the next phase starts, and
-// before an error is returned to be printed. The details are written
-// once the last display has ended, so no display is drawn while they
-// are written.
-func runInfo(ctx context.Context, stdout io.Writer) error {
-	dbPath, source, err := databasePath()
-	if err != nil {
-		return err
-	}
-
-	fmt.Fprintf(os.Stderr, "info: database %s (%s)\n",
-		escapePath(dbPath), source)
-
-	// Opening can wait on a busy database for as long as the busy
-	// timeout.
-	prog := newProgress(progressParams{
-		label: infoPhase(openPhase), total: -1,
-		waitingFor: openingDatabase,
-	})
-	db, err := openReportDatabase(ctx, dbPath)
-
-	prog.finish()
-
-	if err != nil {
-		return err
-	}
-
-	defer func() { _ = db.Close() }()
-
-	// One query counts everything, so the phase never has a total.
-	prog = newProgress(progressParams{
-		label: infoPhase("count"), total: -1,
-		waitingFor: countingRecords,
-	})
-	counts, err := readDatabaseCounts(ctx, db)
-
-	prog.finish()
-
-	if err != nil {
-		return fmt.Errorf("database %s: %w", dbPath, err)
-	}
-
-	return writeInfo(stdout, dbPath, source, counts)
-}
-
-// writeInfo writes the details of the database at dbPath to stdout,
-// one per line, as a name, a tab and the value (README "`info` mode"):
-// the path and source as databasePath returns them, the sizes of the
-// database file and of its -wal file when there is one, when the
-// database file was last changed, and the schema version and record
-// counts that readDatabaseCounts read.
-func writeInfo(stdout io.Writer, dbPath, source string,
-	counts databaseCounts,
-) error {
-	// SQLite keeps the -wal file beside the database file, named from
-	// its path with symlinks resolved.
-	file, err := filepath.EvalSymlinks(dbPath)
-	if err != nil {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	dbFile, err := os.Stat(file)
-	if err != nil {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	walFile, err := os.Stat(file + "-wal")
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	var b strings.Builder
-
-	fmt.Fprintf(&b, "database\t%s (%s)\n", escapePath(dbPath), source)
-	fmt.Fprintf(&b, "database file size\t%d (%s)\n",
-		dbFile.Size(), humanBytes(dbFile.Size()))
-
-	if walFile != nil {
-		fmt.Fprintf(&b, "-wal file size\t%d (%s)\n",
-			walFile.Size(), humanBytes(walFile.Size()))
-	}
-
-	fmt.Fprintf(&b, "modified\t%s\n",
-		dbFile.ModTime().UTC().Format(time.RFC3339))
-	fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
-	fmt.Fprintf(&b, "records\t%d\n", counts.records)
-	fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
-		counts.size, humanBytes(counts.size))
-
-	_, err = io.WriteString(stdout, b.String())
-	if err != nil {
-		return fmt.Errorf("write stdout: %w", err)
-	}
-
-	return nil
-}

info_test.go

@@ -1,166 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-)
-
-// infoOpenLine is the plain line of info's open phase when it opens the
-// database at once; it prints it as the phase starts and as it ends.
-const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
-	"remain unknown"
-
-// infoCountLine is the plain line of info's count phase when the count
-// takes under a second; it prints it as the phase starts and as it
-// ends.
-const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
-	"remain unknown"
-
-// infoRecs are four records: a duplicate pair, a file of 20 MiB and
-// another small file.
-func infoRecs() []scanRec {
-	h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u")
-	bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc")
-
-	return []scanRec{
-		{size: 5, head: h, tail: tl, content: c, path: "/d/a"},
-		{size: 5, head: h, tail: tl, content: c, path: "/d/b"},
-		{size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"},
-		{size: 7, head: u, tail: u, content: u, path: "/d/unique"},
-	}
-}
-
-// TestRunInfo runs info on an empty and on a filled database, with
-// stdout and stderr the same file that is not a terminal, and checks
-// everything written there: the line naming the database, each phase's
-// line as it starts and as it ends, and then the details, each on a
-// line of its own. No -wal file is beside a database scan has closed.
-func TestRunInfo(t *testing.T) {
-	cases := map[string]struct {
-		recs []scanRec
-		want string // the details after the database file's time
-	}{
-		"empty": {
-			want: "schema version\t1\n" +
-				"records\t0\n" +
-				"size of recorded files\t0 (0 B)\n",
-		},
-		"filled": {
-			recs: infoRecs(),
-			want: "schema version\t1\n" +
-				"records\t4\n" +
-				"size of recorded files\t20971537 (20.0 MiB)\n",
-		},
-	}
-
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			path := seedDatabase(t, tc.recs)
-			t.Setenv(databaseEnv, path)
-
-			file, err := os.Stat(path)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			stderr := captureStderr(t)
-
-			code := run([]string{cmdInfo}, os.Stderr, os.Stderr)
-			if code != exitOK {
-				t.Fatalf("run(info) = %d, want %d; stderr: %s",
-					code, exitOK, stderr())
-			}
-
-			named := path + " (from " + databaseEnv + ")\n"
-			want := "info: database " + named +
-				infoOpenLine + "\n" + infoOpenLine + "\n" +
-				infoCountLine + "\n" + infoCountLine + "\n" +
-				"database\t" + named +
-				fmt.Sprintf("database file size\t%d (%s)\n",
-					file.Size(), humanBytes(file.Size())) +
-				"modified\t" + file.ModTime().UTC().Format(time.RFC3339) +
-				"\n" + tc.want
-
-			if got := stderr(); got != want {
-				t.Errorf("stdout and stderr = %q, want %q", got, want)
-			}
-		})
-	}
-}
-
-// TestRunInfoDuringScan runs info while a scan holds the lock and has
-// the database open in WAL mode, with records committed to the -wal
-// file only. info must count those records and show the -wal file's
-// size, also when the database path is a symlink: SQLite keeps the
-// -wal file beside the symlink's target.
-func TestRunInfoDuringScan(t *testing.T) {
-	path := testDBPath(t)
-	link := filepath.Join(t.TempDir(), "link.sqlite")
-
-	err := os.Symlink(path, link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db, err := openScanDatabase(t.Context(), path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	err = applyChanges(t.Context(), db, infoRecs(), nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	holdScanLock(t, path)
-
-	wal, err := os.Stat(path + "-wal")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, named := range []string{path, link} {
-		t.Run(filepath.Base(named), func(t *testing.T) {
-			t.Setenv(databaseEnv, named)
-
-			stdout := runStdout(t, cmdInfo)
-
-			for _, want := range []string{
-				"database\t" + named + " (from " + databaseEnv + ")\n",
-				fmt.Sprintf("\n-wal file size\t%d (%s)\n",
-					wal.Size(), humanBytes(wal.Size())),
-				"\nrecords\t4\n",
-			} {
-				if !strings.Contains(stdout, want) {
-					t.Errorf("stdout = %q, want it to hold %q", stdout, want)
-				}
-			}
-		})
-	}
-}
-
-func TestRunInfoNeedsOnlyReadAccess(t *testing.T) {
-	// README "`info` mode": like report, info needs only read access to
-	// the database file and none to its directory.
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	scanFixture(t)
-	makeReadOnly(t, path)
-
-	stdout := runStdout(t, cmdInfo)
-
-	if want := "\nrecords\t2\n"; !strings.Contains(stdout, want) {
-		t.Errorf("stdout = %q, want it to hold %q", stdout, want)
-	}
-
-	if strings.Contains(stdout, "-wal") {
-		t.Errorf("stdout = %q, want no -wal file between scans", stdout)
-	}
-}

main.go

@@ -1,19 +1,20 @@
 // Command sfdupes quickly identifies candidate duplicate files across
 // very large filesystems without reading every byte of every file.
 // Files are considered duplicates when their sizes are equal and they
-// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
+// agree on a short ladder of SHA-256 hashes. A file under 10 MiB is
-// KiB, and of the content. The content hash covers the whole file under
+// hashed in full. A larger file is compared on the hashes of its first
-// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
+// and last 64 KiB, and only when those match another file's is its
-// file is never read in full. scan maintains a persistent SQLite
+// content hash computed and compared: of the whole file when it is
-// database of file signatures (SFDUPES_DATABASE, default
+// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
-// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
+// or larger. scan maintains a persistent SQLite database of file
+// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
+// that the reporting subcommands read.
 //
 // Usage:
 //
-//	sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
+//	sfdupes scan [--workers N] [-x] PATH...
 //	sfdupes report > dupes.tsv
 //	sfdupes trees > dupetrees.tsv
-//	sfdupes info
 //	sfdupes --version
 //
 // See README.md for the complete specification.
@@ -26,7 +27,6 @@ import (
 	"io"
 	"os"
 	"runtime"
-	"runtime/debug"
 
 	"github.com/spf13/cobra"
 )
@@ -44,7 +44,6 @@ const (
 	cmdScan   = "scan"
 	cmdReport = "report"
 	cmdTrees  = "trees"
-	cmdInfo   = "info"
 )
 
 // errNoSubcommand is returned by the root command when it is invoked
@@ -57,32 +56,10 @@ var errNoSubcommand = errors.New("no subcommand")
 var errWorkersBelowOne = errors.New("--workers must be at least 1")
 
 // Version is the build version, injected at link time via -ldflags
-// (see the Makefile); empty when the linker set none, as for a binary
+// (see the Makefile); "dev" for a plain go build.
-// built by go install. buildVersion turns it into what --version prints.
 //
 //nolint:gochecknoglobals // written only by the linker
-var Version string
+var Version = "dev"
-
-// buildVersion returns the version that sfdupes --version prints
-// (README "Install"): the version the linker set, when it set one;
-// otherwise the main module's version that Go recorded in the binary,
-// a tag such as v0.0.2 or the pseudo-version of an @main build; and
-// "dev" when neither is known. info is what debug.ReadBuildInfo
-// returns, nil when the binary carries none. Go records "(devel)" for
-// a build that has no version, such as one inside a copy of the source
-// without git.
-func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
-	if linkerVersion != "" {
-		return linkerVersion
-	}
-
-	if info == nil || info.Main.Version == "" ||
-		info.Main.Version == "(devel)" {
-		return "dev"
-	}
-
-	return info.Main.Version
-}
 
 func main() {
 	// Once the reader of a stdout pipe has gone, as in "sfdupes report |
@@ -96,8 +73,8 @@ func main() {
 // exit code. It is the program's single exit point: the subcommands
 // return their errors instead of exiting, so every deferred cleanup —
 // above all closing the database, which checkpoints the SQLite WAL —
-// runs before the process ends. The report, trees and info subcommands
+// runs before the process ends. The report and trees subcommands write
-// write their data to stdout.
+// their data to stdout.
 func run(args []string, stdout, stderr io.Writer) int {
 	// A nil slice makes cobra fall back to os.Args, which would let a
 	// test binary's own flags reach the command tree.
@@ -137,10 +114,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 	var showVersion bool
 
 	printVersion := runE(func(context.Context, []string) error {
-		info, _ := debug.ReadBuildInfo()
+		_, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version)
-
-		_, err := fmt.Fprintf(stdout, "sfdupes %s\n",
-			buildVersion(Version, info))
 		if err != nil {
 			return fmt.Errorf("write stdout: %w", err)
 		}
@@ -174,6 +148,30 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 	root.Flags().BoolVarP(&showVersion, "version", "v", false,
 		"print the version to stdout")
 
+	var (
+		scanWorkers int
+		scanOneFS   bool
+	)
+
+	scanCmd := &cobra.Command{
+		Use:   cmdScan + " [--workers N] [-x] PATH...",
+		Short: "Walk trees and synchronize the scan database",
+		Args:  cobra.MinimumNArgs(1),
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			return checkScanWorkers(cmd, scanWorkers)
+		},
+		RunE: runE(func(ctx context.Context, args []string) error {
+			ctx, stop := interruptContext(ctx)
+			defer stop()
+
+			return runScan(ctx, args, scanWorkers, scanOneFS)
+		}),
+	}
+	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
+		"concurrent workers for the walk, hash, and content phases")
+	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
+		"do not cross filesystem boundaries")
+
 	reportCmd := &cobra.Command{
 		Use:   cmdReport,
 		Short: "Read the scan database and print the file-level duplicates report",
@@ -192,59 +190,11 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 		}),
 	}
 
-	infoCmd := &cobra.Command{
+	root.AddCommand(scanCmd, reportCmd, treesCmd)
-		Use:   cmdInfo,
-		Short: "Read the scan database and print its details",
-		Args:  cobra.NoArgs,
-		RunE: runE(func(ctx context.Context, _ []string) error {
-			return runInfo(ctx, stdout)
-		}),
-	}
-
-	root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
 
 	return root
 }
 
-// newScanCommand builds the scan subcommand, with its --workers, -x and
-// --no-default-excludes flags. Its help lists the default excludes of
-// the system it runs on, and the filesystem types it never records
-// anything on.
-func newScanCommand() *cobra.Command {
-	var (
-		scanWorkers           int
-		scanOneFS             bool
-		scanNoDefaultExcludes bool
-	)
-
-	short := "Walk trees and synchronize the scan database"
-
-	scanCmd := &cobra.Command{
-		Use:   cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
-		Short: short,
-		Long:  short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
-		Args:  cobra.MinimumNArgs(1),
-		PreRunE: func(cmd *cobra.Command, _ []string) error {
-			return checkScanWorkers(cmd, scanWorkers)
-		},
-		RunE: runE(func(ctx context.Context, args []string) error {
-			ctx, stop := interruptContext(ctx)
-			defer stop()
-
-			return runScan(ctx, args, scanWorkers, scanOneFS,
-				scanNoDefaultExcludes)
-		}),
-	}
-	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
-		"concurrent workers for the walk and hash phases")
-	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
-		"do not cross filesystem boundaries")
-	scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
-		false, "enter the default excludes, the paths listed above")
-
-	return scanCmd
-}
-
 // checkScanWorkers rejects a scan --workers value below 1. That is a
 // usage error reported in one line: cobra prints the returned message
 // without the usage text, and run exits 2.

main_test.go

@@ -4,12 +4,10 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
-	"runtime/debug"
 	"slices"
 	"strconv"
 	"strings"
@@ -20,36 +18,6 @@ import (
 // error prints and a runtime failure must not.
 const usageMarker = "Usage:"
 
-// TestMain stops the tests at once, saying why, when their temporary
-// directory is not on a persistent filesystem: scan records nothing
-// there (README "Rules for the walk"), so most scan tests would fail
-// without saying why.
-func TestMain(m *testing.M) {
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "list the mounted filesystems: %v\n", err)
-		os.Exit(1)
-	}
-
-	dir := os.TempDir()
-
-	fi, err := os.Stat(dir)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-
-	dev, ok := deviceOfInfo(fi)
-	if fsType, found := nonPersistent[dev]; ok && found {
-		fmt.Fprintf(os.Stderr, "%s is on %s, not a persistent filesystem, "+
-			"and scan records nothing there: set TMPDIR to a directory on a "+
-			"persistent filesystem\n", dir, fsType)
-		os.Exit(1)
-	}
-
-	os.Exit(m.Run())
-}
-
 // walSuffixes are the SQLite sidecar files a WAL-mode database keeps
 // while it is open. A clean close checkpoints the WAL and removes
 // both; finding either afterwards means the database was never closed.
@@ -316,38 +284,23 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) {
 }
 
 func TestRunMissingDatabaseIsFatal(t *testing.T) {
-	// README §Database: report, trees and info need an existing
+	// README §Database: report and trees need an existing database; a
-	// database; a missing one exits 1 with a message telling the user to
+	// missing one exits 1 with a message telling the user to run scan.
-	// run scan, after the line naming the database.
+	for _, name := range []string{cmdReport, cmdTrees} {
-	openLines := map[string]string{
-		cmdReport: reportOpenLine,
-		cmdTrees:  treesOpenLine,
-		cmdInfo:   infoOpenLine,
-	}
-
-	for name, openLine := range openLines {
 		t.Run(name, func(t *testing.T) {
 			path := testDBPath(t)
 			t.Setenv(databaseEnv, path)
 
-			var stdout bytes.Buffer
+			var stdout, stderr bytes.Buffer
 
-			// run gets os.Stderr, where the subcommand prints its line,
+			code := run([]string{name}, &stdout, &stderr)
-			// so the capture shows the two in the order printed.
-			stderr := captureStderr(t)
-
-			code := run([]string{name}, &stdout, os.Stderr)
 			if code != exitFatal {
 				t.Errorf("run(%s) = %d, want %d", name, code, exitFatal)
 			}
 
-			// Each shows its open phase, as it starts and as it ends,
+			want := "sfdupes: " + path + ": no database (run \"sfdupes " +
-			// before the error.
-			want := name + ": database " + path + " (from " + databaseEnv +
-				")\n" + openLine + "\n" + openLine + "\n" +
-				"sfdupes: " + path + ": no database (run \"sfdupes " +
 				"scan\" first, or set " + databaseEnv + ")\n"
-			if got := stderr(); got != want {
+			if got := stderr.String(); got != want {
 				t.Errorf("stderr = %q, want %q", got, want)
 			}
 
@@ -358,256 +311,6 @@ func TestRunMissingDatabaseIsFatal(t *testing.T) {
 	}
 }
 
-func TestRunNamesRelativeDatabase(t *testing.T) {
-	// README §Database: each subcommand's first line on stderr names the
-	// database, a relative SFDUPES_DATABASE made absolute against the
-	// working directory. scan calls it new until a scan has created it.
-	path := testDBPath(t)
-	t.Chdir(filepath.Dir(path))
-	t.Setenv(databaseEnv, filepath.Base(path))
-
-	dir := t.TempDir()
-	from := " (from " + databaseEnv + ")\n"
-
-	want := "scan: new database " + path + from
-	if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
-		t.Errorf("first scan stderr = %q, want it to start %q", got, want)
-	}
-
-	want = "scan: database " + path + from
-	if got := scanOK(t, dir); !strings.HasPrefix(got, want) {
-		t.Errorf("second scan stderr = %q, want it to start %q", got, want)
-	}
-
-	for _, name := range []string{cmdReport, cmdTrees} {
-		stderr := captureStderr(t)
-
-		code := run([]string{name}, io.Discard, os.Stderr)
-
-		want = name + ": database " + path + from
-		if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
-			t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
-				"start %q", name, code, got, exitOK, want)
-		}
-	}
-}
-
-func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
-	// README §Settings: the path names the file exactly. With link a
-	// symlink to the directory real/sub, link/../db.sqlite names
-	// real/db.sqlite, not db.sqlite beside link. Every subcommand must
-	// use that file, relative or absolute, and its first line must name
-	// the path as set, with the working directory put in front of a
-	// relative one.
-	dir, err := filepath.EvalSymlinks(t.TempDir())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Chdir(dir)
-
-	err = os.MkdirAll(filepath.Join(dir, "real", "sub"), 0o750)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.Symlink(filepath.Join(dir, "real", "sub"),
-		filepath.Join(dir, "link"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	named := dir + "/link/../db.sqlite"
-	from := " (from " + databaseEnv + ")\n"
-	tree := t.TempDir()
-
-	t.Setenv(databaseEnv, "link/../db.sqlite")
-
-	want := "scan: new database " + named + from
-	if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
-		t.Errorf("relative scan stderr = %q, want it to start %q", got, want)
-	}
-
-	// The absolute path names the database the relative one created.
-	t.Setenv(databaseEnv, named)
-
-	want = "scan: database " + named + from
-	if got := scanOK(t, tree); !strings.HasPrefix(got, want) {
-		t.Errorf("absolute scan stderr = %q, want it to start %q", got, want)
-	}
-
-	for _, name := range []string{cmdReport, cmdTrees} {
-		stderr := captureStderr(t)
-
-		code := run([]string{name}, io.Discard, os.Stderr)
-
-		want = name + ": database " + named + from
-		if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
-			t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
-				"start %q", name, code, got, exitOK, want)
-		}
-	}
-
-	_, err = os.Stat(filepath.Join(dir, "real", "db.sqlite"))
-	if err != nil {
-		t.Errorf("database the path names: %v", err)
-	}
-
-	// With no file at the cleaned path, report and trees succeeded on
-	// the database in real.
-	_, err = os.Stat(filepath.Join(dir, "db.sqlite"))
-	if !errors.Is(err, fs.ErrNotExist) {
-		t.Errorf("database at the cleaned path: stat error %v, want it "+
-			"missing", err)
-	}
-}
-
-func TestRunScanFromSymlinkedWorkingDirectory(t *testing.T) {
-	// README §Database: a relative path gets the working directory, its
-	// symlinks resolved, put in front of it. With home/proj a symlink
-	// to data/proj, entered as the working directory through the
-	// symlink, ../new/db.sqlite names data/new/db.sqlite. scan must
-	// create that directory and file, and nothing beside home/proj.
-	dir, err := filepath.EvalSymlinks(t.TempDir())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.MkdirAll(filepath.Join(dir, "data", "proj"), 0o750)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.Mkdir(filepath.Join(dir, "home"), 0o750)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.Symlink(filepath.Join(dir, "data", "proj"),
-		filepath.Join(dir, "home", "proj"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// t.Chdir also sets PWD to the path through the symlink, as a
-	// shell does after cd.
-	t.Chdir(filepath.Join(dir, "home", "proj"))
-	t.Setenv(databaseEnv, "../new/db.sqlite")
-
-	want := "scan: new database " + dir + "/data/proj/../new/db.sqlite" +
-		" (from " + databaseEnv + ")\n"
-	if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
-		t.Errorf("scan stderr = %q, want it to start %q", got, want)
-	}
-
-	_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
-	if err != nil {
-		t.Errorf("database the path names: %v", err)
-	}
-
-	_, err = os.Lstat(filepath.Join(dir, "home", "new"))
-	if !errors.Is(err, fs.ErrNotExist) {
-		t.Errorf("home/new: stat error %v, want it missing", err)
-	}
-}
-
-func TestRunEscapesDatabasePath(t *testing.T) {
-	// README §Database: each subcommand's first line escapes the
-	// database path like the paths in a report, so a tab, newline or
-	// backslash in it keeps the line one line.
-	dir := t.TempDir()
-	t.Setenv(databaseEnv, dir+"/a\tb\nc\\d.sqlite")
-
-	escaped := dir + `/a\tb\nc\\d.sqlite`
-	from := " (from " + databaseEnv + ")\n"
-
-	want := "scan: new database " + escaped + from
-	if got := scanOK(t, t.TempDir()); !strings.HasPrefix(got, want) {
-		t.Errorf("scan stderr = %q, want it to start %q", got, want)
-	}
-
-	for _, name := range []string{cmdReport, cmdTrees} {
-		stderr := captureStderr(t)
-
-		code := run([]string{name}, io.Discard, os.Stderr)
-
-		want = name + ": database " + escaped + from
-		if got := stderr(); code != exitOK || !strings.HasPrefix(got, want) {
-			t.Errorf("run(%s) = %d with stderr %q, want %d and it to "+
-				"start %q", name, code, got, exitOK, want)
-		}
-	}
-}
-
-func TestRunNamesDefaultDatabase(t *testing.T) {
-	// README §Database: with SFDUPES_DATABASE empty, each subcommand's
-	// first line on stderr names the default database. A database may
-	// exist at the default path on the machine running the test, so
-	// nothing here may create or read one: scan gets an operand that
-	// does not exist, which stops it before it takes the lock, and
-	// report, trees and info a cancelled context, which stops them
-	// before they read anything. They show their open phase after the
-	// line.
-	t.Setenv(databaseEnv, "")
-
-	// Whether scan calls the database new depends on that machine too.
-	scanWant := "scan: database " + defaultDatabasePath + " (default)\n"
-
-	_, err := os.Stat(defaultDatabasePath)
-	if errors.Is(err, fs.ErrNotExist) {
-		scanWant = "scan: new database " + defaultDatabasePath +
-			" (default)\n"
-	}
-
-	missing := filepath.Join(t.TempDir(), "nope")
-
-	cases := map[string]struct {
-		run  func() error
-		want string
-	}{
-		cmdScan: {
-			run: func() error {
-				return runScan(t.Context(), []string{missing}, 1, false, false)
-			},
-			want: scanWant,
-		},
-		cmdReport: {
-			run: func() error {
-				return runReport(cancelledContext(t), io.Discard)
-			},
-			want: "report: database " + defaultDatabasePath + " (default)\n" +
-				reportOpenLine + "\n" + reportOpenLine + "\n",
-		},
-		cmdTrees: {
-			run: func() error {
-				return runTrees(cancelledContext(t), io.Discard)
-			},
-			want: "trees: database " + defaultDatabasePath + " (default)\n" +
-				treesOpenLine + "\n" + treesOpenLine + "\n",
-		},
-		cmdInfo: {
-			run: func() error {
-				return runInfo(cancelledContext(t), io.Discard)
-			},
-			want: "info: database " + defaultDatabasePath + " (default)\n" +
-				infoOpenLine + "\n" + infoOpenLine + "\n",
-		},
-	}
-
-	for name, tc := range cases {
-		stderr := captureStderr(t)
-
-		err = tc.run()
-		if err == nil {
-			t.Errorf("%s: error = nil, want it stopped", name)
-		}
-
-		if got := stderr(); got != tc.want {
-			t.Errorf("%s stderr = %q, want %q", name, got, tc.want)
-		}
-	}
-}
-
 func TestRunUsageErrors(t *testing.T) {
 	// Usage errors keep exiting 2 with cobra's own report on stderr.
 	cases := map[string]struct {
@@ -618,7 +321,6 @@ func TestRunUsageErrors(t *testing.T) {
 		"scan without paths": {[]string{cmdScan}, usageMarker},
 		"report with args":   {[]string{cmdReport, "x"}, usageMarker},
 		"trees with args":    {[]string{cmdTrees, "x"}, usageMarker},
-		"info with args":     {[]string{cmdInfo, "x"}, usageMarker},
 		"unknown flag":       {[]string{cmdScan, "--nope", "/"}, usageMarker},
 		"unknown subcommand": {[]string{"nope"}, "unknown command"},
 	}
@@ -648,7 +350,7 @@ func TestRunUsageErrors(t *testing.T) {
 }
 
 func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
-	// README §Settings: --workers below 1 is a usage error reported in
+	// README §scan mode: --workers below 1 is a usage error reported in
 	// one line on stderr, before the scan opens the database.
 	for _, workers := range []string{"0", "-1"} {
 		t.Run(workers, func(t *testing.T) {
@@ -686,8 +388,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
 func TestRunHelp(t *testing.T) {
 	t.Parallel()
 
-	// README §Settings: help goes to stderr, exits 0, and leaves stdout
+	// README §Subcommands: help goes to stderr, exits 0, and leaves
-	// empty.
+	// stdout empty.
 	cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
 
 	for _, args := range cases {
@@ -713,10 +415,8 @@ func TestRunHelp(t *testing.T) {
 func TestRunVersion(t *testing.T) {
 	t.Parallel()
 
-	// README §Settings: the version is one line on stdout, with nothing
+	// README §Subcommands: the version is one line on stdout, with
-	// on stderr, and exits 0.
+	// nothing on stderr, and exits 0.
-	info, _ := debug.ReadBuildInfo()
-
 	for _, arg := range []string{"--version", "-v"} {
 		var stdout, stderr bytes.Buffer
 
@@ -725,7 +425,7 @@ func TestRunVersion(t *testing.T) {
 			t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
 		}
 
-		want := "sfdupes " + buildVersion(Version, info) + "\n"
+		want := "sfdupes " + Version + "\n"
 		if got := stdout.String(); got != want {
 			t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
 		}
@@ -736,45 +436,11 @@ func TestRunVersion(t *testing.T) {
 	}
 }
 
-func TestBuildVersion(t *testing.T) {
-	t.Parallel()
-
-	// README "Install": the linker's version wins; otherwise the main
-	// module's version that Go recorded; "dev" when neither is known.
-	tagged := &debug.BuildInfo{Main: debug.Module{Version: "v0.0.2"}}
-	pseudo := &debug.BuildInfo{Main: debug.Module{
-		Version: "v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1",
-	}}
-	devel := &debug.BuildInfo{Main: debug.Module{Version: "(devel)"}}
-
-	cases := []struct {
-		name   string
-		linker string
-		info   *debug.BuildInfo
-		want   string
-	}{
-		{"linker set", "v0.0.2-3-g4e4a94d", tagged, "v0.0.2-3-g4e4a94d"},
-		{"linker set, no build info", "4e4a94d", nil, "4e4a94d"},
-		{"build info with a tag", "", tagged, "v0.0.2"},
-		{"build info with a pseudo-version", "", pseudo,
-			"v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1"},
-		{"(devel)", "", devel, "dev"},
-		{"no build info", "", nil, "dev"},
-	}
-
-	for _, c := range cases {
-		if got := buildVersion(c.linker, c.info); got != c.want {
-			t.Errorf("%s: buildVersion(%q, ...) = %q, want %q",
-				c.name, c.linker, got, c.want)
-		}
-	}
-}
-
 func TestRunVersionWriteFailureIsFatal(t *testing.T) {
 	t.Parallel()
 
-	// README §Error handling: a failed write to stdout is a fatal error,
+	// README §Error handling: a stdout write failure exits 1, reported
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// in one line on stderr.
 	var stderr bytes.Buffer
 
 	code := run([]string{"--version"}, failingWriter{}, &stderr)
@@ -928,21 +594,24 @@ func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
 }
 
 func TestRunScanSkipsZFSOperand(t *testing.T) {
-	zfs := filepath.Join(t.TempDir(), ".zfs")
+	path := testDBPath(t)
-	f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
-
-	// No scan records a file inside a .zfs directory, so the record
-	// beneath the operand is written straight into the database.
-	path := seedDatabase(t, []scanRec{{size: 10, path: f}})
 	t.Setenv(databaseEnv, path)
 
+	zfs := filepath.Join(t.TempDir(), ".zfs")
+	snapshot := filepath.Join(zfs, "snapshot", "hourly")
+	f := writeFile(t, snapshot, "f", pattern(1, 10))
+
+	// An operand beneath a .zfs directory is walked, because it is not
+	// itself named .zfs.
+	scanOK(t, snapshot)
+
 	assertOperandSkipped(t, path, zfs, ".zfs directory", f)
 }
 
 // assertOperandSkipped scans operand alone and checks that it is skipped
 // as kind: a warning naming it, one skip in the summary, exit 0, and the
-// record for kept, stored beneath operand before this scan, still in the
+// record for kept, which an earlier scan stored beneath operand, still
-// database at dbPath.
+// in the database at dbPath.
 func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	kept string,
 ) {
@@ -971,121 +640,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	recordByPath(t, dbRecords(t, db), kept)
 }
 
-// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
-// directory, first alone and then under another operand. Each time it
-// is warned about and counted as skipped, and nothing in it is walked.
-// The other operand is scanned as usual.
-func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
-	writeFile(t, hourly, "f", pattern(1, 10))
-	f := writeFile(t, dir, "f", pattern(2, 10))
-
-	warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
-
-	for _, tc := range []struct {
-		operands []string
-		summary  string
-	}{
-		{
-			[]string{hourly},
-			"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-		{
-			[]string{dir, hourly},
-			"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-	} {
-		stderr := scanOK(t, tc.operands...)
-		if !strings.Contains(stderr, warning) ||
-			!strings.Contains(stderr, tc.summary) {
-			t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
-				stderr, warning, tc.summary)
-		}
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
-		t.Errorf("records = %q, want %q", got, []string{f})
-	}
-}
-
-// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
-// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
-// by the path given, counted as skipped, and not walked.
-func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
-	t.Setenv(databaseEnv, testDBPath(t))
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
-
-	snaps := filepath.Join(dir, "snaps")
-
-	err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(snaps, "daily")
-	stderr := scanOK(t, daily)
-
-	warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
-	summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-		"0 unchanged), 1 skipped\n"
-
-	if !strings.Contains(stderr, warning) ||
-		!strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
-	}
-}
-
-// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
-// a symlink to a directory outside any .zfs directory. The operand is
-// scanned as usual, and its file is recorded by the path given.
-func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
-
-	data := filepath.Join(dir, "data")
-
-	err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(data, "daily")
-	stderr := scanOK(t, daily)
-
-	summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-		"0 unchanged), 0 skipped\n"
-	if !strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q", stderr, summary)
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
-}
-
 func TestRunReportSucceeds(t *testing.T) {
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
@@ -1107,16 +661,10 @@ func TestRunReportSucceeds(t *testing.T) {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
 
-	// Its phases come between the two lines; see
+	want = "report: 2 records read, 1 duplicate groups, 1 dupe files, " +
-	// TestRunReportProgressNamesEachPhase.
-	first := "report: database " + path + " (from " + databaseEnv + ")\n"
-	last := "\nreport: 2 records read, 1 duplicate groups, 1 dupe files, " +
 		"300 B reclaimable\n"
-
+	if got := stderr(); got != want {
-	if got := stderr(); !strings.HasPrefix(got, first) ||
+		t.Errorf("stderr = %q, want %q", got, want)
-		!strings.HasSuffix(got, last) {
-		t.Errorf("stderr = %q, want it to start %q and end %q",
-			got, first, last)
 	}
 
 	assertNoSidecars(t, path)
@@ -1146,16 +694,10 @@ func TestRunTreesSucceeds(t *testing.T) {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
 
-	// Its phases come between the two lines; see
+	want = "trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " +
-	// TestRunTreesProgressNamesEachPhase.
+		"300 B reclaimable\n"
-	first := "trees: database " + path + " (from " + databaseEnv + ")\n"
+	if got := stderr(); got != want {
-	last := "\ntrees: 2 records read, 1 duplicate tree groups, " +
+		t.Errorf("stderr = %q, want %q", got, want)
-		"1 dupe trees, 300 B reclaimable\n"
-
-	if got := stderr(); !strings.HasPrefix(got, first) ||
-		!strings.HasSuffix(got, last) {
-		t.Errorf("stderr = %q, want it to start %q and end %q",
-			got, first, last)
 	}
 
 	assertNoSidecars(t, path)
@@ -1234,7 +776,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 }
 
 func TestRunDatabasePathUsedAsGiven(t *testing.T) {
-	// README §Settings: the path names the database file exactly. In
+	// README §Database: the path names the database file exactly. In
 	// SQLite's connection string an unescaped ? or # would end the file
 	// name and % would start an escape, and a path starting with //
 	// could be read as a host name. %25 is a valid escape, so unescaped
@@ -1280,7 +822,7 @@ func holdScanLock(t *testing.T, path string) {
 
 func TestRunSecondScanFails(t *testing.T) {
 	// README §Database: while one scan holds the lock, a second scan
-	// fails at once, naming the lock file, without opening the
+	// fails at once, naming the lock file, without creating the
 	// database.
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
@@ -1306,99 +848,10 @@ func TestRunSecondScanFails(t *testing.T) {
 		t.Errorf("stdout = %q, want nothing (data only)", got)
 	}
 
-	// Taking the lock created the database file, empty; opening it as
+	_, err := os.Stat(path)
-	// a database would have written to it.
+	if !errors.Is(err, fs.ErrNotExist) {
-	info, err := os.Stat(path)
+		t.Errorf("stat %s = %v, want the database not created", path, err)
-	if err != nil {
-		t.Fatal(err)
 	}
-
-	if info.Size() != 0 {
-		t.Errorf("database size = %d, want 0 (not opened)", info.Size())
-	}
-}
-
-// assertSecondScanFails runs a scan against the database
-// SFDUPES_DATABASE names while the test holds the lock on it. It fails
-// the test unless that scan fails at once, naming the lock file beside
-// path, the database's real path.
-func assertSecondScanFails(t *testing.T, path string) {
-	t.Helper()
-
-	var stderr bytes.Buffer
-
-	code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
-	if code != exitFatal {
-		t.Errorf("run(scan) = %d, want %d", code, exitFatal)
-	}
-
-	want := "sfdupes: another scan is running (lock held on " +
-		path + ".lock)\n"
-	if got := stderr.String(); got != want {
-		t.Errorf("stderr = %q, want %q", got, want)
-	}
-}
-
-func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
-	// README §Database: every path to one database names the same lock
-	// file, so while a scan holds it through one path, a scan through
-	// another fails, naming the lock file beside the database's real
-	// path.
-	t.Run("symlink to the database", func(t *testing.T) {
-		path := testDBPath(t)
-		t.Setenv(databaseEnv, path)
-
-		scanFixture(t)
-
-		link := filepath.Join(t.TempDir(), "link.sqlite")
-
-		err := os.Symlink(path, link)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		holdScanLock(t, path)
-		t.Setenv(databaseEnv, link)
-		assertSecondScanFails(t, path)
-	})
-
-	t.Run("symlink to its directory, no database yet", func(t *testing.T) {
-		path := testDBPath(t)
-		link := filepath.Join(t.TempDir(), "link")
-
-		err := os.Symlink(filepath.Dir(path), link)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		holdScanLock(t, path)
-		t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
-		assertSecondScanFails(t, path)
-	})
-
-	t.Run("symlink whose target does not exist yet", func(t *testing.T) {
-		path := testDBPath(t)
-		link := filepath.Join(t.TempDir(), "link.sqlite")
-
-		err := os.Symlink(path, link)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// The first scan takes the lock through the link before the
-		// database exists, then creates the database through it.
-		holdScanLock(t, link)
-
-		db, err := openScanDatabase(t.Context(), link)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		closeScanDatabase(t.Context(), db, link)
-
-		t.Setenv(databaseEnv, link)
-		assertSecondScanFails(t, path)
-	})
 }
 
 func TestRunScanReleasesLock(t *testing.T) {
@@ -1445,9 +898,9 @@ func TestRunReportsDuringScan(t *testing.T) {
 }
 
 func TestRunStdoutClosedIsFatal(t *testing.T) {
-	// README §Error handling: a failed write to stdout is a fatal error,
+	// README §Error handling: a stdout write failure exits 1, reported
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// in one line on stderr.
-	for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
+	for _, name := range []string{cmdReport, cmdTrees} {
 		t.Run(name, func(t *testing.T) {
 			t.Setenv(databaseEnv, testDBPath(t))
 
@@ -1497,7 +950,6 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
 	cases := map[string]func(context.Context, io.Writer) error{
 		cmdReport: runReport,
 		cmdTrees:  runTrees,
-		cmdInfo:   runInfo,
 	}
 
 	for name, fn := range cases {

progress.go

@@ -2,321 +2,122 @@ package main
 
 import (
 	"fmt"
-	"io"
-	"math"
 	"os"
-	"slices"
-	"strings"
-	"sync"
 	"time"
-	"unicode/utf8"
 
 	"github.com/schollz/progressbar/v3"
 	"golang.org/x/term"
 )
 
-// plainInterval is the time between progress lines while a phase runs,
+// plainInterval is the minimum time between progress lines when stderr
-// when stderr is not a TTY.
+// is not a TTY.
 const plainInterval = 5 * time.Second
 
-// redrawInterval is the time between redraws of the progress display on
+// barThrottle caps how often the TTY progress bar redraws.
-// a TTY.
+const barThrottle = 100 * time.Millisecond
-const redrawInterval = 100 * time.Millisecond
 
-// rateWindow is the time the rate a display shows covers: the items
+// walkSpinnerType selects the progressbar library's spinner style used
-// completed over the last rateWindow, which is the time since the line
+// for the walk pass, which has no known total.
-// before when stderr is not a TTY.
+const walkSpinnerType = 14
-const rateWindow = 5 * time.Second
-
-// stallTime is how long a phase goes with no item completed before its
-// line says it has stalled, and where it waits, in place of the rate.
-const stallTime = 10 * time.Second
-
-// spinnerType selects the progressbar library's spinner style.
-const spinnerType = 14
-
-// barWidth is the width of a phase's bar between its two ends, as in
-// the example line in README "Progress".
-const barWidth = 10
-
-// barCell is one column of the filled part of a bar.
-const barCell = "█"
-
-// barColumns is the width of what percentBar returns: the percent in
-// four columns and its sign, a space, and the bar with its two ends.
-const barColumns = barWidth + 8
 
 // percentScale converts a fraction to a percentage.
 const percentScale = 100
 
-// fallbackWidth is the width taken for a terminal whose width cannot be
-// read, as the progressbar library takes it.
-const fallbackWidth = 80
-
-// spinnerMargin is the columns of a line besides its text: the spinner
-// and the three spaces the library draws around the text, and the
-// terminal's last column, which every line stays clear of so that no
-// terminal wraps it.
-const spinnerMargin = 5
-
-// countingRecords is what the display of a phase that reads the records
-// says while it counts them, before it has its total.
-const countingRecords = "counting the records"
-
-// openPhase is the name of the phase in which report, trees and info
-// open the database, and openingDatabase what its display says while
-// it does.
-const (
-	openPhase       = "open"
-	openingDatabase = "opening the database"
-)
-
 // stderrIsTTY reports whether stderr is attached to a terminal.
 func stderrIsTTY() bool {
 	return term.IsTerminal(int(os.Stderr.Fd()))
 }
 
-// terminalWidth returns the width of the terminal on stderr, or
+// progress renders one scan pass's progress on stderr. On a TTY it
-// fallbackWidth, as the progressbar library does, when it cannot read
+// delegates to the progressbar library (spinner style when the total is
-// one.
+// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
-func terminalWidth() int {
+// stderr is not a TTY it emits no ANSI redraws: it prints a plain
-	width, _, err := term.GetSize(int(os.Stderr.Fd()))
+// one-line update as the pass starts, then no more often than every
-	if err != nil || width <= 0 {
+// plainInterval.
-		return fallbackWidth
-	}
-
-	return width
-}
-
-// phaseLabel names phase with its place among phases, as "[2/4] walk".
-func phaseLabel(phases []string, phase string) string {
-	return fmt.Sprintf("[%d/%d] %s",
-		slices.Index(phases, phase)+1, len(phases), phase)
-}
-
-// progressParams describe one phase's progress display as it starts.
-type progressParams struct {
-	label      string // the phase and its place, from phaseLabel
-	unit       string // what the phase counts, as "files"
-	total      int64  // the items the phase will complete; -1 when not known
-	estimate   bool   // total is an estimate, which the count may pass
-	waitingFor string // what the phase waits for until setTotal is called
-	stalledOn  string // where the phase waits, as "disk/db"; see stallTime
-}
-
-// drawnCount is the count of items completed as of one draw, and when
-// that draw was.
-type drawnCount struct {
-	at    time.Time
-	count int64
-}
-
-// progress renders one phase's progress on stderr. On a TTY the
-// progressbar library draws its spinner and then line, fitted to the
-// terminal's width. A phase with an exact total ends the line with its
-// percent and bar, at the terminal's right edge, when the terminal has
-// room for them. It is drawn again every redrawInterval, so its elapsed
-// time keeps moving while the phase waits for its next item. When
-// stderr is not a TTY it emits no ANSI redraws: it prints line as the
-// phase starts, every plainInterval while it runs, and as it ends.
 //
-// The draws come from a goroutine of their own, so mu guards the fields
+// All methods must be called from the main goroutine only. On a TTY
-// below it. Every method must be called from the main goroutine. A nil
+// the library also redraws a spinner from its own goroutine, several
-// *progress is a valid no-display receiver: every method is a no-op, so
+// times a second, so its count and elapsed time stay current while a
-// batched database flushes during the hash phase can reuse the update
+// pass waits for its next item. A nil *progress is a valid
-// phase's helpers without rendering anything, and its writer writes
+// no-display receiver: every method is a no-op, so batched database
-// with no display to clear.
+// flushes during the streaming pass can reuse the update-pass helpers
+// without rendering anything.
 type progress struct {
-	now   func() time.Time // reads the time
+	label string
-	width func() int       // reads the terminal's width; nil when not a TTY
+	total int64 // -1 when unknown (walk pass)
-	stop  chan struct{}    // closed by finish to end the draws
+	bar   *progressbar.ProgressBar
-	done  chan struct{}    // closed once the draws have ended
+	count int64
-
+	start time.Time
-	mu         sync.Mutex
+	last  time.Time
-	bar        *progressbar.ProgressBar // what the library draws; nil until then
-	widest     int                      // the widest line bar has drawn, in columns
-	label      string
-	unit       string
-	total      int64
-	estimate   bool
-	waitingFor string
-	stalledOn  string
-	count      int64
-	start      time.Time // when the phase started, for the elapsed time
-	rateStart  time.Time // when the count started, for the time remaining
-	lastItem   time.Time // when the last item was completed, or the count started
-	// drawn holds the count at each draw over the last rateWindow, oldest
-	// first; the rate counts from the first. It keeps the draw before the
-	// latest even when that is older, as the line before is when lines
-	// are plainInterval apart and a tick comes late.
-	drawn []drawnCount
 }
 
-// newProgress starts the display of one phase on stderr.
+func newProgress(label string, total int64) *progress {
-func newProgress(params progressParams) *progress {
+	p := &progress{label: label, total: total, start: time.Now()}
 	if stderrIsTTY() {
-		return startProgress(params, terminalWidth, redrawInterval, time.Now)
+		p.bar = newBar(label, total)
+
+		return p
 	}
 
-	return startProgress(params, nil, plainInterval, time.Now)
+	// Print the zero state at once: the first item may take minutes,
-}
+	// and a pass must never look hung.
-
+	p.last = p.start
-// startProgress shows the phase params describes at once, and again
+	fmt.Fprintln(os.Stderr, p.plainLine())
-// every interval until finish: with the progressbar library on a
-// terminal, whose width each draw reads by calling width, or as a plain
-// line when width is nil. The display reads the time by calling now.
-func startProgress(params progressParams, width func() int,
-	interval time.Duration, now func() time.Time,
-) *progress {
-	start := now()
-	p := &progress{
-		now: now, width: width,
-		stop: make(chan struct{}), done: make(chan struct{}),
-		label: params.label, unit: params.unit, total: params.total,
-		estimate: params.estimate, waitingFor: params.waitingFor,
-		stalledOn: params.stalledOn, start: start, rateStart: start,
-		lastItem: start, drawn: []drawnCount{{at: start}},
-	}
-
-	// Show the zero state at once: the first item may take minutes,
-	// and a phase must never look hung.
-	p.draw()
-
-	go func() {
-		defer close(p.done)
-
-		ticker := time.NewTicker(interval)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-p.stop:
-				return
-			case <-ticker.C:
-				p.draw()
-			}
-		}
-	}()
 
 	return p
 }
 
-// draw shows the phase's progress: on a TTY in place of the line
+// newBar builds the TTY display for newProgress.
-// before, and otherwise as a plain line. It first adds the count to
+func newBar(label string, total int64) *progressbar.ProgressBar {
-// drawn, then drops from drawn the draws older than rateWindow, except
+	opts := []progressbar.Option{
-// the one before this draw.
-func (p *progress) draw() {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	now := p.now()
-
-	p.drawn = append(p.drawn, drawnCount{at: now, count: p.count})
-	for len(p.drawn) > 2 && now.Sub(p.drawn[0].at) > rateWindow {
-		p.drawn = p.drawn[1:]
-	}
-
-	if p.width == nil {
-		fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, false))
-
-		return
-	}
-
-	width := p.width()
-	text := p.line(now, width-spinnerMargin, true)
-
-	// Before each draw the library erases its line with as many spaces as
-	// the widest line it has drawn. A display gets a new one when the
-	// terminal has been made too narrow for that erase, which would wrap
-	// and leave a line behind at every draw.
-	if p.bar == nil || p.widest >= width {
-		p.newBar(width)
-	}
-
-	p.bar.Describe(text)
-
-	// The library drew the spinner, the text and the spaces around it,
-	// less the terminal's last column, which spinnerMargin keeps clear. A
-	// bar's cells take one column each but more than one byte.
-	p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
-}
-
-// newBar has the library make what draw draws: its spinner, followed by
-// the text draw gives it and nothing of the library's own. p.mu must be
-// held.
-func (p *progress) newBar(width int) {
-	if p.bar != nil {
-		// Erase what was drawn before, short of the last column of a
-		// terminal that may have been made narrower since: the library's
-		// own erase is as wide as the widest line it drew.
-		blank := strings.Repeat(" ", min(p.widest, width-1))
-		fmt.Fprint(os.Stderr, "\r"+blank+"\r")
-	}
-
-	// A total of -1 gives the library's spinner.
-	p.bar = progressbar.NewOptions64(-1,
 		progressbar.OptionSetWriter(os.Stderr),
-		progressbar.OptionSpinnerType(spinnerType),
+		progressbar.OptionSetDescription(label),
-		// The spinner turns at each draw, not on a timer of the
+		progressbar.OptionShowCount(),
-		// library's own.
+		progressbar.OptionShowIts(),
-		progressbar.OptionSetSpinnerChangeInterval(0),
+		progressbar.OptionSetItsString("files"),
-		// The text carries the elapsed time and the time remaining.
+		progressbar.OptionSetElapsedTime(true),
-		progressbar.OptionSetElapsedTime(false),
+		progressbar.OptionThrottle(barThrottle),
-		progressbar.OptionSetPredictTime(false),
+		// Render at zero immediately: a phase must be visible the
-	)
+		// moment it starts, even before its first item completes.
-	p.widest = 0
+		progressbar.OptionSetRenderBlankState(true),
-}
+	}
-
+	if total >= 0 {
-// percentBar returns the percent and the bar that end the text of a
+		opts = append(opts,
-// phase with an exact total, as "  25% |██        |": the bar's
+			progressbar.OptionSetPredictTime(true),
-// barWidth columns fill in proportion to the items done. A count past
+			progressbar.OptionShowElapsedTimeOnFinish(),
-// the total, as when a scan adds records while trees reads them, shows
+		)
-// as 100%. p.mu must be held.
+	} else {
-func (p *progress) percentBar() string {
+		opts = append(opts,
-	done := min(float64(p.count)/float64(p.total), 1)
+			progressbar.OptionSetPredictTime(false),
-	filled := strings.Repeat(barCell, int(done*barWidth))
+			progressbar.OptionSpinnerType(walkSpinnerType),
-
+		)
-	return fmt.Sprintf("%4.0f%% |%-*s|", done*percentScale, barWidth, filled)
-}
-
-// setTotal gives a phase that started without a total its total, once
-// it is known. The rate, the time remaining and the time with no item
-// completed count from then on: the time spent waiting for the total
-// says nothing about how fast items go.
-func (p *progress) setTotal(total int64) {
-	if p == nil {
-		return
 	}
 
-	now := p.now()
+	return progressbar.NewOptions64(total, opts...)
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	p.total = total
-	p.rateStart, p.lastItem = now, now
-	p.drawn = []drawnCount{{at: now, count: p.count}}
 }
 
-// increment records one completed item, which the display shows at its
+// increment records one completed item and refreshes the display.
-// next draw.
 func (p *progress) increment() {
 	if p == nil {
 		return
 	}
 
-	now := p.now()
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
 	p.count++
-	p.lastItem = now
+	if p.bar != nil {
+		_ = p.bar.Add(1)
+
+		return
+	}
+
+	if time.Since(p.last) >= plainInterval {
+		p.last = time.Now()
+		fmt.Fprintln(os.Stderr, p.plainLine())
+	}
 }
 
-// warnf prints a one-line warning to stderr without corrupting the
+// warnf prints a one-line warning to stderr without corrupting the bar.
-// display. The whole message is escaped like a report's path columns,
+// The whole message is escaped like a report's path columns, so a path
-// so a path holding a newline cannot split the warning.
+// holding a newline cannot split the warning.
 func (p *progress) warnf(format string, args ...any) {
 	if p == nil {
 		return
@@ -324,210 +125,51 @@ func (p *progress) warnf(format string, args ...any) {
 
 	msg := escapePath(fmt.Sprintf(format, args...))
 
-	p.mu.Lock()
+	if p.bar != nil && p.total < 0 {
-	defer p.mu.Unlock()
+		// The library also redraws a spinner from its own goroutine, so
+		// a direct write could land inside a redraw. The bar prints the
+		// warning itself, just before its next redraw.
+		_, _ = progressbar.Bprintln(p.bar, msg)
+
+		return
+	}
 
 	if p.bar != nil {
-		// The warning takes the display's line; the next draw shows
-		// the display again, on the line below.
 		_ = p.bar.Clear()
 	}
 
 	fmt.Fprintln(os.Stderr, msg)
 }
 
-// progressWriter writes a phase's output to out, which may be the file
+// finish terminates the pass's display. A bar whose pass stopped short
-// the phase's display is drawn on, as stdout is in a plain "sfdupes
+// of its total, as an interrupted one does, is left as last drawn; the
-// trees" on a terminal. It holds what is written to it and writes that
+// library's Finish would fill it up.
-// to out in pieces of about ioBufSize bytes. When out is the file
-// stderr is, each piece is written under the display's lock with the
-// display cleared first, as warnf does for a warning: no draw lands in
-// the middle of a piece, and the next draw shows the display again, on
-// the line below. Every write to it must end with a newline, as a row
-// written with one fmt.Fprintf does, so that each piece ends a line.
-// Flush writes what it holds once the last row is written.
-type progressWriter struct {
-	// p is the display to clear, nil unless out is the file stderr is.
-	p   *progress
-	out io.Writer
-	buf []byte
-}
-
-// writer returns a progressWriter writing to out around p's display.
-// It clears the display and takes its lock only when out is the file
-// stderr is: the same device and inode, as os.SameFile compares them,
-// which is one terminal, or one pipe or file that both are sent to.
-// Otherwise it writes to out without the lock, so the display keeps
-// being drawn while a write to out waits for a slow reader.
-func (p *progress) writer(out io.Writer) *progressWriter {
-	w := &progressWriter{out: out}
-
-	if f, ok := out.(*os.File); ok {
-		outInfo, outErr := f.Stat()
-		errInfo, errErr := os.Stderr.Stat()
-
-		if outErr == nil && errErr == nil && os.SameFile(outInfo, errInfo) {
-			w.p = p
-		}
-	}
-
-	return w
-}
-
-// Write holds b, and writes what it holds to out once that comes to
-// ioBufSize bytes.
-func (w *progressWriter) Write(b []byte) (int, error) {
-	w.buf = append(w.buf, b...)
-	if len(w.buf) < ioBufSize {
-		return len(b), nil
-	}
-
-	return len(b), w.Flush()
-}
-
-// Flush writes what w holds to out. When out is the file stderr is, the
-// display is cleared first and not drawn again until it is written.
-func (w *progressWriter) Flush() error {
-	if w.p != nil {
-		w.p.mu.Lock()
-		defer w.p.mu.Unlock()
-
-		if w.p.bar != nil {
-			_ = w.p.bar.Clear()
-		}
-	}
-
-	_, err := w.out.Write(w.buf)
-	w.buf = w.buf[:0]
-
-	return err
-}
-
-// finish ends the phase's display, drawn once more as the phase ends.
-// A phase stopped short of its total, as an interrupted one is, shows
-// the count it stopped at rather than a full bar.
 func (p *progress) finish() {
 	if p == nil {
 		return
 	}
 
-	close(p.stop)
+	if p.bar != nil {
-	<-p.done
+		if p.total >= 0 && p.count < p.total {
+			_ = p.bar.Exit()
+		} else {
+			_ = p.bar.Finish()
+		}
 
-	p.draw()
-
-	if p.width != nil {
-		// The display stays on the TTY as drawn; what follows starts
-		// on a line of its own.
 		fmt.Fprintln(os.Stderr)
+
+		return
 	}
+
+	fmt.Fprintln(os.Stderr, p.plainLine())
 }
 
-// line formats the phase's progress as of now in at most room columns,
+// plainLine formats a non-TTY progress line.
-// as
+func (p *progress) plainLine() string {
-//
+	elapsed := time.Since(p.start)
-//	[3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, remain 13s
-//
-// where "files" is the phase's unit. A phase with no total shows the
-// items done instead, or what it waits for. The rate is that since the
-// first draw in drawn, and the time remaining comes from the rate since
-// the count started. Once no item has been completed for stallTime, the
-// rate, which would read 0, gives way to the word "stalled" and where
-// the phase waits, as "stalled disk/db".
-//
-// When bar is true, as on a terminal, a phase with an exact total ends
-// the line with its percent and bar, in place of the percent after its
-// count. A total of 0 leaves the bar nothing to fill.
-//
-// The phase and its place, the progress, the elapsed time and the time
-// remaining are kept whole. A line wider than room first leaves out the
-// rate, the words of a stalled phase, or what the phase waits for. Next
-// it leaves out the bar. Only if it is still wider is it cut at room.
-// Every part but the bar is ASCII, so its length is its width in
-// columns. p.mu must be held.
-func (p *progress) line(now time.Time, room int, bar bool) string {
-	elapsed := now.Sub(p.start).Round(time.Second)
-
-	// head and tail hold the parts kept whole; extra, between them, is
-	// the part left out.
-	var head, extra, tail string
-
-	if p.total < 0 && p.waitingFor != "" {
-		head, extra = p.label+": ", p.waitingFor+", "
-		tail = fmt.Sprintf("elapsed %s, remain unknown", elapsed)
-	} else {
-		since := p.drawn[0]
-		rate := perSecond(p.count-since.count, now.Sub(since.at))
-		average := perSecond(p.count, now.Sub(p.rateStart))
-
-		head = p.label + ": " + p.itemsDone(true) + ", "
-		extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
-		tail = fmt.Sprintf("elapsed %s, remain %s", elapsed, p.remain(average))
-
-		if now.Sub(p.lastItem) >= stallTime {
-			extra = "stalled " + p.stalledOn + ", "
-		}
-	}
-
-	if bar && p.total > 0 && !p.estimate {
-		// Beside the bar, which shows the percent, the count shows none.
-		// The line is padded to the room the bar leaves it, so the bar
-		// stays at room's right edge however long the line is.
-		beside := room - barColumns
-		barHead := p.label + ": " + p.itemsDone(false) + ", "
-
-		switch {
-		case len(barHead+extra+tail) <= beside:
-			return fmt.Sprintf("%-*s%s", beside, barHead+extra+tail,
-				p.percentBar())
-		case len(barHead+tail) <= beside:
-			return fmt.Sprintf("%-*s%s", beside, barHead+tail, p.percentBar())
-		}
-
-		// The bar gives way. The rate, or the words of a stalled phase,
-		// gave way before it and stay out.
-		extra = ""
-	}
-
-	full, kept := head+extra+tail, head+tail
-
-	switch {
-	case len(full) <= room:
-		return full
-	case len(kept) <= room:
-		return kept
-	}
-
-	return kept[:max(room, 0)]
-}
-
-// perSecond returns count items over d as items a second, or 0 when no
-// time has passed.
-func perSecond(count int64, d time.Duration) float64 {
-	if d <= 0 {
-		return 0
-	}
-
-	return float64(count) / d.Seconds()
-}
-
-// itemsDone formats the items done, as "[1234/5000] 25%", with "~"
-// before a total that is an estimate and without the percent when
-// percent is false, or as "1234 files", in the phase's unit, when the
-// phase has no total. p.mu must be held.
-func (p *progress) itemsDone(percent bool) string {
 	if p.total < 0 {
-		return fmt.Sprintf("%d %s", p.count, p.unit)
+		return fmt.Sprintf("%s: %d files, elapsed %s",
-	}
+			p.label, p.count, elapsed.Round(time.Second))
-
-	approx := ""
-	if p.estimate {
-		approx = "~"
-	}
-
-	done := fmt.Sprintf("[%d/%s%d]", p.count, approx, p.total)
-	if !percent {
-		return done
 	}
 
 	pct := float64(percentScale)
@@ -535,24 +177,19 @@ func (p *progress) itemsDone(percent bool) string {
 		pct = percentScale * float64(p.count) / float64(p.total)
 	}
 
-	return fmt.Sprintf("%s %.0f%%", done, pct)
+	rate := 0.0
-}
+	if s := elapsed.Seconds(); s > 0 {
-
+		rate = float64(p.count) / s
-// remain returns the time the phase has left at rate items a second, or
-// "unknown": with no total, before the first item, and once the count
-// passes an estimate. p.mu must be held.
-func (p *progress) remain(rate float64) string {
-	switch {
-	case p.total < 0, p.estimate && p.count > p.total:
-		return "unknown"
-	case p.count >= p.total:
-		return "0s"
-	case rate <= 0:
-		return "unknown"
 	}
 
-	left := time.Duration(
+	eta := "?"
-		float64(p.total-p.count) / rate * float64(time.Second))
 
-	return left.Round(time.Second).String()
+	if rate > 0 && p.count <= p.total {
+		remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
+		eta = remaining.Round(time.Second).String()
+	}
+
+	return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
+		p.label, p.count, p.total, pct, rate,
+		elapsed.Round(time.Second), eta)
 }

report.go

@@ -1,13 +1,11 @@
 package main
 
 import (
+	"bufio"
 	"context"
-	"crypto/sha256"
-	"database/sql"
 	"fmt"
 	"io"
 	"os"
-	"slices"
 	"strings"
 	"time"
 )
@@ -25,224 +23,80 @@ const minGroupSize = 2
 type scanRec struct {
 	size    int64
 	mtime   time.Time
-	head    [sha256.Size]byte
+	head    string
-	tail    [sha256.Size]byte
+	tail    string
-	content [sha256.Size]byte
+	content string
 	path    string
 }
 
-// reportPhase names one phase of report with its place among all of
-// them, as "[2/2] read", for the phase's progress display. The list
-// here is the one place the phases are counted.
-func reportPhase(name string) string {
-	return phaseLabel([]string{openPhase, "read"}, name)
-}
-
 // runReport implements the report subcommand: it prints the file-level
-// duplicates report as TSV on stdout. It never touches the scanned
+// duplicates report as TSV on stdout. SQLite groups and orders the
-// filesystem; its only I/O is the database, stdout, and stderr. It
+// records, and each row is written as it is read, so no group is held
-// names the database on stderr before it opens it. Any database
+// in memory. It never touches the scanned filesystem; its only I/O is
-// problem, including a missing database, is fatal.
+// the database (with SQLite's temporary sort file), stdout, and stderr.
-//
+// Any database problem, including a missing database, is fatal.
-// The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and read (count the records,
-// then read them in signature order, writing each size's duplicate
-// groups to stdout once that size is read; see readReport). Each
-// display ends before the next phase starts, and before an error is
-// returned to be printed.
 func runReport(ctx context.Context, stdout io.Writer) error {
-	dbPath, source, err := databasePath()
+	dbPath := databasePath()
-	if err != nil {
-		return err
-	}
 
-	fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
-		escapePath(dbPath), source)
-
-	// Opening can wait on a busy database for as long as the busy
-	// timeout.
-	prog := newProgress(progressParams{
-		label: reportPhase(openPhase), total: -1,
-		waitingFor: openingDatabase,
-	})
 	db, err := openReportDatabase(ctx, dbPath)
-
-	prog.finish()
-
 	if err != nil {
 		return err
 	}
 
 	defer func() { _ = db.Close() }()
 
-	// The read phase writes the report to stdout while it reads the
+	out := bufio.NewWriterSize(stdout, ioBufSize)
-	// records, so it may wait on either.
-	prog = newProgress(progressParams{
-		label: reportPhase("read"), unit: "records", total: -1,
-		waitingFor: countingRecords, stalledOn: "db/out",
-	})
-	report, err := readReport(ctx, db, dbPath, stdout, prog)
 
-	prog.finish()
+	_, err = fmt.Fprintln(out, "first\tdupe\tsize")
+	if err != nil {
+		return fmt.Errorf("write stdout: %w", err)
+	}
+
+	var (
+		groups, dupeFiles int
+		reclaimable       int64
+		writeErr          error
+	)
+
+	records, err := loadDupeRows(ctx, db,
+		func(first, path string, size int64) error {
+			// A group's first path is its first row; every other
+			// path is a dupe.
+			if path == first {
+				groups++
+
+				return nil
+			}
+
+			_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
+				escapePath(first), escapePath(path), size)
+			dupeFiles++
+			reclaimable += size
+
+			return writeErr
+		})
+
+	if writeErr != nil {
+		return fmt.Errorf("write stdout: %w", writeErr)
+	}
 
 	if err != nil {
-		return err
+		return fmt.Errorf("database %s: %w", dbPath, err)
+	}
+
+	err = out.Flush()
+	if err != nil {
+		return fmt.Errorf("write stdout: %w", err)
 	}
 
 	fmt.Fprintf(os.Stderr,
 		"report: %d records read, %d duplicate groups, %d dupe files, "+
 			"%s reclaimable\n",
-		report.records, report.dupeGroups, report.dupeFiles,
+		records, groups, dupeFiles, humanBytes(reclaimable))
-		humanBytes(report.reclaimable))
 
 	return nil
 }
 
-// readReport gives prog the number of records in db, the database at
-// dbPath, as its total, then reads every record in signature order,
-// counting each on prog, and writes the report to stdout: the header
-// line, then the rows of each size's duplicate groups once that size is
-// read. stdout may be the terminal prog is drawn on, so the lines go
-// through prog's writer. It returns what it wrote, with the totals for
-// the summary line.
-func readReport(ctx context.Context, db *sql.DB, dbPath string,
-	stdout io.Writer, prog *progress,
-) (*reportWriter, error) {
-	out := prog.writer(stdout)
-	report := &reportWriter{out: out}
-
-	_, err := fmt.Fprintln(out, "first\tdupe\tsize")
-	if err != nil {
-		return nil, fmt.Errorf("write stdout: %w", err)
-	}
-
-	var writeErr error
-
-	err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
-		prog.increment()
-
-		writeErr = report.add(r)
-
-		return writeErr
-	})
-
-	if writeErr != nil {
-		return nil, fmt.Errorf("write stdout: %w", writeErr)
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("database %s: %w", dbPath, err)
-	}
-
-	err = report.finish()
-	if err == nil {
-		err = out.Flush()
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("write stdout: %w", err)
-	}
-
-	return report, nil
-}
-
-// reportWriter writes the report's rows to out from the records, added
-// in signature order, largest size first, as loadSignatureRows reads
-// them. The records sharing a signature come together, and so do the
-// records of one size, so it holds only the duplicate groups of the
-// size being read, and writes them, ordered by first path, once a
-// record of a smaller size comes. It counts what the summary line
-// reports.
-type reportWriter struct {
-	out io.Writer
-
-	sig    scanRec    // the signature being read; its path is not used
-	paths  []string   // the paths read so far with sig
-	groups [][]string // the duplicate groups of sig.size, each sorted
-
-	records, dupeGroups, dupeFiles int
-	reclaimable                    int64
-}
-
-// add takes the next record. One with the signature being read joins
-// its paths; any other ends them, and one of a smaller size first
-// writes the groups of the size before.
-func (w *reportWriter) add(r scanRec) error {
-	w.records++
-
-	if r.size == w.sig.size && r.head == w.sig.head &&
-		r.tail == w.sig.tail && r.content == w.sig.content {
-		w.paths = append(w.paths, r.path)
-
-		return nil
-	}
-
-	w.endGroup()
-
-	if r.size != w.sig.size {
-		err := w.writeGroups()
-		if err != nil {
-			return err
-		}
-	}
-
-	w.sig = r
-	w.paths = append(w.paths, r.path)
-
-	return nil
-}
-
-// endGroup ends the paths read with w.sig: two or more are a duplicate
-// group, held with its paths sorted until writeGroups writes it.
-func (w *reportWriter) endGroup() {
-	if len(w.paths) < minGroupSize {
-		// A path with no duplicate is dropped, and its slice reused.
-		w.paths = w.paths[:0]
-
-		return
-	}
-
-	slices.Sort(w.paths)
-	w.groups = append(w.groups, w.paths)
-	w.paths = nil
-}
-
-// writeGroups writes the duplicate groups held, all of size w.sig.size,
-// ordered by first path: for each group, a row for every path but the
-// first. No two groups share a first path, since no two records share
-// a path.
-func (w *reportWriter) writeGroups() error {
-	slices.SortFunc(w.groups, func(a, b []string) int {
-		return strings.Compare(a[0], b[0])
-	})
-
-	for _, g := range w.groups {
-		for _, path := range g[1:] {
-			_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
-				escapePath(g[0]), escapePath(path), w.sig.size)
-			if err != nil {
-				return err
-			}
-
-			w.dupeFiles++
-			w.reclaimable += w.sig.size
-		}
-
-		w.dupeGroups++
-	}
-
-	w.groups = nil
-
-	return nil
-}
-
-// finish writes the groups of the last size read.
-func (w *reportWriter) finish() error {
-	w.endGroup()
-
-	return w.writeGroups()
-}
-
 // escapePath returns a path as it is written in a report column (README
 // "Report output format"): a backslash, tab, newline or carriage return
 // becomes \\, \t, \n or \r, and every other byte is kept as it is.

report_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -10,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -19,32 +17,18 @@ import (
 // awkwardDir is a directory name holding every byte the reports escape.
 const awkwardDir = "/d/\tone\ntwo\rthree\\four"
 
-// testHash returns a stand-in hash for a test record: the bytes of s,
-// then zeros. Stand-ins sort as their strings do, so a test can choose
-// the order in which report reads its records.
-func testHash(s string) [sha256.Size]byte {
-	var h [sha256.Size]byte
-
-	copy(h[:], s)
-
-	return h
-}
-
 // awkwardPairRecs is a duplicate pair in sibling directories /d/A and
 // awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
 // sorts after it, so awkwardDir coming first shows that sorting uses
 // the raw path.
 func awkwardPairRecs() []scanRec {
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
-
 	return []scanRec{
-		{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
+		{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
-		{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
+		{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
 	}
 }
 
-// seedDatabase writes recs into a fresh database, closes it as scan
+// seedDatabase writes recs into a fresh database and returns its path.
-// does, out of WAL mode, and returns its path.
 func seedDatabase(t *testing.T, recs []scanRec) string {
 	t.Helper()
 
@@ -60,7 +44,10 @@ func seedDatabase(t *testing.T, recs []scanRec) string {
 		t.Fatal(err)
 	}
 
-	closeScanDatabase(t.Context(), db, path)
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	return path
 }
@@ -73,39 +60,25 @@ type dupeGroup struct {
 }
 
 // dupeGroups returns the duplicate groups report reads from db, in
-// report order, from the rows it writes: a group's rows come together,
+// report order.
-// each naming the group's first path.
 func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
 	t.Helper()
 
-	var out bytes.Buffer
-
-	_, err := readReport(t.Context(), db, "", &out, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	var groups []dupeGroup
 
-	rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
+	_, err := loadDupeRows(t.Context(), db,
-	for _, row := range rows[1:] { // after the header
+		func(first, path string, size int64) error {
-		fields := strings.Split(row, "\t")
+			if path == first {
-		if len(fields) != 3 {
+				groups = append(groups, dupeGroup{size: size})
-			t.Fatalf("row %q, want first, dupe and size", row)
+			}
-		}
 
-		size, err := strconv.ParseInt(fields[2], 10, 64)
+			g := &groups[len(groups)-1]
-		if err != nil {
+			g.paths = append(g.paths, path)
-			t.Fatal(err)
-		}
 
-		if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] {
+			return nil
-			groups = append(groups,
+		})
-				dupeGroup{size: size, paths: []string{fields[0]}})
+	if err != nil {
-		}
+		t.Fatal(err)
-
-		g := &groups[len(groups)-1]
-		g.paths = append(g.paths, fields[1])
 	}
 
 	return groups
@@ -145,160 +118,38 @@ func TestRunReportEscapesPaths(t *testing.T) {
 }
 
 func TestReportStdoutFailsWhileReading(t *testing.T) {
-	t.Parallel()
+	// Each row holds two paths longer than dir, so the report is more
-
+	// than twice the stdout buffer and stdout fails while rows are
-	// Every two records are a duplicate pair of a size of their own, and
+	// still being read, not at the final flush.
-	// a pair's row is written once a smaller size is read. Each row
-	// holds two paths longer than dir, so the rows come to more than
-	// twice the stdout buffer, and stdout fails while records are still
-	// being read, not once every record is.
 	dir := "/" + strings.Repeat("d", 4096)
 
-	recs := make([]scanRec, 2*ioBufSize/len(dir))
+	recs := make([]scanRec, ioBufSize/len(dir))
 	for i := range recs {
 		recs[i] = scanRec{
-			size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
+			size: 1, head: "h", tail: "t", content: "c",
-			content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
+			path: fmt.Sprintf("%s/%d", dir, i),
 		}
 	}
 
-	db := openTestDB(t)
+	t.Setenv(databaseEnv, seedDatabase(t, recs))
 
-	err := applyChanges(t.Context(), db, recs, nil, nil)
+	err := runReport(t.Context(), failingWriter{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Counts the records read, and draws nothing.
-	prog := &progress{now: time.Now}
-
-	_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
 	if !errors.Is(err, errWriteFailed) ||
 		!strings.HasPrefix(err.Error(), "write stdout: ") {
 		t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
 	}
-
-	if prog.count >= int64(len(recs)) {
-		t.Errorf("stdout failed after all %d records were read, want it "+
-			"to stop the read", prog.count)
-	}
-}
-
-// reportOpenLine is the plain line of report's open phase when it opens
-// the database at once; it prints it as the phase starts and as it ends.
-const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
-	"remain unknown"
-
-// smokeReport is report's stdout for the smoke-test records.
-const smokeReport = "first\tdupe\tsize\n" +
-	"/d/t1/f1\t/d/t2/f1\t3000\n" +
-	"/d/t1/f1\t/d/t3/f1\t3000\n" +
-	"/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" +
-	"/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n"
-
-// TestRunReportProgressNamesEachPhase runs report on the smoke-test
-// records with stderr not a terminal, and checks every line it prints
-// there. After the line naming the database, each phase prints a line
-// naming it with its place among the two as it starts and another as
-// it ends; read counts the 6 records before it reads them. The summary
-// comes last, and stdout holds the report alone.
-func TestRunReportProgressNamesEachPhase(t *testing.T) {
-	path := seedDatabase(t, smokeTreeRecs())
-	t.Setenv(databaseEnv, path)
-
-	stderr := captureStderr(t)
-
-	var stdout bytes.Buffer
-
-	code := run([]string{cmdReport}, &stdout, os.Stderr)
-	if code != exitOK {
-		t.Fatalf("run(report) = %d, want %d; stderr: %s",
-			code, exitOK, stderr())
-	}
-
-	if got := stdout.String(); got != smokeReport {
-		t.Errorf("stdout = %q, want %q", got, smokeReport)
-	}
-
-	// How each line starts; the rate at the end of a phase varies.
-	want := []string{
-		"report: database " + path + " (from " + databaseEnv + ")",
-		reportOpenLine,
-		reportOpenLine,
-		"[2/2] read: counting the records, elapsed 0s, remain unknown",
-		"[2/2] read: [6/6] 100%, ",
-		"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
-			"6.1 KiB reclaimable",
-	}
-
-	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
-	if len(lines) != len(want) {
-		t.Fatalf("stderr lines = %q, want %d lines starting %q",
-			lines, len(want), want)
-	}
-
-	for i := range want {
-		if !strings.HasPrefix(lines[i], want[i]) {
-			t.Errorf("stderr line %d = %q, want it to start %q",
-				i+1, lines[i], want[i])
-		}
-	}
-}
-
-// TestReadReportOnTerminal writes the smoke-test report to the terminal
-// its read phase's display is drawn on, as a plain "sfdupes report"
-// does: the terminal must show the header and every row whole, each on
-// a line of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestReadReportOnTerminal(t *testing.T) {
-	stderr := captureStderr(t)
-	db := openTestDB(t)
-
-	err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// No timed draws: the display is drawn as it starts and as it ends.
-	p := startProgress(progressParams{
-		label: reportPhase("read"), unit: "records", total: -1,
-		waitingFor: countingRecords,
-	}, terminalWidth, time.Hour, time.Now)
-
-	_, err = readReport(t.Context(), db, "", os.Stderr, p)
-
-	p.finish()
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// A terminal shows each line as the text after its last carriage
-	// return.
-	var shown []string
-
-	for line := range strings.SplitSeq(stderr(), "\n") {
-		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
-	}
-
-	want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n")
-	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
-		t.Errorf("terminal shows lines %q, want %q first", shown, want)
-	}
 }
 
 func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
 	// README §Constraints: identical database contents give identical
 	// output, whatever order the records were inserted in.
-	a, b := testHash("a"), testHash("b")
-
 	recs := append(smokeTreeRecs(), awkwardPairRecs()...)
 	recs = append(recs,
-		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
+		scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
-		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
+		scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
-		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
+		scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
-		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
+		scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
+		scanRec{size: 50, path: "/x/unhashed"},
 	)
 
 	reversed := slices.Clone(recs)
@@ -401,19 +252,16 @@ func TestWarnfEscapes(t *testing.T) {
 func TestDupeGroups(t *testing.T) {
 	t.Parallel()
 
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
-	bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
-	u := testHash("u")
 	recs := []scanRec{
-		{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
-		{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
-		{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
-		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
+		{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
-		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
+		{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
 		// Same size as the /z group but a different head hash.
-		{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
+		{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
 		// A singleton signature must not form a group.
-		{size: 7, head: u, tail: u, content: u, path: "/lonely"},
+		{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -437,13 +285,16 @@ func TestDupeGroups(t *testing.T) {
 func TestDupeGroupsContentSeparates(t *testing.T) {
 	t.Parallel()
 
-	// Same size, head, and tail: /b differs from the others only in its
+	// Same size, head, and tail, but different content hashes: the final
-	// content hash, so it joins no group, while /a and /c group.
+	// rung keeps them apart, so no group forms. Matching content groups.
-	h, tl := testHash("h"), testHash("t")
+	// Records without a content hash never group, not even with each
+	// other.
 	recs := []scanRec{
-		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
+		{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
-		{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
+		{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
-		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
+		{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
+		{size: 100, head: "h", tail: "t", path: "/d"},
+		{size: 100, head: "h", tail: "t", path: "/e"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -462,15 +313,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
 
 	// mtime is informational only; records differing only in mtime
 	// still group together.
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 	recs := []scanRec{
 		{
-			size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
+			size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
-			content: c, path: "/m/1",
+			content: "c", path: "/m/1",
 		},
 		{
-			size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
+			size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
-			content: c, path: "/m/2",
+			content: "c", path: "/m/2",
 		},
 	}
 
@@ -485,12 +335,11 @@ func TestDupeGroupsTieBreak(t *testing.T) {
 
 	// The hashes sort opposite to the first paths, so ordering the
 	// groups by hash instead of by first path fails this test.
-	a, b := testHash("a"), testHash("b")
 	recs := []scanRec{
-		{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
-		{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
-		{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -505,47 +354,14 @@ func TestDupeGroupsTieBreak(t *testing.T) {
 	}
 }
 
-func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
-	t.Parallel()
-
-	// report reads the groups of size 50 in signature order, here /a,
-	// /z, /k, which is neither the order of their first paths nor its
-	// reverse, and must write them ordered by first path. The group of
-	// size 7 comes after them all, though its first path sorts first.
-	a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
-	recs := []scanRec{
-		{size: 50, head: a, tail: a, content: a, path: "/k/2"},
-		{size: 50, head: a, tail: a, content: a, path: "/k/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/z/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/z/2"},
-		{size: 50, head: c, tail: c, content: c, path: "/a/2"},
-		{size: 50, head: c, tail: c, content: c, path: "/a/1"},
-		{size: 7, head: d, tail: d, content: d, path: "/0/2"},
-		{size: 7, head: d, tail: d, content: d, path: "/0/1"},
-	}
-
-	groups := dupeGroupsOf(t, recs)
-
-	got := make([]string, 0, len(groups))
-	for _, g := range groups {
-		got = append(got, g.paths[0])
-	}
-
-	want := []string{"/a/1", "/k/1", "/z/1", "/0/1"}
-	if !slices.Equal(got, want) {
-		t.Errorf("groups' first paths = %q, want %q", got, want)
-	}
-}
-
 func TestDupeGroupsDeterministic(t *testing.T) {
 	t.Parallel()
 
-	a, b := testHash("a"), testHash("b")
 	recs := []scanRec{
-		{size: 1, head: a, tail: a, content: a, path: "/p/1"},
+		{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
-		{size: 1, head: a, tail: a, content: a, path: "/p/2"},
+		{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
-		{size: 2, head: b, tail: b, content: b, path: "/q/1"},
+		{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
-		{size: 2, head: b, tail: b, content: b, path: "/q/2"},
+		{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
 	}
 
 	forward := dupeGroupsOf(t, recs)

script/bootstrap

@@ -2,12 +2,10 @@
 # script/bootstrap: install all dependencies needed to build and develop
 # this repo. Idempotent: every install is guarded by a check so already
 # installed tools are skipped. Base tooling comes from nix, apt, brew,
-# or apk (detected in that order); assumes nothing is present. The
+# or apk (detected in that order); assumes nothing is present. Node is
-# installed node is used only when its major version is the pinned one;
+# used directly if installed; otherwise it is installed at a pinned
-# otherwise (node missing, or another major) the pinned version is
+# version via nvm (installing nvm itself first, from a hash-verified
-# installed via nvm (installing nvm itself first, from a hash-verified
+# release archive, never curl | sh).
-# release archive, never curl | sh), and yarn and the packages are
-# installed under it.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -102,52 +100,31 @@ ensure_nvm() {
     rm -rf "$tmp"
 }
 
-# True when the node on PATH has the pinned major version. Only the
-# major is compared: the Dockerfile stages start from a node 22 alpine
-# image whose exact version is not the pin, and nvm cannot install a
-# prebuilt node on alpine. Another major is not used: the pinned yarn 1
-# prints a deprecation warning under node 24. script/fmt and
-# script/fmt-check carry this function unchanged, to run the yarn
-# installed here.
-node_is_pinned_major() {
-    if ! command -v node >/dev/null 2>&1; then return 1; fi
-    installed="$(node --version)"
-    installed="${installed#v}"
-    [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
-}
-
 ensure_node() {
-    if node_is_pinned_major; then
+    if ! missing node; then return 0; fi
-        echo "bootstrap: using node $(node --version)"
-        return 0
-    fi
     ensure_nvm
     nvm_sh "nvm install $NODE_VERSION"
-    nvm_sh "nvm use $NODE_VERSION >/dev/null && \
-        echo \"bootstrap: using node \$(node --version)\""
 }
 
 ensure_yarn() {
-    if ! node_is_pinned_major; then
-        nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
-            corepack prepare yarn@$YARN_VERSION --activate"
-        return 0
-    fi
     if ! missing yarn; then return 0; fi
     if ! missing corepack; then
         corepack enable
         corepack prepare "yarn@$YARN_VERSION" --activate
+    elif [ -s "$HOME/.nvm/nvm.sh" ]; then
+        nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
+            corepack prepare yarn@$YARN_VERSION --activate"
     else
         npm install -g "yarn@$YARN_VERSION"
     fi
 }
 
 install_js_deps() {
-    if node_is_pinned_major; then
+    if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
-        yarn install --frozen-lockfile
-    else
         nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
             yarn install --frozen-lockfile"
+    else
+        yarn install --frozen-lockfile
     fi
 }
 

script/cibuild

@@ -2,12 +2,9 @@
 # script/cibuild: run the CI build. It bootstraps first: a CI runner
 # checks out and runs this and nothing else, and script/fmt-check runs
 # the formatter on the host, which a pristine checkout cannot do.
-# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
+# --no-cache for the same reason as script/docker: the gate phases the
-# line, since a cached gate phase is a check that did not run; unlike
+# final stage depends on are RUN steps, and a cached one is a check that
-# --no-cache, it keeps the layers above that line and the cache mounts.
+# did not run.
-# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
-# before every step that lints, tests or compiles; a stage without that
-# line is served from cache on an unchanged tree, with no warning.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -25,7 +22,7 @@ main() {
     version="$(git describe --tags --always --dirty 2>/dev/null || true)"
     [ -n "$version" ] || version="unknown"
     tag="$("$SCRIPT_DIR/projectname")"
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+    docker build --no-cache \
         --build-arg VERSION="$version" \
         -t "$tag" .
 }

script/docker

@@ -1,12 +1,8 @@
 #!/bin/sh
 # script/docker: build the Docker image tagged with the project name.
 # Identical in all repos; the tag comes from script/projectname.
-# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
+# --no-cache because the gate phases the final stage depends on are RUN
-# line, since a cached gate phase is a check that did not run; unlike
+# steps, and a cached one is a check that did not run.
-# --no-cache, it keeps the layers above that line and the cache mounts.
-# This relies on every stage of the Dockerfile declaring ARG CACHEBUST
-# before every step that lints, tests or compiles; a stage without that
-# line is served from cache on an unchanged tree, with no warning.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -22,7 +18,7 @@ main() {
     version="$(git describe --tags --always --dirty 2>/dev/null || true)"
     [ -n "$version" ] || version="unknown"
     tag="$("$SCRIPT_DIR/projectname")"
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+    docker build --no-cache \
         --build-arg VERSION="$version" \
         -t "$tag" .
 }

script/fmt

@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 # Must match the pin in script/bootstrap.
 NODE_VERSION="22.17.0"
 
-# True when the node on PATH has the pinned major version: the same
+# script/bootstrap installs node and yarn under nvm and leaves neither
-# function as in script/bootstrap, which uses it to decide where it
+# on the PATH of the shell that called it, so resolve the pinned
-# installs yarn.
+# toolchain here the way bootstrap's own install step does. nvm is a
-node_is_pinned_major() {
+# bash script, hence the subshell.
-    if ! command -v node >/dev/null 2>&1; then return 1; fi
-    installed="$(node --version)"
-    installed="${installed#v}"
-    [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
-}
-
-# Run the yarn script/bootstrap installed: the yarn on PATH when the
-# node on PATH has the pinned major version, and otherwise yarn under
-# the pinned node in nvm, which bootstrap leaves off the PATH of the
-# shell that called it. nvm is a bash script, hence the subshell.
 run_yarn() {
-    if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
+    if command -v yarn >/dev/null 2>&1; then
         exec yarn "$@"
     fi
-    if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
+    if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
         echo "fmt: no yarn; run script/bootstrap first" >&2
         exit 1
     fi

script/fmt-check

@@ -7,25 +7,15 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 # Must match the pin in script/bootstrap.
 NODE_VERSION="22.17.0"
 
-# True when the node on PATH has the pinned major version: the same
+# script/bootstrap installs node and yarn under nvm and leaves neither
-# function as in script/bootstrap, which uses it to decide where it
+# on the PATH of the shell that called it, so resolve the pinned
-# installs yarn.
+# toolchain here the way bootstrap's own install step does. nvm is a
-node_is_pinned_major() {
+# bash script, hence the subshell.
-    if ! command -v node >/dev/null 2>&1; then return 1; fi
-    installed="$(node --version)"
-    installed="${installed#v}"
-    [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ]
-}
-
-# Run the yarn script/bootstrap installed: the yarn on PATH when the
-# node on PATH has the pinned major version, and otherwise yarn under
-# the pinned node in nvm, which bootstrap leaves off the PATH of the
-# shell that called it. nvm is a bash script, hence the subshell.
 run_yarn() {
-    if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then
+    if command -v yarn >/dev/null 2>&1; then
         exec yarn "$@"
     fi
-    if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then
+    if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
         echo "fmt-check: no yarn; run script/bootstrap first" >&2
         exit 1
     fi

script/lint

@@ -5,14 +5,9 @@
 # make its answer untrustworthy.
 #
 # The phase is not the last stage in the file, so it is built only when
-# --target names it. A new CACHEBUST value reruns every RUN step after
+# --target names it. --no-cache because a cached lint layer is a lint
-# the ARG CACHEBUST line, since a cached lint layer is a lint that did
+# that did not run. --output type=cacheonly writes no image, since
-# not run; unlike --no-cache, it keeps the layers above that line and
+# nothing uses one.
-# the cache mounts. This relies on every stage of the Dockerfile
-# declaring ARG CACHEBUST before every step that lints, tests or
-# compiles; a stage without that line is served from cache on an
-# unchanged tree, with no warning.
-# --output type=cacheonly writes no image, since nothing uses one.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -20,7 +15,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+    docker build --no-cache \
         --target lint \
         --output type=cacheonly .
 }

script/test

@@ -2,13 +2,8 @@
 # script/test: run the test suite. Testing is a phase of the Dockerfile
 # and this builds that phase alone, on the same terms as script/lint:
 # --target because a phase that is not the last stage is built only when
-# named, and a new CACHEBUST value because a cached test layer is a test
+# named, and --no-cache because a cached test layer is a test that did
-# that did not run; unlike --no-cache, it keeps the layers above the ARG
+# not run. --output type=cacheonly writes no image, since nothing uses one.
-# CACHEBUST line and the cache mounts. This relies on every stage of the
-# Dockerfile declaring ARG CACHEBUST before every step that lints, tests
-# or compiles; a stage without that line is served from cache on an
-# unchanged tree, with no warning. --output type=cacheonly writes no
-# image, since nothing uses one.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -16,7 +11,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+    docker build --no-cache \
         --target test \
         --output type=cacheonly .
 }

trees.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"crypto/sha256"
-	"database/sql"
 	"fmt"
 	"io"
 	"os"
@@ -24,132 +24,44 @@ type treeNode struct {
 	totalSize int64
 }
 
-// treesPhase names one phase of trees with its place among all of them,
-// as "[2/4] read", for the phase's progress display. The list here is
-// the one place the phases are counted.
-func treesPhase(name string) string {
-	return phaseLabel([]string{openPhase, "read", "group", "write"}, name)
-}
-
 // runTrees implements the trees subcommand: it reads every record from
 // the database in path order, reconstructs the directory hierarchy from
 // the record paths, computes a Merkle-style digest per directory, and
 // prints maximal duplicate-tree groups as TSV on stdout. It never
 // touches the scanned filesystem; its only I/O is the database, stdout,
-// and stderr. It names the database on stderr before it opens it. Any
+// and stderr. Any database problem, including a missing database, is
-// database problem, including a missing database, is fatal.
+// fatal.
-//
-// The work runs in four phases, each under a progress display of its
-// own on stderr: open (open the database), read (count the records,
-// then read them into the hierarchy), group (group the directories by
-// digest) and write (write the groups to stdout). Each display ends
-// before the next phase starts, and before an error is returned to be
-// printed.
 func runTrees(ctx context.Context, stdout io.Writer) error {
-	dbPath, source, err := databasePath()
+	dbPath := databasePath()
-	if err != nil {
-		return err
-	}
 
-	fmt.Fprintf(os.Stderr, "trees: database %s (%s)\n",
-		escapePath(dbPath), source)
-
-	// Opening can wait on a busy database for as long as the busy
-	// timeout.
-	prog := newProgress(progressParams{
-		label: treesPhase(openPhase), total: -1,
-		waitingFor: openingDatabase,
-	})
 	db, err := openReportDatabase(ctx, dbPath)
-
-	prog.finish()
-
 	if err != nil {
 		return err
 	}
 
 	defer func() { _ = db.Close() }()
 
-	prog = newProgress(progressParams{
+	records := 0
-		label: treesPhase("read"), unit: "records", total: -1,
+	tree := newTreeBuilder()
-		waitingFor: countingRecords, stalledOn: "db",
+
+	err = loadFileRows(ctx, db, func(r scanRec) {
+		records++
+
+		tree.add(r)
 	})
-	records, super, allDirs, err := readTree(ctx, db, prog)
-
-	prog.finish()
-
 	if err != nil {
 		return fmt.Errorf("database %s: %w", dbPath, err)
 	}
 
-	// Grouping works in memory, so it waits on nothing but the
-	// processor.
-	prog = newProgress(progressParams{
-		label: treesPhase("group"), unit: "directories",
-		total: int64(len(allDirs)), stalledOn: "cpu",
-	})
-	dupes := collectTreeGroups(allDirs, super, prog)
-
-	prog.finish()
-
-	prog = newProgress(progressParams{
-		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
-		stalledOn: "out",
-	})
-	dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
-
-	prog.finish()
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Fprintf(os.Stderr,
-		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
-			"%s reclaimable\n",
-		records, len(dupes), dupeTrees, humanBytes(reclaimable))
-
-	return nil
-}
-
-// readTree gives prog the number of records in db as its total, then
-// reads every record in path order into the directory hierarchy,
-// counting each on prog. It returns the number of records read, the
-// super-root and every directory.
-func readTree(ctx context.Context, db *sql.DB,
-	prog *progress,
-) (int, *treeNode, []*treeNode, error) {
-	records := 0
-	tree := newTreeBuilder()
-
-	err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
-		records++
-
-		prog.increment()
-		tree.add(r)
-	})
-	if err != nil {
-		return 0, nil, nil, err
-	}
-
 	super, allDirs := tree.finish()
 
-	return records, super, allDirs, nil
+	dupes := collectTreeGroups(allDirs, super)
-}
 
-// writeTreeGroups writes dupes to stdout as TSV, a header line and then
+	out := bufio.NewWriterSize(stdout, ioBufSize)
-// a row for every member of each group but its first, counting each
-// group on prog once its rows are written. stdout may be the terminal
-// prog is drawn on, so the lines go through prog's writer. It returns
-// the number of rows (dupe trees) and the bytes they could reclaim.
-func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
-	prog *progress,
-) (int, int64, error) {
-	out := prog.writer(stdout)
 
-	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
+	_, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
 	if err != nil {
-		return 0, 0, fmt.Errorf("write stdout: %w", err)
+		return fmt.Errorf("write stdout: %w", err)
 	}
 
 	dupeTrees := 0
@@ -163,22 +75,25 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 				escapePath(first.path), escapePath(n.path),
 				first.fileCount, first.totalSize)
 			if err != nil {
-				return 0, 0, fmt.Errorf("write stdout: %w", err)
+				return fmt.Errorf("write stdout: %w", err)
 			}
 
 			dupeTrees++
 			reclaimable += first.totalSize
 		}
-
-		prog.increment()
 	}
 
 	err = out.Flush()
 	if err != nil {
-		return 0, 0, fmt.Errorf("write stdout: %w", err)
+		return fmt.Errorf("write stdout: %w", err)
 	}
 
-	return dupeTrees, reclaimable, nil
+	fmt.Fprintf(os.Stderr,
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
+			"%s reclaimable\n",
+		records, len(dupes), dupeTrees, humanBytes(reclaimable))
+
+	return nil
 }
 
 // treeBuilder reconstructs the directory hierarchy from records added
@@ -286,23 +201,28 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
 // name and its signature (size, head, tail, content); mtime is
 // excluded.
 func fileEntry(name string, r scanRec) string {
+	content := r.content
+
+	// A record without a content hash has unknown content (README
+	// "Database"): give it a signature no other file can share, so
+	// trees containing it never compare equal. Real hashes are hex, so
+	// the NUL-prefixed form cannot collide.
+	if content == "" {
+		content = "unhashed\x00" + r.path
+	}
+
 	return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
-		"\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) +
+		"\x00" + r.head + "\x00" + r.tail + "\x00" + content
-		"\x00" + string(r.content[:])
 }
 
-// collectTreeGroups groups directories by digest, counting each one on
+// collectTreeGroups groups directories by digest and returns every
-// prog as it is grouped, and returns every maximal group with two or
+// maximal group with two or more members, each group's members sorted
-// more members, each group's members sorted by path, groups ordered by
+// by path, groups ordered by tree size descending then by first path
-// tree size descending then by first path ascending.
+// ascending.
-func collectTreeGroups(allDirs []*treeNode, super *treeNode,
+func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
-	prog *progress,
-) [][]*treeNode {
 	groups := make(map[[sha256.Size]byte][]*treeNode)
 	for _, d := range allDirs {
 		groups[d.digest] = append(groups[d.digest], d)
-
-		prog.increment()
 	}
 
 	var dupes [][]*treeNode
@@ -338,8 +258,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 // digest is the SHA-256 of its child entries — files serialized with
 // name and signature, subdirectories with name and recursive digest —
 // sorted byte-lexicographically. Filenames cannot contain NUL or "/",
-// and every hash and digest is 32 bytes, so NUL delimiters are
+// so NUL delimiters are unambiguous.
-// unambiguous.
 func (n *treeNode) computeDigest() {
 	slices.Sort(n.entries)
 

trees_test.go

@@ -3,25 +3,23 @@ package main
 import (
 	"bytes"
 	"database/sql"
-	"os"
 	"slices"
-	"strings"
 	"testing"
-	"time"
 )
 
-// treesOpenLine is the plain line of trees' open phase when it opens
+// Signature hashes shared by the smoke-test records.
-// the database at once; it prints it as the phase starts and as it ends.
+const (
-const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
+	f1Head    = "f1h"
-	"remain unknown"
+	f1Tail    = "f1t"
+	f1Content = "f1c"
+	f2Head    = "f2h"
+	f2Tail    = "f2t"
+	f2Content = "f2c"
+)
 
-// smokeTreeRecs mirrors the tree of the test under README "Definition
+// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
-// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
+// /d/t2 are identical, /d/t3 differs from them only by one filename.
-// by one filename.
 func smokeTreeRecs() []scanRec {
-	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
-	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
-
 	return []scanRec{
 		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
 		{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
@@ -40,7 +38,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
 
 	tree := newTreeBuilder()
 
-	err := loadFileRows(t.Context(), db, func(int64) {}, tree.add)
+	err := loadFileRows(t.Context(), db, tree.add)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -147,14 +145,13 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
 
 	for _, top := range []string{"/a", "/c"} {
 		for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
-			content := testHash("c")
+			content := "c"
 			if p == "/b-x/f" {
-				content = testHash("other")
+				content = "other"
 			}
 
 			recs = append(recs, scanRec{
-				size: 1, head: testHash("h"), tail: testHash("t"),
+				size: 1, head: "h", tail: "t", content: content, path: top + p,
-				content: content, path: top + p,
 			})
 		}
 	}
@@ -173,7 +170,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
 		t.Fatalf("directory paths = %q, want %q", got, want)
 	}
 
-	groups := collectTreeGroups(dirs, super, nil)
+	groups := collectTreeGroups(dirs, super)
 
 	gotGroups := groupPaths(groups)
 	wantGroups := [][]string{{"/a", "/c"}}
@@ -206,98 +203,6 @@ func TestRunTreesEscapesPaths(t *testing.T) {
 	}
 }
 
-// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
-// records with stderr not a terminal, and checks every line it prints
-// there. After the line naming the database, each phase prints a line
-// naming it with its place among the four as it starts and another as
-// it ends. read counts the 6 records before it reads them, group counts
-// the 8 directories, and write the 1 group. The summary comes last, and
-// stdout holds the report alone.
-func TestRunTreesProgressNamesEachPhase(t *testing.T) {
-	path := seedDatabase(t, smokeTreeRecs())
-	t.Setenv(databaseEnv, path)
-
-	stderr := captureStderr(t)
-
-	var stdout bytes.Buffer
-
-	code := run([]string{cmdTrees}, &stdout, os.Stderr)
-	if code != exitOK {
-		t.Fatalf("run(trees) = %d, want %d; stderr: %s",
-			code, exitOK, stderr())
-	}
-
-	report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
-	if got := stdout.String(); got != report {
-		t.Errorf("stdout = %q, want %q", got, report)
-	}
-
-	// How each line starts; the rate at the end of a phase varies.
-	want := []string{
-		"trees: database " + path + " (from " + databaseEnv + ")",
-		treesOpenLine,
-		treesOpenLine,
-		"[2/4] read: counting the records, elapsed 0s, remain unknown",
-		"[2/4] read: [6/6] 100%, ",
-		"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
-		"[3/4] group: [8/8] 100%, ",
-		"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
-		"[4/4] write: [1/1] 100%, ",
-		"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
-	}
-
-	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
-	if len(lines) != len(want) {
-		t.Fatalf("stderr lines = %q, want %d lines starting %q",
-			lines, len(want), want)
-	}
-
-	for i := range want {
-		if !strings.HasPrefix(lines[i], want[i]) {
-			t.Errorf("stderr line %d = %q, want it to start %q",
-				i+1, lines[i], want[i])
-		}
-	}
-}
-
-// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
-// terminal its write phase's display is drawn on, as a plain "sfdupes
-// trees" does: the terminal must show the header and the row whole,
-// each on a line of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestWriteTreeGroupsOnTerminal(t *testing.T) {
-	stderr := captureStderr(t)
-	super, dirs := treeOf(t, smokeTreeRecs())
-	dupes := collectTreeGroups(dirs, super, nil)
-
-	// No timed draws: the display is drawn as it starts and as it ends.
-	p := startProgress(progressParams{
-		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
-	}, terminalWidth, time.Hour, time.Now)
-
-	_, _, err := writeTreeGroups(os.Stderr, dupes, p)
-
-	p.finish()
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// A terminal shows each line as the text after its last carriage
-	// return.
-	var shown []string
-
-	for line := range strings.SplitSeq(stderr(), "\n") {
-		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
-	}
-
-	want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
-	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
-		t.Errorf("terminal shows lines %q, want %q first", shown, want)
-	}
-}
-
 func TestTreeDigests(t *testing.T) {
 	t.Parallel()
 
@@ -327,10 +232,11 @@ func TestTreeDigests(t *testing.T) {
 func TestTreeDigestContentSensitivity(t *testing.T) {
 	t.Parallel()
 
-	same, c := testHash("same"), testHash("c")
+	const sharedTail = "same"
+
 	recs := []scanRec{
-		{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
+		{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
-		{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
+		{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
 	}
 
 	_, dirs := treeOf(t, recs)
@@ -348,7 +254,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
 
 	super, dirs := treeOf(t, smokeTreeRecs())
 
-	groups := collectTreeGroups(dirs, super, nil)
+	groups := collectTreeGroups(dirs, super)
 
 	want := [][]string{{"/d/t1", "/d/t2"}}
 	if got := groupPaths(groups); !slices.EqualFunc(got, want,
@@ -372,14 +278,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
 
 	super, dirs := treeOf(t, recs)
 
-	forward := groupPaths(collectTreeGroups(dirs, super, nil))
+	forward := groupPaths(collectTreeGroups(dirs, super))
 
 	reversed := slices.Clone(recs)
 	slices.Reverse(reversed)
 
 	superR, dirsR := treeOf(t, reversed)
 
-	backward := groupPaths(collectTreeGroups(dirsR, superR, nil))
+	backward := groupPaths(collectTreeGroups(dirsR, superR))
 	if !slices.EqualFunc(forward, backward, slices.Equal) {
 		t.Fatalf("output depends on record order: %v vs %v",
 			forward, backward)
@@ -391,15 +297,14 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
 
 	// Identical sibling dirs share a parent, so their group cannot be
 	// implied by a parent group and must be reported.
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 	recs := []scanRec{
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
 	}
 
 	super, dirs := treeOf(t, recs)
 
-	got := groupPaths(collectTreeGroups(dirs, super, nil))
+	got := groupPaths(collectTreeGroups(dirs, super))
 
 	want := [][]string{{"/p/x1", "/p/x2"}}
 	if !slices.EqualFunc(got, want, slices.Equal) {
@@ -413,16 +318,15 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 	// /p/a and /q/b contain identical x subtrees, but /p/a has an
 	// extra file, so the parents' digests differ and the x group must
 	// be reported.
-	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
 	recs := []scanRec{
-		{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
-		{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
+		{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
-		{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
 	}
 
 	super, dirs := treeOf(t, recs)
 
-	got := groupPaths(collectTreeGroups(dirs, super, nil))
+	got := groupPaths(collectTreeGroups(dirs, super))
 
 	want := [][]string{{"/p/a/x", "/q/b/x"}}
 	if !slices.EqualFunc(got, want, slices.Equal) {