scan: show each phase as N of M, with progress, elapsed time and ETA, also while it waits (closes #112)

Every scan display now starts with its phase and its place among the
five, as "[2/5] walk", and shows its progress, rate, elapsed time and
ETA. On a terminal hash, update and the content reads keep the
library's bar, which fills what the rest of the line leaves; load,
the walk and the content checks are a spinner whose text is cut to
the terminal's width. load counts the records first. The content
phase adds up the sizes of the matching groups, without the join and
sort, and says it is searching until the first record returns. The
walk's total is estimated from the records under the operands. Each
display is drawn from its own goroutine: several times a second on a
terminal, every 5 seconds in plain mode.

Judgement call: load and the content checks stay spinners once their totals are known.

Model: opus-5-5

Dockerfile

@@ -1,23 +1,16 @@
-# In every stage ARG CACHEBUST comes right after the steps that install
+# Every stage declares ARG CACHEBUST first: script/ passes a new value
-# system packages and download Go's modules, or right after FROM in a
+# on every build, which reruns every RUN step after it. The steps that
-# stage with neither, and every step that lints, tests or compiles
+# compile mount Go's build cache, which survives from build to build.
-# comes after it. script/ passes a new value on every build, which
-# reruns every RUN step after it. Docker keeps the layers above it
-# until the base image, their own lines or go.mod and go.sum change.
-# The steps that compile mount Go's build cache, and golangci-lint run
-# also mounts golangci-lint's; both survive from build to build. The
-# test phase runs as nobody and so mounts Go's build cache at
-# /tmp/.cache/go-build; its own comment says why.
 
 # Lint phase, built alone by script/lint. The tools are invoked directly
 # rather than through `make lint`, which runs docker itself and so cannot
 # run inside a build step.
 # golangci/golangci-lint:v2.14.0, 2026-10-07
 FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
+ARG CACHEBUST
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
-ARG CACHEBUST
 COPY . .
 
 # The gofmt half of `make fmt-check`. gofmt's output is assigned to a
@@ -31,7 +24,6 @@ RUN files="$(gofmt -s -l .)" && \
 # Validates .golangci.yml against the schema the pinned binary embeds.
 RUN golangci-lint config verify --config .golangci.yml
 RUN --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/root/.cache/golangci-lint \
     golangci-lint run --config .golangci.yml ./...
 
 # Test phase, built alone by script/test. -race needs cgo and so a C
@@ -46,12 +38,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
 # nobody (65534) so the tests can write to it.
 # golang:1.25-trixie, 2026-10-04
 FROM golang@sha256:2c4c60ef415fbfa5e90300722293bef36c5e63fae17570ce18f580af933dbd73 AS test
+ARG CACHEBUST
 USER nobody
 ENV HOME=/tmp
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
-ARG CACHEBUST
 COPY . .
 RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
     go test -count=1 -timeout 90s -race -cover ./... || \
@@ -59,19 +51,19 @@ RUN --mount=type=cache,target=/tmp/.cache/go-build,uid=65534,gid=65534 \
       go test -count=1 -timeout 90s -race -v ./...; exit 1; }
 
 # Build stage. Nothing is wanted from either phase above; the copies are
-# what make BuildKit build them first, so nothing after them can run
+# what make BuildKit build them first, so this stage cannot run unless
-# unless lint and test passed.
+# lint and test passed.
 # golang:1.25-alpine, 2026-07-23
 FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
-RUN apk add --no-cache git make
-WORKDIR /src
-COPY go.mod go.sum ./
-RUN go mod download
 ARG CACHEBUST
 COPY --from=lint /src/go.sum /dev/null
 COPY --from=test /src/go.sum /dev/null
+RUN apk add --no-cache git make
 # A tar-stream context keeps the sender's file owners, which git refuses.
 RUN git config --system --add safe.directory /src
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
 COPY . .
 
 # The version stamped into the binary: the VERSION build argument when

REPO_POLICIES.md

@@ -1,6 +1,6 @@
 ---
 title: Repository Policies
-last_modified: 2026-10-18
+last_modified: 2026-10-15
 ---
 
 This document covers repository structure, tooling, and workflow standards. Code
@@ -159,11 +159,8 @@ style conventions are in separate documents:
     one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
     exit non-zero with `parallel golangci-lint is running` — a status a caller
     cannot tell from real findings. Both have produced wrong verdicts in this
-    org, in both directions. A container has its own `TMPDIR`, so the lock is
+    org, in both directions. A container has its own golangci-lint cache, its
-    out of reach. The lint phase does keep golangci-lint's cache between check
+    own `TMPDIR` and a digest-pinned binary, so neither is reachable.
-    builds (below), but it lints every checkout at `/src`, and a finding is
-    taken from that cache only for the same files, settings and golangci-lint
-    version.
 
 - **Any build that runs checks is given a new `CACHEBUST` value.** Docker
   invalidates a `COPY` layer only when the copied content changes, so on an
@@ -173,21 +170,15 @@ style conventions are in separate documents:
   id, so no two builds share a value, not even the three in one
   `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
   `script/docker` are the four, and there is no fifth — `script/check` runs the
-  two gate phases and `script/fmt-check`, and builds no image of its own. In
+  two gate phases and `script/fmt-check`, and builds no image of its own. Every
-  every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
+  stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after
-  that install system packages and download Go's modules, or right after `FROM`
+  `FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps
-  in a stage with neither, and every step that lints, tests or compiles comes
+  are compared by content and need nothing. A stage without the `ARG` line is
-  after it. A step that installs anything else as well, such as a
+  served from cache on an unchanged tree, so a repository re-vendors the scripts
-  `script/bootstrap` that also installs node packages, comes after it too. A new
+  and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`:
-  value reruns every `RUN` step after that line; `COPY` steps are compared by
+  it reruns every step too, but it also empties every cache mount, and Go's
-  content and need nothing. Docker keeps the layers above the line until the
+  build cache, which the canonical Go `Dockerfile` below mounts so that compiled
-  base image, their own lines or the files they copy change. A stage without the
+  packages survive between check builds, would start empty every time. A bare
-  `ARG` line is served from cache on an unchanged tree, so a repository
-  re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
-  not pass `--no-cache`: it reruns every step too, but it also builds the layers
-  above the line again and empties every cache mount, so Go's build cache and
-  golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
-  survive between check builds, would start empty every time. A bare
   `docker build .` is not evidence that anything ran: a sub-second build
   reporting success is a cache hit, not a result. Never invalidate by pruning —
   `docker builder prune` and friends destroy a build cache shared with every
@@ -203,25 +194,19 @@ style conventions are in separate documents:
   `Dockerfile`:
 
     ```dockerfile
-    # In every stage ARG CACHEBUST comes right after the steps that install
+    # Every stage declares ARG CACHEBUST first: script/ passes a new value
-    # system packages and download Go's modules, or right after FROM in a
+    # on every build, which reruns every RUN step after it. The steps that
-    # stage with neither, and every step that lints, tests or compiles
+    # compile mount Go's build cache, which survives from build to build.
-    # comes after it. script/ passes a new value on every build, which
-    # reruns every RUN step after it. Docker keeps the layers above it
-    # until the base image, their own lines or go.mod and go.sum change.
-    # The steps that compile mount Go's build cache, and golangci-lint run
-    # also mounts golangci-lint's; both survive from build to build.
 
     # Lint phase
     # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
     FROM golangci/golangci-lint@sha256:... AS lint
+    ARG CACHEBUST
     WORKDIR /src
     COPY go.mod go.sum ./
     RUN go mod download
-    ARG CACHEBUST
     COPY . .
     RUN --mount=type=cache,target=/root/.cache/go-build \
-        --mount=type=cache,target=/root/.cache/golangci-lint \
         golangci-lint run --config .golangci.yml ./...
 
     # Test phase. -race needs cgo and so a C compiler, which the Debian Go
@@ -229,10 +214,10 @@ style conventions are in separate documents:
     # cache also holds test results, which go test would otherwise replay.
     # golang:1.x, YYYY-MM-DD
     FROM golang@sha256:... AS test
+    ARG CACHEBUST
     WORKDIR /src
     COPY go.mod go.sum ./
     RUN go mod download
-    ARG CACHEBUST
     COPY . .
     RUN --mount=type=cache,target=/root/.cache/go-build \
         go test -count=1 -timeout 90s -race -cover ./... || \
@@ -240,19 +225,19 @@ style conventions are in separate documents:
           go test -count=1 -timeout 90s -race -v ./...; exit 1; }
 
     # Build stage. Nothing is wanted from either phase above; the copies
-    # are what make BuildKit build them first, so nothing after them can
+    # are what make BuildKit build them first, so this stage cannot run
-    # run unless lint and test passed.
+    # unless lint and test passed.
     # golang:1.x-alpine, YYYY-MM-DD
     FROM golang@sha256:... AS builder
-    RUN apk add --no-cache git
-    WORKDIR /src
-    COPY go.mod go.sum ./
-    RUN go mod download
     ARG CACHEBUST
     COPY --from=lint /src/go.sum /dev/null
     COPY --from=test /src/go.sum /dev/null
+    RUN apk add --no-cache git
     # A tar-stream context keeps the sender's file owners, which git refuses.
     RUN git config --system --add safe.directory /src
+    WORKDIR /src
+    COPY go.mod go.sum ./
+    RUN go mod download
     COPY . .
 
     # The VERSION build arg when one is given, otherwise
@@ -289,26 +274,16 @@ style conventions are in separate documents:
     - Keep the runtime stage last, and if you add a stage after it, give it the
       same two copies. A plain `docker build .` builds the last stage's chain
       and nothing else.
-    - Check builds keep these, and none of them can hand back a stale result:
+    - Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps
-        - The layers above `ARG CACHEBUST`: the system packages and Go's
+      it for root), and only on the steps that compile: `golangci-lint run`,
-          modules. Docker installs the packages again when the base image or the
+      both `go test` runs and `go build`. It keeps compiled packages between
-          install line changes, and downloads the modules again when `go.mod` or
+      check builds, and Go reuses one only when all its inputs are unchanged, so
-          `go.sum` changes. Go checks every module against `go.sum` as it
+      it saves the compile without skipping a check. Nothing else is kept:
-          downloads it, and no later build can change a kept layer. The modules
+      golangci-lint's own cache would replay stored findings in place of a lint
-          are kept in a layer rather than a cache mount because a cache mount is
+      run, and Go builds from whatever files a kept module cache holds without
-          shared by every build on the host, and Go builds from whatever files
+      checking them again (only `go mod verify` does), so a file changed there
-          it holds without checking them again (only `go mod verify` does), so a
+      would go unnoticed. Go's build cache also holds test results, which is why
-          file changed there would go unnoticed.
+      both `go test` runs carry `-count=1`.
-        - Go's build cache (`/root/.cache/go-build`, where Go keeps it for
-          root), mounted on the steps that compile: `golangci-lint run`, both
-          `go test` runs and `go build`. Go reuses a compiled package only when
-          all its inputs are unchanged. The cache also holds test results, which
-          is why both `go test` runs carry `-count=1`.
-        - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
-          image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
-          a package's findings only when the package's files and those of every
-          package it imports, `go.mod`, the linters and their settings in
-          `.golangci.yml`, and the golangci-lint version are all unchanged.
     - If the project uses `//go:embed` directives that reference build artifacts
       (e.g. a web frontend compiled in a separate stage), the lint phase must
       create placeholder files so the embed directives resolve. Example:
@@ -317,20 +292,12 @@ style conventions are in separate documents:
       in the lint phase. The `golangci/golangci-lint` image is Debian-based and
       has no `apk`, so install with `apt-get` under the Debian package name
       (`libvips-dev`, where alpine says `vips-dev`), and delete the package
-      lists in the same `RUN`, so the layer does not keep them. The test phase,
+      lists in the same `RUN`, so the layer does not keep them:
-      on the Debian Go image, installs the libraries its tests need the same
-      way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
-      Docker keeps it until the base image or the install line changes:
 
         ```dockerfile
-        FROM golangci/golangci-lint@sha256:... AS lint
         RUN apt-get update \
             && apt-get install -y --no-install-recommends libvips-dev \
             && rm -rf /var/lib/apt/lists/*
-        WORKDIR /src
-        COPY go.mod go.sum ./
-        RUN go mod download
-        ARG CACHEBUST
         ```
 
     - `.dockerignore` lets `.git` into the build context. It keeps out every git

TODO.md

@@ -29,166 +29,17 @@
 
 # Completed Steps
 
-- each record holds `disk_size`, the bytes the file takes on disk: its allocated
-  blocks, `st_blocks` × 512, from the lstat the walk already does. `trees`
-  prints a `disk_size` column after `size`, the sum over the `dupe` copy's
-  files; on a compressed or sparse file it differs from `size`. A rescan updates
-  the `disk_size` of an unchanged file whose size on disk changed, without
-  reading the file (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/147)
-
-- the database keeps the number of records and the sum of their sizes in the one
-  row of a `totals` table, which three triggers on `files` keep current. `info`
-  reads that row, and so do `scan`'s `load` phase and the `read` phases of
-  `report` and `trees` for their totals; no count reads every record any more.
-  The triggers make each write to `files` slower. `scan` keeps SQLite's
-  temporary data in memory, which wins back part of that time (2026-10-27,
-  https://git.eeqj.de/sneak/sfdupes/issues/146)
-
-- an operand at or inside a directory named `.zfs` is never scanned, whether
-  given alone or under another operand. `scan` warns about it, counts it as
-  skipped and does not walk it (2026-10-26,
-  https://git.eeqj.de/sneak/sfdupes/issues/142)
-
-- a file or directory removed while the scan runs is not an error, in the walk
-  or in the hash phase. The scan prints no line for it and does not count it as
-  skipped. When the first of several hard links to a file is gone, the hash
-  phase reads the next one instead. On a terminal every progress display starts
-  with the spinner, the hash phase's included. A phase with an exact total ends
-  its line with its percent and a 10-column bar at the terminal's right edge,
-  which no longer moves as the line's length changes. A line too wide for the
-  terminal now loses its rate before its bar. The time remaining is labelled
-  `remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
-
-- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
-  bytes of a SHA-256, half the size of the hex text they held. The hashes stay
-  raw from the hash phase through the database to `report` and `trees`. Reading
-  a record whose hash is not 32 bytes is an error (2026-10-26,
-  https://git.eeqj.de/sneak/sfdupes/issues/134)
-
-- the hash phase gives every file the walk records three hashes: `head` and
-  `tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
-  of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
-  The 10 MiB and 50 MiB thresholds, the size census, the records stored without
-  hashes and the content phase are gone, so a scan has four phases and a rescan
-  reads only new and changed files. `info` no longer counts the records with
-  hashes, since every record has them (2026-10-25,
-  https://git.eeqj.de/sneak/sfdupes/issues/132)
-
-- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
-  `sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
-  stages of the `Dockerfile` the module download, and in `builder` the
-  system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
-  until the base image, the install line, `go.mod` or `go.sum` changes, and
-  every step that lints, tests or compiles still comes after it;
-  `golangci-lint run` also keeps golangci-lint's cache. README says the same
-  (2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
-
-- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
-  `go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
-  passed the build; README "Install" says so, says what `@main` and `v0.0.1`
-  give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
-  early build without a database. When the linker set no version,
-  `sfdupes --version` prints the main module's version that Go recorded in the
-  binary, the tag or the pseudo-version of a `go install` build, and `dev` only
-  when neither is known (2026-10-17,
-  https://git.eeqj.de/sneak/sfdupes/issues/116)
-
-- `scan` creates the database's directory where the database path leads: the
-  path with its last element cut off as text, never cleaned, so a `..` after a
-  symlinked directory is followed through the symlink, as when the file is
-  opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
-  symlink to `/data/x`, it creates `/data/new` and the database in it, where it
-  used to create `/srv/new` and then fail (2026-10-17,
-  https://git.eeqj.de/sneak/sfdupes/issues/119)
-
-- `scan` never records its own database: once the database and its `.lock` file
-  are open, it notes the device and inode of the database file, its `-wal` and
-  `-shm` files and its `.lock` file, and the walk passes over a file with one of
-  them wherever the operands reach it, through a symlinked directory included,
-  printing no line and counting nothing. A record an earlier scan left for one
-  of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
-
-- `scan` never records or reads anything that is not on a persistent filesystem
-  (`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
-  matched by the device ID of each such filesystem, listed as the scan starts
-  from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
-  however the walk or the content phase reaches it; no flag turns that off.
-  Below an operand, its walk does not enter the default excludes either, so
-  `sfdupes scan /` skips the kernel's view of the running system and the
-  temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
-  `/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
-  `/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
-  Windows list, not used yet. An operand at or inside a default exclude is still
-  walked. The scan prints a line for each path it leaves out, without counting
-  it as skipped, and the records there count as outside the scanned operands, as
-  for `-x`. `--no-default-excludes` turns off the default excludes only, and
-  `scan --help` lists them (2026-10-17,
-  https://git.eeqj.de/sneak/sfdupes/issues/124)
-
-- the rate a progress display shows covers the items completed over the last 5
-  seconds, or since the line before when stderr is not a terminal, so it falls
-  to 0 while nothing completes, and a burst after a wait no longer reads as a
-  steady rate of over 1,000 files a second; the ETA still comes from the phase's
-  average. Once no item has completed for 10 seconds, the word `stalled` and
-  where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
-  `db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
-  either, each phase giving its word where it is defined. A line too wide for
-  the terminal loses its bar first, then its rate or those words, so the phase,
-  progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
-  like those of README's example, the content phase's checks and reads lose
-  those words at most times in minutes, and every other phase keeps them
-  whenever the elapsed time and the ETA are both in minutes. The content phase's
-  last partial batch is now written under a display of its own (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/125)
-
-- new subcommand `info` prints the database's details on stdout, one per line as
-  a name, a tab and the value: its path and where it came from, the sizes of the
-  database file and of its `-wal` file when there is one, when the database file
-  was last changed, the schema version, the number of records, how many have
-  head and tail hashes and how many a content hash, and the sizes of all records
-  added up. It opens the database as `report` does, and shows two phases on
-  stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
-  from one read-only transaction, the counts from one query (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/117)
-
-- `report` shows its work on stderr in two phases, `[1/2] open` and
-  `[2/2] read`, with the progress display `scan` and `trees` use, from the
-  moment it starts until the summary line: `open` says it is opening the
-  database, also while it waits on a busy one; `read` counts the records, then
-  reads them against that count in signature order, which SQLite takes from the
-  `files_signature` index without sorting, so the first record comes at once.
-  `report` forms the duplicate groups itself, holding only those of the size it
-  is reading, and writes them ordered by first path before it reads the next
-  size. stdout and the summary line are unchanged, and SQLite no longer needs a
-  temporary file to sort in (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/114)
-
-- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
-  `[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
-  from the moment it starts until the summary line: `open` says it is opening
-  the database, also while it waits on a busy one; `read` counts the records,
-  then reads them against that count; `group` counts the directories it groups,
-  and `write` the groups it writes. A display's rate names what its phase
-  counts: files, records, directories or groups. stdout and the summary line are
-  unchanged. When stdout and stderr are the same file, as one terminal is, none
-  of stdout's lines shares a line with the display; when they are not, the
-  display keeps being drawn while stdout waits on a slow reader (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/113)
-
 - each `scan` phase's progress display names the phase and its place among the
   five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
   time and ETA: `load` counts the records first, the walk's total is estimated
   from the records under the operands, and the content phase counts the records
   it checks from SQLite's grouping alone, then says it is searching the database
-  until the first one is returned. On a terminal a phase with an exact total
+  until the first one is returned. On a terminal `hash`, `update` and the
-  draws its line followed by the library's bar, and any other a spinner followed
+  content phase's reads keep the library's bar and the other displays are a
-  by the line; every display is drawn again several times a second. No line
+  spinner followed by the line; every display is drawn again several times a
-  reaches the terminal's last column, however narrow, also after the terminal is
+  second, and every line fits the terminal's width. In plain mode the line is
-  made narrower during a phase: a line too wide leaves out its rate, then its
+  printed every 5 seconds whether or not an item completed (2026-10-16,
-  bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
+  https://git.eeqj.de/sneak/sfdupes/issues/112)
-  only when those alone do not fit, as below about 70 columns late in a large
-  scan. In plain mode the line is printed every 5 seconds whether or not an item
-  completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
 
 - `scan`, `report` and `trees` first print a line on stderr naming the database
   they use, as an absolute path, and whether that path is the default or comes
@@ -208,11 +59,7 @@
   and the build stage installs `git` and `make` itself; a new push cancels the
   workflow's older run on the same branch, and a run stops after 20 minutes;
   `.claude/settings.json` is deleted (2026-10-15,
-  https://git.eeqj.de/sneak/sfdupes/issues/95). Since
+  https://git.eeqj.de/sneak/sfdupes/issues/95)
-  https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
-  `test` and the build stage, and the install of `git` and `make` in the build
-  stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
-  every step that lints, tests or compiles still runs on every build
 
 - with `-x`, an operand under another operand is walked on its own instead of
   being dropped when a directory from it (from its directory, for a regular
@@ -413,9 +260,9 @@
   a `CHECK_EPOCH` build argument; since
   https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
   passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
-  declares before every step that lints, tests or compiles. Run as root, the
+  declares first. Run as root, the tests fail
-  tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
+  `TestScanHardlinkRunFailsTogether`, because root reads through the `chmod(0)`
-  `chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
+  the test relies on, so the `test` phase runs them as `nobody`
 - check the installed golangci-lint version in `script/bootstrap` instead of
   only its presence (2026-08-09, branch `bootstrap-version-check`, closes
   https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in

cancel_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -169,8 +168,8 @@ func assertRecordsIntact(t *testing.T, db *sql.DB, before []string) {
 
 // TestSyncScanCancelledMidWalkKeepsRecords is the regression net under
 // the post-walk guard. The scan is cancelled part-way through the
-// walk, so it reaches the guard having walked a genuinely partial tree
+// walk, so it reaches the guard holding a genuinely partial size
-// and holding a still-populated index of records the walk never got to.
+// census and a still-populated index of records the walk never got to.
 // Every one of those records would look vanished to the update phase.
 // The guard is what stops the scan there, and this test is what
 // notices if it stops doing so: deleting the guard, or making it
@@ -197,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
 	base := baselineGoroutines(t)
 
 	st, err := syncScan(newWalkClock(walkCancelAtDone), db,
-		[]string{dir}, walkCancelWorkers, false, walkRules{})
+		[]string{dir}, walkCancelWorkers, false)
 
 	assertWalkGuardAborted(t, st, err)
 	assertRecordsIntact(t, db, before)
@@ -209,10 +208,10 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) {
 }
 
 // assertWalkGuardAborted checks that the scan stopped at the post-walk
-// guard: with a walk that reached some files (it really ran) but not
+// guard: with a census that is neither empty (the walk really ran)
-// all of them (it really was cut short), and with no record counted for
+// nor complete (it really was cut short), and with no record counted
-// removal. A removal count means the partial walk was carried past
+// for removal. A removal count means the partial census was carried
-// the guard into the update phase, which is the failure this test
+// past the guard into the update phase, which is the failure this test
 // exists to catch.
 func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
 	t.Helper()
@@ -223,30 +222,30 @@ func assertWalkGuardAborted(t *testing.T, st scanStats, err error) {
 	}
 
 	if st.unchanged == 0 {
-		t.Fatalf("stats = %+v: the walk reached no file, so it never "+
+		t.Fatalf("stats = %+v: the census is empty, so the walk never "+
 			"ran and the guard was reached for the wrong reason", st)
 	}
 
 	if st.unchanged >= walkCancelFiles {
-		t.Fatalf("stats = %+v: the walk reached the whole tree, so it "+
+		t.Fatalf("stats = %+v: the census covers the whole tree, so the "+
-			"was not cut short", st)
+			"walk was not cut short", st)
 	}
 
 	// The workers drop every directory still queued once the scan is
 	// cancelled, so only the files in the directories already in flight
-	// can still be reached after the fact. A walk beyond that bound
+	// can add to the census after the fact. A census beyond that bound
 	// would mean the cancellation was not observed where it should have
 	// been.
 	limit := walkCancelAtDone + walkCancelInFlightFiles
 	if st.unchanged > limit {
-		t.Errorf("the walk reached %d files, want at most %d: it kept "+
+		t.Errorf("census covers %d files, want at most %d: the walk kept "+
 			"taking directories off the queue after cancellation",
 			st.unchanged, limit)
 	}
 
 	if st.removed != 0 {
 		t.Errorf("stats = %+v: the scan counted records for removal from "+
-			"a partial walk", st)
+			"a partial census", st)
 	}
 }
 
@@ -269,8 +268,7 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	cancel()
 
-	st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false,
+	st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false)
-		walkRules{})
 	if !errors.Is(err, context.Canceled) {
 		t.Fatalf("syncScan on a cancelled context = %v, want %v",
 			err, context.Canceled)
@@ -303,7 +301,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) {
 	db := openTestDB(t)
 
 	st, err := syncScan(newWalkClock(hashCancelAtDone), db,
-		[]string{dir}, walkCancelWorkers, false, walkRules{})
+		[]string{dir}, walkCancelWorkers, false)
 	if !errors.Is(err, context.Canceled) {
 		t.Fatalf("syncScan cancelled mid-hash = %v, want %v",
 			err, context.Canceled)
@@ -358,8 +356,7 @@ func TestRunScanInterrupted(t *testing.T) {
 	stderr := captureStderr(t)
 	dir := buildSmokeTree(t)
 
-	err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
+	err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
-		false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -383,7 +380,7 @@ func TestRunScanInterrupted(t *testing.T) {
 		defer close(done)
 
 		err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers,
-			false, false)
+			false)
 	}()
 
 	awaitReturn(t, done, "runScan")
@@ -410,8 +407,7 @@ func TestRunScanInterrupted(t *testing.T) {
 			got, before)
 	}
 
-	err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false,
+	err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false)
-		false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -437,7 +433,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) {
 	dir := buildWalkCancelTree(t)
 
 	err := runScan(newWalkClock(hashCancelAtDone), []string{dir},
-		walkCancelWorkers, false, false)
+		walkCancelWorkers, false)
 	if !errors.Is(err, errInterrupted) {
 		t.Fatalf("runScan interrupted mid-hash = %v, want %v",
 			err, errInterrupted)
@@ -621,8 +617,7 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) {
 	// Unbuffered and unread: on a cancelled scan every send gives up.
 	events := make(chan walkEvent)
 
-	subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false,
+	subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events)
-		walkRules{}, events)
 	if len(subs) != 0 {
 		t.Errorf("cancelled walkOneDir returned %+v to descend into, "+
 			"want none", subs)
@@ -642,8 +637,7 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) {
 
 	missing := filepath.Join(t.TempDir(), "missing")
 
-	jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false,
+	jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false)
-		walkRules{})
 
 	for range 64 {
 		jobs <- dirJob{path: missing}
@@ -674,7 +668,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
-	jobs, subdirs, events := startWalkWorkers(ctx, 1, false, walkRules{})
+	jobs, subdirs, events := startWalkWorkers(ctx, 1, false)
 
 	// Fill the hand-back channel to its capacity — one slot per worker
 	// — so the worker's own hand-back is certain to block.
@@ -763,9 +757,7 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) {
 
 	var hashed atomic.Bool
 
-	hash := func(path string, size int64) (
+	hash := func(path string, size int64) (string, string, string, error) {
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-	) {
 		hashed.Store(true)
 
 		return hashSignature(path, size)
@@ -805,9 +797,7 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) {
 
 	// The scan is cancelled while the worker hashes, so the worker has
 	// already passed the check that drops queued runs.
-	hash := func(path string, size int64) (
+	hash := func(path string, size int64) (string, string, string, error) {
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-	) {
 		cancel()
 
 		return hashSignature(path, size)

db.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -47,88 +46,40 @@ const dbFilePerm = 0o644
 const lockFilePerm = 0o600
 
 // createTableSQL is the schema applied to a fresh database. Paths are
-// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
-// disk_size holds the bytes the file takes on disk. mtime holds whole
+// holds whole Unix seconds and mtime_nsec the nanoseconds within that
-// Unix seconds and mtime_nsec the nanoseconds within that second. head,
+// second.
-// tail and content each hold the raw 32 bytes of a SHA-256.
 const createTableSQL = `
 CREATE TABLE files (
     path       BLOB PRIMARY KEY,
     size       INTEGER NOT NULL,
-    disk_size  INTEGER NOT NULL,
     mtime      INTEGER NOT NULL,
     mtime_nsec INTEGER NOT NULL,
-    head       BLOB NOT NULL,
+    head       TEXT NOT NULL,
-    tail       BLOB NOT NULL,
+    tail       TEXT NOT NULL,
-    content    BLOB NOT NULL
+    content    TEXT NOT NULL
 ) WITHOUT ROWID
 `
 
-// createIndexSQL indexes the records by signature, so report can read
+// createIndexSQL indexes the records by signature, so report can have
-// them in signature order without sorting the table.
+// SQLite group them without sorting the whole table.
 const createIndexSQL = `
 CREATE INDEX files_signature ON files (size, head, tail, content)
 `
 
-// createTotalsSQL creates the totals table, which holds one row: the
-// number of records and the sizes of all records added up. Counting
-// them over files would read every record. The three triggers below
-// keep the row current, in the transaction of each change to files.
-const createTotalsSQL = `
-CREATE TABLE totals (
-    records INTEGER NOT NULL,
-    size    INTEGER NOT NULL
-)
-`
-
-// insertTotalsSQL adds the one row of totals, for an empty files table.
-const insertTotalsSQL = `
-INSERT INTO totals (records, size) VALUES (0, 0)
-`
-
-// totalsAfterInsertSQL counts each inserted record into totals.
-const totalsAfterInsertSQL = `
-CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
-    UPDATE totals SET records = records + 1, size = size + NEW.size;
-END
-`
-
-// totalsAfterDeleteSQL takes each deleted record out of totals.
-const totalsAfterDeleteSQL = `
-CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
-    UPDATE totals SET records = records - 1, size = size - OLD.size;
-END
-`
-
-// totalsAfterUpdateSQL replaces a record's old size in totals with its
-// new one. scan's upsert of an existing record is such an update.
-const totalsAfterUpdateSQL = `
-CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
-    UPDATE totals SET size = size + NEW.size - OLD.size;
-END
-`
-
-// recordCountSQL reads the number of records from totals.
-const recordCountSQL = `
-SELECT records FROM totals
-`
-
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
-INSERT INTO files (
+INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
-    path, size, disk_size, mtime, mtime_nsec, head, tail, content
+VALUES (?, ?, ?, ?, ?, ?, ?)
-)
-VALUES (?, ?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT (path) DO UPDATE SET
-    size = excluded.size, disk_size = excluded.disk_size,
+    size = excluded.size, mtime = excluded.mtime,
-    mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
+    mtime_nsec = excluded.mtime_nsec,
     head = excluded.head, tail = excluded.tail,
     content = excluded.content
 `
 
-// errNoDatabase reports a missing database file for report, trees and
+// errNoDatabase reports a missing database file for report/trees.
-// info.
 var errNoDatabase = errors.New(
 	"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
 
@@ -140,10 +91,6 @@ var errSchemaVersion = errors.New("unsupported database schema version")
 // database.
 var errScanRunning = errors.New("another scan is running")
 
-// errHashSize reports a record whose head, tail or content is not the
-// 32 bytes of a SHA-256.
-var errHashSize = errors.New("hash is not 32 bytes")
-
 // databasePath resolves the database location: SFDUPES_DATABASE when
 // set and non-empty, the compiled-in default otherwise. It returns the
 // path, with the working directory, its symlinks resolved, put in front
@@ -161,8 +108,10 @@ func databasePath() (string, string, error) {
 	}
 
 	// The working directory as the kernel reports it, with no symlink
-	// in it, as README "Database" says. os.Getwd may return PWD
+	// in it. os.Getwd may return PWD instead, which can name it through
-	// instead, which can name it through a symlink.
+	// a symlink; a ".." after that symlink then leads somewhere else
+	// once the path is cleaned, as scan cleans it to create the
+	// database's directory.
 	wd, err := unix.Getwd()
 	if err != nil {
 		return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
@@ -175,39 +124,16 @@ func databasePath() (string, string, error) {
 	return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
 }
 
-// createDatabaseDirectory creates the directory the database at path
-// goes in, and any directory above it, if missing. path is absolute, as
-// databasePath returns it. The directory is path with its last element
-// cut off as text, by filepath.Split. filepath.Dir would clean it,
-// dropping each ".." together with the directory name before it; when
-// that name is a symlink, the cleaned path names another directory than
-// the one the system reaches when it opens path.
-func createDatabaseDirectory(path string) error {
-	dir, _ := filepath.Split(path)
-
-	err := os.MkdirAll(dir, dbDirPerm)
-	if err != nil {
-		return fmt.Errorf("create database directory: %w", err)
-	}
-
-	return nil
-}
-
 // scanParams are the connection parameters for scan: read-write, with
 // WAL journaling and a busy timeout, so a report can run while a cron
 // scan is in progress. closeScanDatabase leaves WAL mode again.
-//
-// Each write to files fires a trigger, so SQLite keeps a statement
-// journal for it. temp_store(MEMORY) keeps that journal in memory,
-// which costs far less than a temporary file.
 const scanParams = "_pragma=busy_timeout(10000)" +
 	"&_pragma=journal_mode(WAL)" +
-	"&_pragma=synchronous(NORMAL)" +
+	"&_pragma=synchronous(NORMAL)"
-	"&_pragma=temp_store(MEMORY)"
 
-// reportParams are the connection parameters for report, trees and
+// reportParams are the connection parameters for report and trees:
-// info: read-only, with the same busy timeout. They set no journal
+// read-only, with the same busy timeout. They set no journal mode,
-// mode, because setting one is a write.
+// because setting one is a write.
 const reportParams = "mode=ro" +
 	"&_pragma=busy_timeout(10000)" +
 	"&_pragma=query_only(1)"
@@ -252,9 +178,9 @@ func openDB(path, params string) (*sql.DB, error) {
 // that deleted it would let the next scan lock a new file while
 // another still holds the old one.
 func lockScanDatabase(path string) (*os.File, error) {
-	err := createDatabaseDirectory(path)
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create database directory: %w", err)
 	}
 
 	// Like SQLite, this follows a symlink at path and creates its
@@ -298,9 +224,9 @@ func lockScanDatabase(path string) (*os.File, error) {
 // openScanDatabase opens the database for the scan subcommand, creating
 // the file, its parent directory, and the schema as needed.
 func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
-	err := createDatabaseDirectory(path)
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create database directory: %w", err)
 	}
 
 	db, err := openDB(path, scanParams)
@@ -318,42 +244,6 @@ func openScanDatabase(ctx context.Context, path string) (*sql.DB, error) {
 	return db, nil
 }
 
-// databaseFiles returns, by device and inode, the files of the database
-// at path that the scan keeps open: the database file, its -wal and
-// -shm files and its .lock file, each named from the database's path
-// with symlinks resolved, as SQLite and lockScanDatabase name them. The
-// scan calls it once the database and the lock file are open, so that
-// its walk passes over them however it reaches them, through a
-// symlinked directory included. A file that does not exist then is left
-// out, since no walk can reach it. There is no -journal file while the
-// database is in WAL mode.
-func databaseFiles(path string) ([]fileRec, error) {
-	resolved, err := filepath.EvalSymlinks(path)
-	if err != nil {
-		return nil, fmt.Errorf("resolve database path: %w", err)
-	}
-
-	var files []fileRec
-
-	for _, p := range []string{
-		resolved, resolved + "-wal", resolved + "-shm", resolved + ".lock",
-	} {
-		fi, err := os.Lstat(p)
-		if errors.Is(err, fs.ErrNotExist) {
-			continue
-		}
-
-		if err != nil {
-			return nil, fmt.Errorf("database: %w", err)
-		}
-
-		dev, ino := inodeOfInfo(fi)
-		files = append(files, fileRec{path: p, dev: dev, ino: ino})
-	}
-
-	return files, nil
-}
-
 // closeScanDatabase switches the database at path from WAL back to
 // rollback-journal mode and closes it. Out of WAL mode the database
 // file alone holds the whole database, so a reader needs no -wal or
@@ -372,9 +262,9 @@ func closeScanDatabase(ctx context.Context, db *sql.DB, path string) {
 	_ = db.Close()
 }
 
-// openReportDatabase opens an existing database for the report, trees
+// openReportDatabase opens an existing database for the report and
-// and info subcommands. A missing database file is an error directing
+// trees subcommands. A missing database file is an error directing the
-// the user to run scan first; the schema version must match exactly.
+// user to run scan first; the schema version must match exactly.
 func openReportDatabase(ctx context.Context,
 	path string,
 ) (*sql.DB, error) {
@@ -477,14 +367,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
 
 	defer func() { _ = tx.Rollback() }()
 
-	for _, stmt := range []string{
+	_, err = tx.ExecContext(ctx, createTableSQL)
-		createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
+	if err != nil {
-		totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
+		return fmt.Errorf("create schema: %w", err)
-	} {
+	}
-		_, err = tx.ExecContext(ctx, stmt)
+
-		if err != nil {
+	_, err = tx.ExecContext(ctx, createIndexSQL)
-			return fmt.Errorf("create schema: %w", err)
+	if err != nil {
-		}
+		return fmt.Errorf("create schema: %w", err)
 	}
 
 	_, err = tx.ExecContext(ctx,
@@ -513,34 +403,12 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
 	return v, nil
 }
 
-// loadFileRows gives total the number of records, then streams every
+// loadFileRows streams every record to fn in path order: byte order,
-// record to fn in path order: byte order, which is the order of the
+// which is the order of the primary key, so SQLite does not sort.
-// primary key, so SQLite does not sort. The count and the records are
+func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
-// read in one transaction, so they agree while a scan is committing.
+	rows, err := db.QueryContext(ctx,
-func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
+		"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
-	fn func(r scanRec),
+			"FROM files ORDER BY path")
-) error {
-	// Everything goes through tx: db has one connection, so a query on
-	// db would wait for tx forever.
-	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
-	if err != nil {
-		return fmt.Errorf("read records: %w", err)
-	}
-
-	defer func() { _ = tx.Rollback() }()
-
-	var n int64
-
-	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
-	if err != nil {
-		return fmt.Errorf("count records: %w", err)
-	}
-
-	total(n)
-
-	rows, err := tx.QueryContext(ctx,
-		"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
-			"content FROM files ORDER BY path")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -549,25 +417,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 
 	for rows.Next() {
 		var (
-			path, head, tail, content []byte
+			path      []byte
-			sec, nsec                 int64
+			sec, nsec int64
-			r                         scanRec
+			r         scanRec
 		)
 
-		err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
-			&head, &tail, &content)
+			&r.content)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
 		r.path = string(path)
 		r.mtime = time.Unix(sec, nsec)
-
-		err = r.setHashes(head, tail, content)
-		if err != nil {
-			return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
-		}
-
 		fn(r)
 	}
 
@@ -579,44 +441,100 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 	return nil
 }
 
-// signatureRowsSQL selects every record's signature and path in
+// dupeRowsSQL selects every record in a duplicate group, with the
-// signature order, largest size first. That is the order of the
+// group's first path. A group is the records with a content hash that
-// files_signature index read backwards, so SQLite reads the index alone
+// share a size, head, tail, and content, when there are two or more of
-// and returns the first row at once, with nothing to sort.
+// them. The rows come in report order: groups by size descending, then
-const signatureRowsSQL = `
+// by first path, and each group's paths ascending.
-SELECT size, head, tail, content, path
+const dupeRowsSQL = `
-FROM files
+SELECT g.first, f.path, f.size
-ORDER BY size DESC, head DESC, tail DESC, content DESC
+FROM files AS f
+JOIN (
+    SELECT size, head, tail, content, MIN(path) AS first
+    FROM files
+    WHERE content <> ''
+    GROUP BY size, head, tail, content
+    HAVING COUNT(*) > 1
+) AS g USING (size, head, tail, content)
+ORDER BY f.size DESC, g.first, f.path
 `
 
-// loadSignatureRows gives total the number of records, then streams
+// loadDupeRows streams the rows of dupeRowsSQL to fn and returns the
-// every record's signature and path to fn, without its mtime or its
+// number of records in the database. The count and the rows are read
-// disk_size, in the order of signatureRowsSQL. The count and the
+// in one transaction, so they agree while a scan is committing. An
-// records are read in one transaction, so they agree while a scan is
+// error from fn stops the reading and is returned as it is.
-// committing. An error from fn stops the reading and is returned as it
+func loadDupeRows(ctx context.Context, db *sql.DB,
-// is.
+	fn func(first, path string, size int64) error,
-func loadSignatureRows(ctx context.Context, db *sql.DB,
+) (int, error) {
-	total func(n int64), fn func(r scanRec) error,
-) error {
 	// Everything goes through tx: the report connection is the only
 	// one, so a query on db would wait for tx forever.
 	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
 	if err != nil {
-		return fmt.Errorf("read records: %w", err)
+		return 0, fmt.Errorf("read records: %w", err)
 	}
 
 	defer func() { _ = tx.Rollback() }()
 
-	var n int64
+	var records int
 
-	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
+	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&records)
 	if err != nil {
-		return fmt.Errorf("count records: %w", err)
+		return 0, fmt.Errorf("read records: %w", err)
 	}
 
-	total(n)
+	rows, err := tx.QueryContext(ctx, dupeRowsSQL)
+	if err != nil {
+		return 0, fmt.Errorf("read records: %w", err)
+	}
 
-	rows, err := tx.QueryContext(ctx, signatureRowsSQL)
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var (
+			first, path []byte
+			size        int64
+		)
+
+		err = rows.Scan(&first, &path, &size)
+		if err != nil {
+			return 0, fmt.Errorf("read record: %w", err)
+		}
+
+		err = fn(string(first), string(path), size)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return 0, fmt.Errorf("read records: %w", err)
+	}
+
+	return records, nil
+}
+
+// countRecords returns the number of records in the database.
+func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
+	var n int64
+
+	err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("count records: %w", err)
+	}
+
+	return n, nil
+}
+
+// loadFileMeta streams every record's path, size, mtime, and whether
+// it carries hashes to fn. Scan change detection needs no hash
+// values, and skipping the hash columns keeps the scan's in-memory
+// index small on multi-million-file databases.
+func loadFileMeta(ctx context.Context, db *sql.DB,
+	fn func(path string, size int64, mtime time.Time, hashed bool),
+) error {
+	rows, err := db.QueryContext(ctx,
+		"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -625,26 +543,17 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 
 	for rows.Next() {
 		var (
-			path, head, tail, content []byte
+			path            []byte
-			r                         scanRec
+			size, sec, nsec int64
+			hashed          int64
 		)
 
-		err = rows.Scan(&r.size, &head, &tail, &content, &path)
+		err = rows.Scan(&path, &size, &sec, &nsec, &hashed)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
-		r.path = string(path)
+		fn(string(path), size, time.Unix(sec, nsec), hashed != 0)
-
-		err = r.setHashes(head, tail, content)
-		if err != nil {
-			return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
-		}
-
-		err = fn(r)
-		if err != nil {
-			return err
-		}
 	}
 
 	err = rows.Err()
@@ -655,89 +564,52 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 	return nil
 }
 
-// setHashes sets r's head, tail and content to the values read from
+// contentGroupsSQL selects each group (the records sharing a size,
-// those columns. Each must be the raw 32 bytes of a SHA-256.
+// head, and tail) of records of at least headTailMin bytes that has more
-func (r *scanRec) setHashes(head, tail, content []byte) error {
+// than one record and at least one record without a content hash, with
-	for _, b := range [][]byte{head, tail, content} {
+// the number of its records.
-		if len(b) != sha256.Size {
+const contentGroupsSQL = `
-			return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
+SELECT size, head, tail, COUNT(*) AS records
-		}
+FROM files
-	}
+WHERE size >= ? AND head <> ''
+GROUP BY size, head, tail
+HAVING COUNT(*) > 1 AND SUM(content = '') > 0
+`
 
-	r.head = [sha256.Size]byte(head)
+// contentCandidatesSQL selects every record of the groups
-	r.tail = [sha256.Size]byte(tail)
+// contentGroupsSQL selects, with whether each record has a content
-	r.content = [sha256.Size]byte(content)
+// hash. SQLite does the grouping, so no other record's hashes are
+// loaded into memory; the rows come ordered by size, head, and tail, so
+// each group's rows arrive together.
+const contentCandidatesSQL = `
+SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
+FROM files AS f
+JOIN (` + contentGroupsSQL + `) AS g USING (size, head, tail)
+ORDER BY size, head, tail
+`
 
-	return nil
+// countContentCandidates returns the number of rows of
-}
+// contentCandidatesSQL, adding up the records of the groups
-
+// contentGroupsSQL selects, so it neither joins nor sorts.
-// countRecords returns the number of records in the database.
+func countContentCandidates(ctx context.Context, db *sql.DB) (int64, error) {
-func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	var n int64
 
-	err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
+	err := db.QueryRowContext(ctx,
+		"SELECT COALESCE(SUM(records), 0) FROM ("+contentGroupsSQL+")",
+		headTailMin).Scan(&n)
 	if err != nil {
-		return 0, fmt.Errorf("count records: %w", err)
+		return 0, fmt.Errorf("count content candidates: %w", err)
 	}
 
 	return n, nil
 }
 
-// databaseCountsSQL reads the number of records and the sizes of all
+// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
-// records added up from totals.
+// each record, without its content hash, and whether it has one.
-const databaseCountsSQL = `
+func loadContentCandidates(ctx context.Context, db *sql.DB,
-SELECT records, size FROM totals
+	fn func(r scanRec, hashed bool),
-`
-
-// databaseCounts is what info reads from the database: its schema
-// version and the counts of databaseCountsSQL.
-type databaseCounts struct {
-	schemaVersion int
-	records       int64 // every record
-	size          int64 // the sizes of all records added up
-}
-
-// readDatabaseCounts reads the schema version, then the counts of
-// databaseCountsSQL, in one read-only transaction, so both describe the
-// database at one moment while a scan is committing.
-func readDatabaseCounts(ctx context.Context,
-	db *sql.DB,
-) (databaseCounts, error) {
-	var c databaseCounts
-
-	// Everything goes through tx: the report connection is the only
-	// one, so a query on db would wait for tx forever.
-	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
-	if err != nil {
-		return c, fmt.Errorf("count records: %w", err)
-	}
-
-	defer func() { _ = tx.Rollback() }()
-
-	err = tx.QueryRowContext(ctx, "PRAGMA user_version").
-		Scan(&c.schemaVersion)
-	if err != nil {
-		return c, fmt.Errorf("read schema version: %w", err)
-	}
-
-	err = tx.QueryRowContext(ctx, databaseCountsSQL).
-		Scan(&c.records, &c.size)
-	if err != nil {
-		return c, fmt.Errorf("count records: %w", err)
-	}
-
-	return c, nil
-}
-
-// loadFileMeta streams every record's path, size, disk_size and mtime
-// to fn. Scan change detection needs no hash values, and skipping the
-// hash columns keeps the scan's in-memory index small on
-// multi-million-file databases.
-func loadFileMeta(ctx context.Context, db *sql.DB,
-	fn func(path string, m fileMeta),
 ) error {
-	rows, err := db.QueryContext(ctx,
+	rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
-		"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -747,18 +619,20 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 	for rows.Next() {
 		var (
 			path      []byte
-			m         fileMeta
 			sec, nsec int64
+			r         scanRec
+			hashed    int64
 		)
 
-		err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
+			&hashed)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
-		m.mtime = time.Unix(sec, nsec)
+		r.path = string(path)
-
+		r.mtime = time.Unix(sec, nsec)
-		fn(string(path), m)
+		fn(r, hashed != 0)
 	}
 
 	err = rows.Err()
@@ -776,21 +650,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 // WAL small and let concurrent reports observe progress.
 const updateBatchSize = 10000
 
-// applyChanges writes upserts for new and changed files and deletes for
+// applyChanges writes one scan's database changes — upserts for new and
-// vanished ones, in batched transactions. Progress is rendered on prog
+// changed files, deletes for vanished ones — in batched transactions.
-// (one increment per change).
+// Progress is rendered on prog (one increment per change).
 func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
 	deletes []string, prog *progress,
 ) error {
 	for batch := range slices.Chunk(upserts, updateBatchSize) {
-		err := applyBatch(ctx, db, batch, nil, nil, prog)
+		err := applyBatch(ctx, db, batch, nil, prog)
 		if err != nil {
 			return err
 		}
 	}
 
 	for batch := range slices.Chunk(deletes, updateBatchSize) {
-		err := applyBatch(ctx, db, nil, nil, batch, prog)
+		err := applyBatch(ctx, db, nil, batch, prog)
 		if err != nil {
 			return err
 		}
@@ -799,26 +673,10 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
 	return nil
 }
 
-// applyDiskSizes writes the disk_size of each of recs into its record,
+// applyBatch commits one batch of upserts and deletes in a single
-// in batched transactions, and leaves the rest of the record alone.
+// transaction.
-// Progress is rendered on prog (one increment per record).
-func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
-	prog *progress,
-) error {
-	for batch := range slices.Chunk(recs, updateBatchSize) {
-		err := applyBatch(ctx, db, nil, batch, nil, prog)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// applyBatch commits one batch of upserts, disk_size updates and
-// deletes in a single transaction.
 func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
-	diskSizes []fileRec, deletes []string, prog *progress,
+	deletes []string, prog *progress,
 ) error {
 	tx, err := db.BeginTx(ctx, nil)
 	if err != nil {
@@ -832,11 +690,6 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
 		return err
 	}
 
-	err = execDiskSizes(ctx, tx, diskSizes, prog)
-	if err != nil {
-		return err
-	}
-
 	err = execDeletes(ctx, tx, deletes, prog)
 	if err != nil {
 		return err
@@ -862,9 +715,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 	defer func() { _ = st.Close() }()
 
 	for _, r := range upserts {
-		_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
+		_, err = st.ExecContext(ctx, []byte(r.path), r.size,
-			r.mtime.Unix(), r.mtime.Nanosecond(),
+			r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
-			r.head[:], r.tail[:], r.content[:])
 		if err != nil {
 			return fmt.Errorf("upsert %s: %w", r.path, err)
 		}
@@ -875,31 +727,6 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 	return nil
 }
 
-// execDiskSizes sets the disk_size of the record of each of recs. It
-// does not touch size, so the totals triggers do not fire.
-func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
-	prog *progress,
-) error {
-	st, err := tx.PrepareContext(ctx,
-		"UPDATE files SET disk_size = ? WHERE path = ?")
-	if err != nil {
-		return fmt.Errorf("prepare disk_size update: %w", err)
-	}
-
-	defer func() { _ = st.Close() }()
-
-	for _, r := range recs {
-		_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
-		if err != nil {
-			return fmt.Errorf("update disk_size of %s: %w", r.path, err)
-		}
-
-		prog.increment()
-	}
-
-	return nil
-}
-
 // execDeletes removes the records for paths no longer present.
 func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
 	prog *progress,

db_test.go

@@ -2,11 +2,9 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
-	"io/fs"
 	"os"
 	"path/filepath"
 	"slices"
@@ -120,116 +118,6 @@ func TestOpenScanDatabaseCreates(t *testing.T) {
 	}
 }
 
-// TestScanCreatesDatabaseDirectoryAfterSymlink has a database path go
-// through a symlink to a directory, then "..", to a missing directory.
-// With srv/link a symlink to data/x, srv/link/../new/db.sqlite leads to
-// data/new/db.sqlite. lockScanDatabase and openScanDatabase, the two
-// places scan creates the database's directory, must each create the
-// database there, and nothing at srv/new, where the path leads once
-// cleaned as text.
-func TestScanCreatesDatabaseDirectoryAfterSymlink(t *testing.T) {
-	t.Parallel()
-
-	creates := map[string]func(ctx context.Context, path string) error{
-		"lockScanDatabase": func(_ context.Context, path string) error {
-			lock, err := lockScanDatabase(path)
-			if err == nil {
-				_ = lock.Close()
-			}
-
-			return err
-		},
-		"openScanDatabase": func(ctx context.Context, path string) error {
-			db, err := openScanDatabase(ctx, path)
-			if err == nil {
-				_ = db.Close()
-			}
-
-			return err
-		},
-	}
-
-	for name, create := range creates {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			dir, err := filepath.EvalSymlinks(t.TempDir())
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = os.MkdirAll(filepath.Join(dir, "data", "x"), 0o750)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = os.Mkdir(filepath.Join(dir, "srv"), 0o750)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = os.Symlink(filepath.Join(dir, "data", "x"),
-				filepath.Join(dir, "srv", "link"))
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = create(t.Context(), dir+"/srv/link/../new/db.sqlite")
-			if err != nil {
-				t.Fatalf("%s: %v", name, err)
-			}
-
-			_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
-			if err != nil {
-				t.Errorf("database the path leads to: %v", err)
-			}
-
-			_, err = os.Lstat(filepath.Join(dir, "srv", "new"))
-			if !errors.Is(err, fs.ErrNotExist) {
-				t.Errorf("srv/new: stat error %v, want it missing", err)
-			}
-		})
-	}
-}
-
-// TestDatabaseFiles opens a database through a symlink to its file:
-// databaseFiles finds the database file and its -wal and -shm files
-// beside the symlink's target, where SQLite keeps them, and leaves out
-// the .lock file, which only a scan creates.
-func TestDatabaseFiles(t *testing.T) {
-	t.Parallel()
-
-	path := testDBPath(t)
-	link := filepath.Join(t.TempDir(), "link.sqlite")
-
-	err := os.Symlink(path, link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db, err := openScanDatabase(t.Context(), link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = db.Close() }()
-
-	files, err := databaseFiles(link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	got := make([]string, 0, len(files))
-	for _, f := range files {
-		got = append(got, f.path)
-	}
-
-	want := []string{path, path + "-wal", path + "-shm"}
-	if !slices.Equal(got, want) {
-		t.Errorf("databaseFiles(%s) = %q, want %q", link, got, want)
-	}
-}
-
 func TestOpenDatabaseUnversionedForeign(t *testing.T) {
 	t.Parallel()
 
@@ -396,336 +284,12 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
 
 	defer func() { _ = reportDB.Close() }()
 
-	err = loadFileRows(t.Context(), reportDB, func(int64) {},
+	err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
-		func(scanRec) {})
 	if err != nil {
 		t.Fatalf("loadFileRows: %v", err)
 	}
 }
 
-// TestLoadRowsCountAgreeDuringScan has a scan commit records after
-// loadFileRows, as trees calls it, or loadSignatureRows, as report
-// calls it, has counted the records and before it reads them, as when
-// trees or report runs during a scan: each must read exactly as many
-// records as it counted.
-func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
-	t.Parallel()
-
-	loads := map[string]func(context.Context, *sql.DB, func(int64),
-		func(scanRec)) error{
-		"loadFileRows": loadFileRows,
-		"loadSignatureRows": func(ctx context.Context, db *sql.DB,
-			total func(int64), fn func(scanRec),
-		) error {
-			return loadSignatureRows(ctx, db, total, func(r scanRec) error {
-				fn(r)
-
-				return nil
-			})
-		},
-	}
-
-	for name, load := range loads {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			path := testDBPath(t)
-
-			scanDB, err := openScanDatabase(t.Context(), path)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			defer func() { _ = scanDB.Close() }()
-
-			recs := smokeTreeRecs()
-
-			err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			reportDB, err := openReportDatabase(t.Context(), path)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			defer func() { _ = reportDB.Close() }()
-
-			var counted, read int64
-
-			err = load(t.Context(), reportDB, func(n int64) {
-				counted = n
-
-				err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
-				if err != nil {
-					t.Error(err)
-				}
-			}, func(scanRec) { read++ })
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if counted != 3 || read != counted {
-				t.Errorf("counted %d records and read %d, want 3 and 3",
-					counted, read)
-			}
-		})
-	}
-}
-
-// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
-// content is not the 32 bytes of a SHA-256: either one byte, or the 64
-// bytes of a SHA-256 written as hex text. loadFileRows and
-// loadSignatureRows must each fail on it with an error naming its path.
-func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
-	t.Parallel()
-
-	hexText := strings.Repeat("ab", sha256.Size)
-
-	cases := []struct {
-		name   string
-		update string
-		value  any
-	}{
-		{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
-		{"hex-text head", "UPDATE files SET head = ?", hexText},
-		{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
-		{"hex-text tail", "UPDATE files SET tail = ?", hexText},
-		{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
-		{"hex-text content", "UPDATE files SET content = ?", hexText},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			t.Parallel()
-
-			db := openTestDB(t)
-
-			err := applyChanges(t.Context(), db,
-				[]scanRec{{path: "/a/f"}}, nil, nil)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			_, err = db.ExecContext(t.Context(), c.update, c.value)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			loads := map[string]func() error{
-				"loadFileRows": func() error {
-					return loadFileRows(t.Context(), db, func(int64) {},
-						func(scanRec) {})
-				},
-				"loadSignatureRows": func() error {
-					return loadSignatureRows(t.Context(), db, func(int64) {},
-						func(scanRec) error { return nil })
-				},
-			}
-
-			for name, load := range loads {
-				err := load()
-				if !errors.Is(err, errHashSize) ||
-					!strings.Contains(err.Error(), "/a/f") {
-					t.Errorf("%s: err = %v, want %v naming /a/f",
-						name, err, errHashSize)
-				}
-			}
-		})
-	}
-}
-
-// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
-// PLAN for query.
-func queryPlan(t *testing.T, db *sql.DB, query string) []string {
-	t.Helper()
-
-	rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = rows.Close() }()
-
-	var plan []string
-
-	for rows.Next() {
-		var (
-			id, parent, unused int
-			detail             string
-		)
-
-		err = rows.Scan(&id, &parent, &unused, &detail)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		plan = append(plan, detail)
-	}
-
-	err = rows.Err()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return plan
-}
-
-// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
-// from the files_signature index alone, with nothing to sort: the
-// first record comes at once, so report's read phase counts each
-// record as it comes rather than waiting on SQLite for all of them.
-func TestSignatureRowsNeedNoSort(t *testing.T) {
-	t.Parallel()
-
-	db := openTestDB(t)
-
-	plan := queryPlan(t, db, signatureRowsSQL)
-
-	want := []string{"SCAN files USING COVERING INDEX files_signature"}
-	if !slices.Equal(plan, want) {
-		t.Errorf("query plan = %q, want %q", plan, want)
-	}
-}
-
-// TestCountsReadOnlyTotals checks that SQLite answers the queries for
-// the number of records and their total size from the one row of
-// totals. A plan that reads files or its index reads every record.
-func TestCountsReadOnlyTotals(t *testing.T) {
-	t.Parallel()
-
-	db := openTestDB(t)
-
-	for _, query := range []string{recordCountSQL, databaseCountsSQL} {
-		plan := queryPlan(t, db, query)
-
-		want := []string{"SCAN totals"}
-		if !slices.Equal(plan, want) {
-			t.Errorf("query plan of %q = %q, want %q", query, plan, want)
-		}
-	}
-}
-
-// TestCountsComeFromTotals sets the row of totals to numbers that files
-// does not hold, then checks that scan's load phase, the read phases of
-// trees and report, and info each report the numbers from totals.
-func TestCountsComeFromTotals(t *testing.T) {
-	t.Parallel()
-
-	// files is empty, so a count over files gives 0 records of 0 bytes.
-	const records, size = 7, 9000
-
-	db := openTestDB(t)
-
-	_, err := db.ExecContext(t.Context(),
-		"UPDATE totals SET records = ?, size = ?", records, size)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	scanTotal, err := countRecords(t.Context(), db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var treesTotal, reportTotal int64
-
-	err = loadFileRows(t.Context(), db,
-		func(n int64) { treesTotal = n }, func(scanRec) {})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = loadSignatureRows(t.Context(), db,
-		func(n int64) { reportTotal = n },
-		func(scanRec) error { return nil })
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	info, err := readDatabaseCounts(t.Context(), db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for name, got := range map[string]int64{
-		"scan's load":   scanTotal,
-		"trees' read":   treesTotal,
-		"report's read": reportTotal,
-		"info":          info.records,
-	} {
-		if got != records {
-			t.Errorf("%s: %d records, want %d from totals",
-				name, got, records)
-		}
-	}
-
-	if info.size != size {
-		t.Errorf("info: size %d, want %d from totals", info.size, size)
-	}
-}
-
-// assertTotalsMatchFiles fails unless the row of totals holds the
-// number of records and the sizes of all records added up, as counted
-// over files.
-func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
-	t.Helper()
-
-	got, err := readDatabaseCounts(t.Context(), db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var records, size int64
-
-	err = db.QueryRowContext(t.Context(),
-		"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
-		Scan(&records, &size)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if got.records != records || got.size != size {
-		t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
-			got.records, got.size, records, size)
-	}
-}
-
-// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
-// change size, and rescans it after two files are removed. After each
-// scan the row of totals must agree with files.
-func TestTotalsMatchFiles(t *testing.T) {
-	t.Parallel()
-
-	dir := buildSmokeTree(t)
-	db := openTestDB(t)
-
-	syncTree(t, db, dir)
-	assertTotalsMatchFiles(t, db)
-
-	writeFile(t, dir, "a/one.bin", pattern(10, 2500))
-	writeFile(t, dir, "t1/f1", pattern(30, 1000))
-
-	if st := syncTree(t, db, dir); st.updated != 2 {
-		t.Fatalf("rescan stats = %+v, want 2 updated", st)
-	}
-
-	assertTotalsMatchFiles(t, db)
-
-	for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
-		err := os.Remove(filepath.Join(dir, name))
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if st := syncTree(t, db, dir); st.removed != 2 {
-		t.Fatalf("rescan stats = %+v, want 2 removed", st)
-	}
-
-	assertTotalsMatchFiles(t, db)
-}
-
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()
 
@@ -736,13 +300,12 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// written.
 	recs := []scanRec{
 		{
-			size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
+			size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
-			tail: testHash("t2"), content: testHash("c2"),
+			content: "c2", path: "/a/tab\tnew\nline",
-			path: "/a/tab\tnew\nline",
 		},
 		{
-			size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
+			size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
-			tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
+			content: "c1", path: "/a/x",
 		},
 	}
 
@@ -763,8 +326,8 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// An upsert for an existing path updates in place; a delete
 	// removes exactly its path.
 	upd := scanRec{
-		size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
+		size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
-		tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
+		path: "/a/x",
 	}
 
 	err = applyChanges(t.Context(), db, []scanRec{upd},
@@ -791,8 +354,8 @@ func TestApplyChangesBatching(t *testing.T) {
 	recs := make([]scanRec, 0, n)
 	for i := range n {
 		recs = append(recs, scanRec{
-			size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
+			size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
-			tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
+			path: fmt.Sprintf("/batch/%07d", i),
 		})
 	}
 
@@ -822,3 +385,62 @@ func TestApplyChangesBatching(t *testing.T) {
 		t.Fatalf("records = %d, want 0", len(got))
 	}
 }
+
+// TestCountContentCandidates checks that countContentCandidates counts
+// the records the content phase's search returns: those of each group
+// of two or more records of headTailMin or more with a head, where at
+// least one has no content hash.
+func TestCountContentCandidates(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// Each group's records: size, head, and content, one per record.
+	groups := []struct {
+		size     int64
+		head     string
+		contents []string
+	}{
+		{headTailMin, "h", []string{"", "c", "c"}}, // returned
+		{headTailMin + 1, "h", []string{"", ""}},   // returned
+		{headTailMin + 2, "h", []string{"c", "c"}}, // all hashed
+		{headTailMin + 3, "h", []string{""}},       // one record
+		{headTailMin - 1, "h", []string{"", ""}},   // too small
+		{headTailMin + 4, "", []string{"", ""}},    // no head
+	}
+
+	var recs []scanRec
+
+	for _, g := range groups {
+		for _, content := range g.contents {
+			recs = append(recs, scanRec{
+				size: g.size, mtime: time.Unix(1, 0), head: g.head,
+				tail: "t", content: content,
+				path: fmt.Sprintf("/r/%d", len(recs)),
+			})
+		}
+	}
+
+	err := applyChanges(t.Context(), db, recs, nil, nil)
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := countContentCandidates(t.Context(), db)
+	if err != nil {
+		t.Fatalf("countContentCandidates: %v", err)
+	}
+
+	var returned int64
+
+	err = loadContentCandidates(t.Context(), db,
+		func(scanRec, bool) { returned++ })
+	if err != nil {
+		t.Fatalf("loadContentCandidates: %v", err)
+	}
+
+	if got != 5 || returned != 5 {
+		t.Errorf("count = %d, records returned = %d, want 5 and 5",
+			got, returned)
+	}
+}

exclude.go

@@ -1,195 +0,0 @@
-package main
-
-// This file holds two rules for the walk (README "Rules for the walk"):
-// the default excludes, the paths a scan's walk does not enter below an
-// operand unless scan is given --no-default-excludes, and the
-// filesystems that are not persistent, on which the scan never records
-// or reads anything.
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"path/filepath"
-	"slices"
-	"strings"
-
-	"golang.org/x/sys/unix"
-)
-
-// The systems that have default excludes, as runtime.GOOS names them.
-const (
-	goosLinux   = "linux"
-	goosDarwin  = "darwin"
-	goosWindows = "windows"
-)
-
-// The fields of a line of mountInfoPath that readMountInfo reads,
-// counted from 0 (see proc(5)): the device ID, as major:minor, is the
-// third field, and after the sixth come optional fields, then a field
-// holding "-", then the filesystem type.
-const (
-	deviceField         = 2
-	optionalFieldsStart = 6
-)
-
-// typesPerHelpLine is how many filesystem types the scan command's help
-// lists on each line.
-const typesPerHelpLine = 6
-
-// errMountInfoLine reports a line of mountInfoPath without the fields
-// readMountInfo reads.
-var errMountInfoLine = errors.New("unexpected line")
-
-// walkRules holds the rules for the walk that a scan sets up as it
-// starts. excludes are the default excludes, or none with
-// --no-default-excludes. nonPersistent maps the device ID of every
-// mounted filesystem that is not persistent to its type; the scan
-// records and reads nothing on one of those, whatever the flags.
-// databaseFiles are the scan's own database files, which the walk
-// passes over (see databaseFiles in db.go).
-type walkRules struct {
-	excludes      []string
-	nonPersistent map[uint64]string
-	databaseFiles []fileRec
-}
-
-// defaultExcludes returns the default excludes of the system goos
-// names, as runtime.GOOS does: a fixed list on Linux, macOS and
-// Windows, and none on any other system.
-func defaultExcludes(goos string) []string {
-	switch goos {
-	case goosLinux:
-		return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
-	case goosDarwin:
-		// A walk of / reaches everything in /System/Volumes/Data through
-		// the firmlinks, so walking it as well would record each of
-		// those files twice.
-		return []string{
-			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
-			"/System/Volumes/VM", "/System/Volumes/Data",
-		}
-	case goosWindows:
-		// Relative to the root of the drive an operand lies on. The
-		// first three are the page, swap and hibernation files.
-		return []string{
-			"pagefile.sys", "swapfile.sys", "hiberfil.sys",
-			"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
-			`Users\*\AppData\Local\Temp`,
-		}
-	default:
-		return nil
-	}
-}
-
-// isDefaultExclude reports whether path is one of excludes, in which a
-// * matches one path component.
-func isDefaultExclude(path string, excludes []string) bool {
-	for _, pattern := range excludes {
-		// The patterns are fixed, so none is malformed.
-		if ok, _ := filepath.Match(pattern, path); ok {
-			return true
-		}
-	}
-
-	return false
-}
-
-// nonPersistentFilesystemTypes returns the filesystem types that are
-// not persistent: filesystems that the kernel makes up or that are kept
-// in memory. devfs is the macOS filesystem of /dev; the others are
-// Linux names.
-func nonPersistentFilesystemTypes() []string {
-	return []string{
-		"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs",
-		"cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore",
-		"bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs",
-		"binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", "devfs",
-	}
-}
-
-// nonPersistentLine returns the line the scan prints for path when fi,
-// its lstat, puts it on a filesystem in rules.nonPersistent, and ""
-// when it is on any other.
-func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string {
-	dev, ok := deviceOfInfo(fi)
-	if !ok {
-		return ""
-	}
-
-	fsType, found := rules.nonPersistent[dev]
-	if !found {
-		return ""
-	}
-
-	return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)",
-		path, fsType)
-}
-
-// readMountInfo reads mountinfo, laid out as mountInfoPath is, and
-// returns the device ID of every mounted filesystem that is not
-// persistent, with its type. The device ID is the one each line gives,
-// not that of the file now at its mount point, so a mount that a later
-// mount hides never adds the device of another filesystem.
-func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) {
-	types := nonPersistentFilesystemTypes()
-	devices := make(map[uint64]string)
-	lines := bufio.NewScanner(mountinfo)
-
-	for lines.Scan() {
-		fields := strings.Fields(lines.Text())
-
-		sep := slices.Index(fields, "-")
-		if sep < optionalFieldsStart || sep+1 >= len(fields) {
-			return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
-		}
-
-		var major, minor uint32
-
-		_, err := fmt.Sscanf(fields[deviceField], "%d:%d", &major, &minor)
-		if err != nil {
-			return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text())
-		}
-
-		fsType := fields[sep+1]
-		if slices.Contains(types, fsType) {
-			devices[unix.Mkdev(major, minor)] = fsType
-		}
-	}
-
-	return devices, lines.Err()
-}
-
-// defaultExcludesHelp describes, for the scan command's help, the
-// default excludes of the system goos names and, where the scan lists
-// the mounted filesystems, the types it never records anything on.
-func defaultExcludesHelp(goos string) string {
-	var help strings.Builder
-
-	dirs := defaultExcludes(goos)
-	if len(dirs) == 0 {
-		help.WriteString("This system has no default excludes.\n")
-	} else {
-		help.WriteString("Below an operand, the walk does not enter these " +
-			"default excludes, unless\n--no-default-excludes is given:\n\n")
-
-		for _, dir := range dirs {
-			fmt.Fprintf(&help, "  %s\n", dir)
-		}
-	}
-
-	if goos == goosLinux || goos == goosDarwin {
-		help.WriteString("\nWith or without --no-default-excludes, the scan " +
-			"records nothing on a\nfilesystem that is not persistent, " +
-			"one of these types:\n\n")
-
-		types := nonPersistentFilesystemTypes()
-		for line := range slices.Chunk(types, typesPerHelpLine) {
-			fmt.Fprintf(&help, "  %s\n", strings.Join(line, ", "))
-		}
-	}
-
-	return help.String()
-}

exclude_darwin.go

@@ -1,52 +0,0 @@
-//go:build darwin
-
-package main
-
-import (
-	"slices"
-
-	"golang.org/x/sys/unix"
-)
-
-// nonPersistentFilesystems returns the device ID of every mounted
-// filesystem that is not persistent, with its type, from the system's
-// list of mounted filesystems. The device ID is read with lstat on the
-// mount point. Where one filesystem is mounted over another on the same
-// mount point, the later one in the list counts. A mount point that
-// lstat cannot reach is left out.
-func nonPersistentFilesystems() (map[uint64]string, error) {
-	// The first call counts the filesystems, the second lists them.
-	n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT)
-	if err != nil {
-		return nil, err
-	}
-
-	list := make([]unix.Statfs_t, n)
-
-	n, err = unix.Getfsstat(list, unix.MNT_NOWAIT)
-	if err != nil {
-		return nil, err
-	}
-
-	mounts := make(map[string]string, n)
-
-	for _, st := range list[:n] {
-		mounts[unix.ByteSliceToString(st.Mntonname[:])] =
-			unix.ByteSliceToString(st.Fstypename[:])
-	}
-
-	types := nonPersistentFilesystemTypes()
-	devices := make(map[uint64]string)
-
-	for mountPoint, fsType := range mounts {
-		if !slices.Contains(types, fsType) {
-			continue
-		}
-
-		if dev, ok := deviceOfPath(mountPoint); ok {
-			devices[dev] = fsType
-		}
-	}
-
-	return devices, nil
-}

exclude_other.go

@@ -1,36 +0,0 @@
-//go:build !darwin
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"runtime"
-)
-
-// mountInfoPath is the file in which Linux lists the mounts the reading
-// process sees.
-const mountInfoPath = "/proc/self/mountinfo"
-
-// nonPersistentFilesystems returns the device ID of every mounted
-// filesystem that is not persistent, with its type: on Linux read from
-// mountInfoPath (see readMountInfo), and on any other system none.
-func nonPersistentFilesystems() (map[uint64]string, error) {
-	if runtime.GOOS != goosLinux {
-		return map[uint64]string{}, nil
-	}
-
-	f, err := os.Open(mountInfoPath)
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() { _ = f.Close() }()
-
-	devices, err := readMountInfo(f)
-	if err != nil {
-		return nil, fmt.Errorf("read %s: %w", mountInfoPath, err)
-	}
-
-	return devices, nil
-}

exclude_test.go

@@ -1,513 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"errors"
-	"maps"
-	"os"
-	"path/filepath"
-	"runtime"
-	"slices"
-	"strings"
-	"testing"
-
-	"golang.org/x/sys/unix"
-)
-
-func TestDefaultExcludes(t *testing.T) {
-	t.Parallel()
-
-	// README "Rules for the walk": each system's list.
-	cases := map[string][]string{
-		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
-		goosDarwin: {
-			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
-			"/System/Volumes/VM", "/System/Volumes/Data",
-		},
-		goosWindows: {
-			"pagefile.sys", "swapfile.sys", "hiberfil.sys",
-			"System Volume Information", "$Recycle.Bin", `Windows\Temp`,
-			`Users\*\AppData\Local\Temp`,
-		},
-		"freebsd": nil,
-	}
-
-	for goos, want := range cases {
-		if got := defaultExcludes(goos); !slices.Equal(got, want) {
-			t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want)
-		}
-	}
-}
-
-func TestIsDefaultExclude(t *testing.T) {
-	t.Parallel()
-
-	// A * matches one path component, no more and no fewer.
-	excludes := []string{"/scratch", "/home/*/cache"}
-
-	cases := map[string]bool{
-		"/scratch":        true,
-		"/scratch/x":      false,
-		"/home/u/cache":   true,
-		"/home/u/v/cache": false,
-		"/home/cache":     false,
-	}
-
-	for path, want := range cases {
-		if got := isDefaultExclude(path, excludes); got != want {
-			t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
-		}
-	}
-}
-
-func TestReadMountInfo(t *testing.T) {
-	t.Parallel()
-
-	// The second-to-last line has no optional fields. The tmpfs on
-	// /data/a/b is hidden by the root filesystem's /srv, mounted over
-	// /data/a after it: the tmpfs still gives its own device, and the
-	// root filesystem's device is not listed.
-	mountinfo := strings.Join([]string{
-		"22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw",
-		"23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw",
-		"24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw",
-		"25 22 0:23 / /data/a/b rw shared:2 - tmpfs none rw",
-		"26 22 8:1 /srv /data/a rw - ext4 /dev/sda1 rw",
-		"27 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw",
-	}, "\n")
-
-	got, err := readMountInfo(strings.NewReader(mountinfo))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := map[uint64]string{
-		unix.Mkdev(0, 21): "proc", unix.Mkdev(0, 22): "sysfs",
-		unix.Mkdev(0, 23): "tmpfs", unix.Mkdev(0, 25): "autofs",
-	}
-	if !maps.Equal(got, want) {
-		t.Errorf("readMountInfo = %v, want %v", got, want)
-	}
-
-	for _, line := range []string{
-		"22 1 8:1 / / rw,relatime\n",
-		"22 1 sda / / rw,relatime shared:1 - ext4 /dev/sda1 rw\n",
-	} {
-		_, err = readMountInfo(strings.NewReader(line))
-		if !errors.Is(err, errMountInfoLine) {
-			t.Errorf("readMountInfo(%q) = %v, want %v", line, err,
-				errMountInfoLine)
-		}
-	}
-}
-
-func TestNonPersistentFilesystems(t *testing.T) {
-	t.Parallel()
-
-	if runtime.GOOS != goosLinux {
-		t.Skip("the test knows a mount only on Linux: /proc")
-	}
-
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dev, ok := deviceOfPath("/proc")
-	if !ok {
-		t.Fatal("no device for /proc")
-	}
-
-	if got := nonPersistent[dev]; got != "proc" {
-		t.Errorf("filesystem of /proc = %q, want %q", got, "proc")
-	}
-}
-
-func TestDefaultExcludesHelp(t *testing.T) {
-	t.Parallel()
-
-	for _, goos := range []string{goosLinux, goosDarwin, goosWindows} {
-		help := defaultExcludesHelp(goos)
-
-		for _, dir := range defaultExcludes(goos) {
-			if !strings.Contains(help, "\n  "+dir+"\n") {
-				t.Errorf("help for %s does not list %s:\n%s", goos, dir, help)
-			}
-		}
-
-		// Only Linux and macOS list their mounted filesystems.
-		for _, fsType := range nonPersistentFilesystemTypes() {
-			listed := strings.Contains(help, " "+fsType+",") ||
-				strings.Contains(help, " "+fsType+"\n")
-			if listed != (goos != goosWindows) {
-				t.Errorf("help for %s lists %s: %v", goos, fsType, listed)
-			}
-		}
-	}
-
-	want := "This system has no default excludes.\n"
-	if got := defaultExcludesHelp("freebsd"); got != want {
-		t.Errorf("help for freebsd = %q, want %q", got, want)
-	}
-
-	// scan's help shows those of the system it runs on.
-	var stdout, stderr bytes.Buffer
-
-	run([]string{cmdScan, "-h"}, &stdout, &stderr)
-
-	help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS))
-	if !strings.Contains(stderr.String(), help) {
-		t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help)
-	}
-}
-
-// TestScanDoesNotEnterDefaultExclude checks that the walk does not
-// enter a default exclude below an operand. It says so on a line of its
-// own and does not count it as skipped. The records below it count as
-// outside the operands: the record of a file gone from it is kept,
-// while a file added to it is not recorded.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	db := openTestDB(t)
-	excluded := filepath.Join(dir, "tmp")
-
-	// A scan with no default excludes records the file in excluded.
-	gone := writeFile(t, excluded, "gone", pattern(1, 100))
-	syncTree(t, db, dir)
-
-	err := os.Remove(gone)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	unseen := writeFile(t, excluded, "new", pattern(2, 10))
-
-	st, err := syncScan(t.Context(), db, []string{dir}, 1, false,
-		walkRules{excludes: []string{excluded}})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.skipped != 0 {
-		t.Errorf("skipped = %d, want 0", st.skipped)
-	}
-
-	recs := dbRecords(t, db)
-	recordByPath(t, recs, gone)
-
-	if slices.Contains(recordPaths(recs), unseen) {
-		t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
-	}
-
-	line := "\nscan: skipped " + excluded + " (default exclude)\n"
-	if !strings.Contains(stderr(), line) {
-		t.Errorf("stderr = %q, want %q", stderr(), line)
-	}
-}
-
-// TestScanWalksOperandInDefaultExclude checks that an operand at or
-// inside a default exclude is walked, also when it lies under another
-// operand, whose walk does not enter the default exclude.
-func TestScanWalksOperandInDefaultExclude(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	excluded := filepath.Join(dir, "tmp")
-	f := writeFile(t, excluded, "sub/f", pattern(1, 10))
-	sub := filepath.Dir(f)
-
-	cases := map[string][]string{
-		"at it":                      {excluded},
-		"inside it":                  {sub},
-		"inside it under an operand": {dir, sub},
-	}
-
-	for name, roots := range cases {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			db := openTestDB(t)
-
-			st, err := syncScan(t.Context(), db, roots, 1, false,
-				walkRules{excludes: []string{excluded}})
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if st.added != 1 {
-				t.Errorf("added = %d, want %s recorded once", st.added, f)
-			}
-
-			recordByPath(t, dbRecords(t, db), f)
-		})
-	}
-}
-
-// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default
-// exclude given as an operand under another operand, which the walk of
-// that other operand does not enter, gets no line saying it was
-// skipped: it is walked as an operand of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	db := openTestDB(t)
-	excluded := filepath.Join(dir, "tmp")
-	f := writeFile(t, excluded, "f", pattern(1, 10))
-
-	st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false,
-		walkRules{excludes: []string{excluded}})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.added != 1 {
-		t.Errorf("added = %d, want %s recorded once", st.added, f)
-	}
-
-	recordByPath(t, dbRecords(t, db), f)
-
-	if strings.Contains(stderr(), "scan: skipped") {
-		t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
-	}
-}
-
-// TestRunScanTempDirOperand scans a tree in the tests' temporary
-// directory, which on Linux lies inside the default exclude /tmp unless
-// TMPDIR names another place, with the system's own default excludes:
-// the scan walks the whole tree and reports no default exclude.
-func TestRunScanTempDirOperand(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
-		t.Skip(dir + " is not inside a default exclude")
-	}
-
-	f := writeFile(t, dir, "sub/f", pattern(1, 10))
-
-	stderr := scanOK(t, dir)
-	if strings.Contains(stderr, "scan: skipped") {
-		t.Errorf("stderr = %q, want no line saying a path was skipped",
-			stderr)
-	}
-
-	assertRecorded(t, path, f, true)
-}
-
-// TestRunScanNoDefaultExcludes scans /var through the command. Its
-// /var/tmp, on a persistent filesystem, is a default exclude on Linux:
-// without --no-default-excludes the walk does not enter it and says so,
-// and with that flag the walk records a file the test puts there.
-func TestRunScanNoDefaultExcludes(t *testing.T) {
-	if runtime.GOOS != goosLinux {
-		t.Skip("only Linux has /var/tmp among its default excludes")
-	}
-
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	rules := walkRules{nonPersistent: nonPersistent}
-
-	for _, dir := range []string{"/var", "/var/tmp"} {
-		fi, lstatErr := os.Lstat(dir)
-		if lstatErr != nil || !fi.IsDir() ||
-			rules.nonPersistentLine(dir, fi) != "" {
-			t.Skip(dir + " is not a directory on a persistent filesystem here")
-		}
-	}
-
-	//nolint:usetesting // the directory must be inside /var/tmp
-	tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-")
-	if err != nil {
-		t.Skip("cannot write to /var/tmp here: " + err.Error())
-	}
-
-	t.Cleanup(func() { _ = os.RemoveAll(tmp) })
-
-	f := writeFile(t, tmp, "f", pattern(1, 10))
-
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	line := "scan: skipped /var/tmp (default exclude)\n"
-
-	if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
-		t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
-	}
-
-	assertRecorded(t, path, f, false)
-
-	stderr := scanOK(t, "--no-default-excludes", "/var")
-	if strings.Contains(stderr, line) {
-		t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q",
-			stderr, line)
-	}
-
-	assertRecorded(t, path, f, true)
-}
-
-// TestRunScanSkipsNonPersistentOperand scans, through the command, a
-// directory on a filesystem that is not persistent, with and without
-// --no-default-excludes: either way it is skipped, with a line saying
-// so, and nothing in it is recorded.
-func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
-	shm, fsType, _ := shmTempDir(t)
-	writeFile(t, shm, "f", pattern(1, 10))
-
-	t.Setenv(databaseEnv, testDBPath(t))
-
-	line := "scan: skipped " + shm + " (" + fsType +
-		", not a persistent filesystem)\n"
-	summary := "scan: 0 files seen"
-
-	for _, flags := range [][]string{nil, {"--no-default-excludes"}} {
-		stderr := scanOK(t, append(flags, shm)...)
-		if !strings.Contains(stderr, line) ||
-			!strings.Contains(stderr, summary) {
-			t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr,
-				line, summary)
-		}
-	}
-}
-
-// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem
-// that is not persistent beside one on the tests' temporary directory.
-// The records an earlier scan left in it count as outside the operands:
-// the record of a file gone from it is kept, while a file added to it
-// is not recorded.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
-	shm, fsType, rules := shmTempDir(t)
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	db := openTestDB(t)
-
-	// A scan that, without the list of filesystems, records gone.
-	gone := writeFile(t, shm, "gone", pattern(1, 100))
-	syncTree(t, db, shm)
-
-	err := os.Remove(gone)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	unseen := writeFile(t, shm, "new", pattern(2, 10))
-
-	st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if st.skipped != 0 {
-		t.Errorf("skipped = %d, want 0", st.skipped)
-	}
-
-	recs := dbRecords(t, db)
-	recordByPath(t, recs, gone)
-
-	if slices.Contains(recordPaths(recs), unseen) {
-		t.Errorf("%s recorded", unseen)
-	}
-
-	line := "scan: skipped " + shm + " (" + fsType +
-		", not a persistent filesystem)\n"
-	if !strings.Contains(stderr(), line) {
-		t.Errorf("stderr = %q, want %q", stderr(), line)
-	}
-}
-
-// TestWalkLeavesOutNonPersistent starts the walk itself in a directory
-// on a filesystem that is not persistent, which a scan never does: it
-// drops such an operand first. Below a directory on a persistent
-// filesystem, one on another takes a mount, which a test cannot make.
-// The walk enters no directory and records no file there, and names
-// each.
-func TestWalkLeavesOutNonPersistent(t *testing.T) {
-	t.Parallel()
-
-	shm, fsType, rules := shmTempDir(t)
-	f := writeFile(t, shm, "f", pattern(1, 10))
-	sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10)))
-
-	var leftOut []string
-
-	for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) {
-		if ev.rec.path != "" {
-			t.Errorf("walk recorded %s", ev.rec.path)
-		}
-
-		if ev.leftOut == "" {
-			continue
-		}
-
-		leftOut = append(leftOut, ev.leftOut)
-
-		want := "scan: skipped " + ev.leftOut + " (" + fsType +
-			", not a persistent filesystem)"
-		if ev.warn != want {
-			t.Errorf("line = %q, want %q", ev.warn, want)
-		}
-	}
-
-	slices.Sort(leftOut)
-
-	if want := []string{f, sub}; !slices.Equal(leftOut, want) {
-		t.Errorf("walk left out %q, want %q", leftOut, want)
-	}
-}
-
-// shmTempDir returns a new directory in /dev/shm, removed when the test
-// ends, the type of its filesystem, and the rules holding the real
-// filesystems that are not persistent. It skips the test where /dev/shm
-// is on a persistent filesystem, or cannot be written to.
-func shmTempDir(t *testing.T) (string, string, walkRules) {
-	t.Helper()
-
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dev, ok := deviceOfPath("/dev/shm")
-
-	fsType, found := nonPersistent[dev]
-	if !ok || !found {
-		t.Skip("/dev/shm is not a memory-backed filesystem here")
-	}
-
-	//nolint:usetesting // the directory must be inside /dev/shm
-	dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-")
-	if err != nil {
-		t.Skip("cannot write to /dev/shm here: " + err.Error())
-	}
-
-	t.Cleanup(func() { _ = os.RemoveAll(dir) })
-
-	return dir, fsType, walkRules{nonPersistent: nonPersistent}
-}
-
-// assertRecorded checks that the database at dbPath holds a record for
-// path when want is set, and none when it is not.
-func assertRecorded(t *testing.T, dbPath, path string, want bool) {
-	t.Helper()
-
-	db, err := openDB(dbPath, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = db.Close() }()
-
-	if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want {
-		t.Errorf("%s recorded: %v, want %v", path, got, want)
-	}
-}

go.mod

@@ -24,5 +24,3 @@ require (
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 )
-
-retract v0.0.1 // early build without a database; does not work as the README describes

info.go

@@ -1,125 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-)
-
-// infoPhase names one phase of info with its place among all of them,
-// as "[2/2] count", for the phase's progress display. The list here is
-// the one place the phases are counted.
-func infoPhase(name string) string {
-	return phaseLabel([]string{openPhase, "count"}, name)
-}
-
-// runInfo implements the info subcommand: it prints the database's
-// details on stdout (see writeInfo). It opens the database as report
-// does, read-only and without the scan lock, and never touches the
-// scanned filesystem. It names the database on stderr before it opens
-// it. Any database problem, including a missing database, is fatal.
-//
-// The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and count (read the number of
-// records and their total size from the totals table). Each display
-// ends before the next phase starts, and before an error is returned
-// to be printed. The details are written once the last display has
-// ended, so no display is drawn while they are written.
-func runInfo(ctx context.Context, stdout io.Writer) error {
-	dbPath, source, err := databasePath()
-	if err != nil {
-		return err
-	}
-
-	fmt.Fprintf(os.Stderr, "info: database %s (%s)\n",
-		escapePath(dbPath), source)
-
-	// Opening can wait on a busy database for as long as the busy
-	// timeout.
-	prog := newProgress(progressParams{
-		label: infoPhase(openPhase), total: -1,
-		waitingFor: openingDatabase,
-	})
-	db, err := openReportDatabase(ctx, dbPath)
-
-	prog.finish()
-
-	if err != nil {
-		return err
-	}
-
-	defer func() { _ = db.Close() }()
-
-	// One row holds both counts, so the phase never has a total.
-	prog = newProgress(progressParams{
-		label: infoPhase("count"), total: -1,
-		waitingFor: countingRecords,
-	})
-	counts, err := readDatabaseCounts(ctx, db)
-
-	prog.finish()
-
-	if err != nil {
-		return fmt.Errorf("database %s: %w", dbPath, err)
-	}
-
-	return writeInfo(stdout, dbPath, source, counts)
-}
-
-// writeInfo writes the details of the database at dbPath to stdout,
-// one per line, as a name, a tab and the value (README "`info` mode"):
-// the path and source as databasePath returns them, the sizes of the
-// database file and of its -wal file when there is one, when the
-// database file was last changed, and the schema version and record
-// counts that readDatabaseCounts read.
-func writeInfo(stdout io.Writer, dbPath, source string,
-	counts databaseCounts,
-) error {
-	// SQLite keeps the -wal file beside the database file, named from
-	// its path with symlinks resolved.
-	file, err := filepath.EvalSymlinks(dbPath)
-	if err != nil {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	dbFile, err := os.Stat(file)
-	if err != nil {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	walFile, err := os.Stat(file + "-wal")
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	var b strings.Builder
-
-	fmt.Fprintf(&b, "database\t%s (%s)\n", escapePath(dbPath), source)
-	fmt.Fprintf(&b, "database file size\t%d (%s)\n",
-		dbFile.Size(), humanBytes(dbFile.Size()))
-
-	if walFile != nil {
-		fmt.Fprintf(&b, "-wal file size\t%d (%s)\n",
-			walFile.Size(), humanBytes(walFile.Size()))
-	}
-
-	fmt.Fprintf(&b, "modified\t%s\n",
-		dbFile.ModTime().UTC().Format(time.RFC3339))
-	fmt.Fprintf(&b, "schema version\t%d\n", counts.schemaVersion)
-	fmt.Fprintf(&b, "records\t%d\n", counts.records)
-	fmt.Fprintf(&b, "size of recorded files\t%d (%s)\n",
-		counts.size, humanBytes(counts.size))
-
-	_, err = io.WriteString(stdout, b.String())
-	if err != nil {
-		return fmt.Errorf("write stdout: %w", err)
-	}
-
-	return nil
-}

info_test.go

@@ -1,166 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-)
-
-// infoOpenLine is the plain line of info's open phase when it opens the
-// database at once; it prints it as the phase starts and as it ends.
-const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
-	"remain unknown"
-
-// infoCountLine is the plain line of info's count phase when the count
-// takes under a second; it prints it as the phase starts and as it
-// ends.
-const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
-	"remain unknown"
-
-// infoRecs are four records: a duplicate pair, a file of 20 MiB and
-// another small file.
-func infoRecs() []scanRec {
-	h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u")
-	bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc")
-
-	return []scanRec{
-		{size: 5, head: h, tail: tl, content: c, path: "/d/a"},
-		{size: 5, head: h, tail: tl, content: c, path: "/d/b"},
-		{size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"},
-		{size: 7, head: u, tail: u, content: u, path: "/d/unique"},
-	}
-}
-
-// TestRunInfo runs info on an empty and on a filled database, with
-// stdout and stderr the same file that is not a terminal, and checks
-// everything written there: the line naming the database, each phase's
-// line as it starts and as it ends, and then the details, each on a
-// line of its own. No -wal file is beside a database scan has closed.
-func TestRunInfo(t *testing.T) {
-	cases := map[string]struct {
-		recs []scanRec
-		want string // the details after the database file's time
-	}{
-		"empty": {
-			want: "schema version\t1\n" +
-				"records\t0\n" +
-				"size of recorded files\t0 (0 B)\n",
-		},
-		"filled": {
-			recs: infoRecs(),
-			want: "schema version\t1\n" +
-				"records\t4\n" +
-				"size of recorded files\t20971537 (20.0 MiB)\n",
-		},
-	}
-
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			path := seedDatabase(t, tc.recs)
-			t.Setenv(databaseEnv, path)
-
-			file, err := os.Stat(path)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			stderr := captureStderr(t)
-
-			code := run([]string{cmdInfo}, os.Stderr, os.Stderr)
-			if code != exitOK {
-				t.Fatalf("run(info) = %d, want %d; stderr: %s",
-					code, exitOK, stderr())
-			}
-
-			named := path + " (from " + databaseEnv + ")\n"
-			want := "info: database " + named +
-				infoOpenLine + "\n" + infoOpenLine + "\n" +
-				infoCountLine + "\n" + infoCountLine + "\n" +
-				"database\t" + named +
-				fmt.Sprintf("database file size\t%d (%s)\n",
-					file.Size(), humanBytes(file.Size())) +
-				"modified\t" + file.ModTime().UTC().Format(time.RFC3339) +
-				"\n" + tc.want
-
-			if got := stderr(); got != want {
-				t.Errorf("stdout and stderr = %q, want %q", got, want)
-			}
-		})
-	}
-}
-
-// TestRunInfoDuringScan runs info while a scan holds the lock and has
-// the database open in WAL mode, with records committed to the -wal
-// file only. info must count those records and show the -wal file's
-// size, also when the database path is a symlink: SQLite keeps the
-// -wal file beside the symlink's target.
-func TestRunInfoDuringScan(t *testing.T) {
-	path := testDBPath(t)
-	link := filepath.Join(t.TempDir(), "link.sqlite")
-
-	err := os.Symlink(path, link)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db, err := openScanDatabase(t.Context(), path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	err = applyChanges(t.Context(), db, infoRecs(), nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	holdScanLock(t, path)
-
-	wal, err := os.Stat(path + "-wal")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, named := range []string{path, link} {
-		t.Run(filepath.Base(named), func(t *testing.T) {
-			t.Setenv(databaseEnv, named)
-
-			stdout := runStdout(t, cmdInfo)
-
-			for _, want := range []string{
-				"database\t" + named + " (from " + databaseEnv + ")\n",
-				fmt.Sprintf("\n-wal file size\t%d (%s)\n",
-					wal.Size(), humanBytes(wal.Size())),
-				"\nrecords\t4\n",
-			} {
-				if !strings.Contains(stdout, want) {
-					t.Errorf("stdout = %q, want it to hold %q", stdout, want)
-				}
-			}
-		})
-	}
-}
-
-func TestRunInfoNeedsOnlyReadAccess(t *testing.T) {
-	// README "`info` mode": like report, info needs only read access to
-	// the database file and none to its directory.
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	scanFixture(t)
-	makeReadOnly(t, path)
-
-	stdout := runStdout(t, cmdInfo)
-
-	if want := "\nrecords\t2\n"; !strings.Contains(stdout, want) {
-		t.Errorf("stdout = %q, want it to hold %q", stdout, want)
-	}
-
-	if strings.Contains(stdout, "-wal") {
-		t.Errorf("stdout = %q, want no -wal file between scans", stdout)
-	}
-}

main.go

@@ -1,19 +1,20 @@
 // Command sfdupes quickly identifies candidate duplicate files across
 // very large filesystems without reading every byte of every file.
 // Files are considered duplicates when their sizes are equal and they
-// agree on three SHA-256 hashes: of the first 64 KiB, of the last 64
+// agree on a short ladder of SHA-256 hashes. A file under 10 MiB is
-// KiB, and of the content. The content hash covers the whole file under
+// hashed in full. A larger file is compared on the hashes of its first
-// 100 MiB, and 1 MiB at each gigabyte at 100 MiB or more, so a large
+// and last 64 KiB, and only when those match another file's is its
-// file is never read in full. scan maintains a persistent SQLite
+// content hash computed and compared: of the whole file when it is
-// database of file signatures (SFDUPES_DATABASE, default
+// under 50 MiB, or of gigabyte-spaced 1 MiB samples when it is 50 MiB
-// /var/lib/sfdupes/db.sqlite) that the other subcommands read.
+// or larger. scan maintains a persistent SQLite database of file
+// signatures (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite)
+// that the reporting subcommands read.
 //
 // Usage:
 //
-//	sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...
+//	sfdupes scan [--workers N] [-x] PATH...
 //	sfdupes report > dupes.tsv
 //	sfdupes trees > dupetrees.tsv
-//	sfdupes info
 //	sfdupes --version
 //
 // See README.md for the complete specification.
@@ -26,7 +27,6 @@ import (
 	"io"
 	"os"
 	"runtime"
-	"runtime/debug"
 
 	"github.com/spf13/cobra"
 )
@@ -44,7 +44,6 @@ const (
 	cmdScan   = "scan"
 	cmdReport = "report"
 	cmdTrees  = "trees"
-	cmdInfo   = "info"
 )
 
 // errNoSubcommand is returned by the root command when it is invoked
@@ -57,32 +56,10 @@ var errNoSubcommand = errors.New("no subcommand")
 var errWorkersBelowOne = errors.New("--workers must be at least 1")
 
 // Version is the build version, injected at link time via -ldflags
-// (see the Makefile); empty when the linker set none, as for a binary
+// (see the Makefile); "dev" for a plain go build.
-// built by go install. buildVersion turns it into what --version prints.
 //
 //nolint:gochecknoglobals // written only by the linker
-var Version string
+var Version = "dev"
-
-// buildVersion returns the version that sfdupes --version prints
-// (README "Install"): the version the linker set, when it set one;
-// otherwise the main module's version that Go recorded in the binary,
-// a tag such as v0.0.2 or the pseudo-version of an @main build; and
-// "dev" when neither is known. info is what debug.ReadBuildInfo
-// returns, nil when the binary carries none. Go records "(devel)" for
-// a build that has no version, such as one inside a copy of the source
-// without git.
-func buildVersion(linkerVersion string, info *debug.BuildInfo) string {
-	if linkerVersion != "" {
-		return linkerVersion
-	}
-
-	if info == nil || info.Main.Version == "" ||
-		info.Main.Version == "(devel)" {
-		return "dev"
-	}
-
-	return info.Main.Version
-}
 
 func main() {
 	// Once the reader of a stdout pipe has gone, as in "sfdupes report |
@@ -96,8 +73,8 @@ func main() {
 // exit code. It is the program's single exit point: the subcommands
 // return their errors instead of exiting, so every deferred cleanup —
 // above all closing the database, which checkpoints the SQLite WAL —
-// runs before the process ends. The report, trees and info subcommands
+// runs before the process ends. The report and trees subcommands write
-// write their data to stdout.
+// their data to stdout.
 func run(args []string, stdout, stderr io.Writer) int {
 	// A nil slice makes cobra fall back to os.Args, which would let a
 	// test binary's own flags reach the command tree.
@@ -137,10 +114,7 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 	var showVersion bool
 
 	printVersion := runE(func(context.Context, []string) error {
-		info, _ := debug.ReadBuildInfo()
+		_, err := fmt.Fprintf(stdout, "sfdupes %s\n", Version)
-
-		_, err := fmt.Fprintf(stdout, "sfdupes %s\n",
-			buildVersion(Version, info))
 		if err != nil {
 			return fmt.Errorf("write stdout: %w", err)
 		}
@@ -174,6 +148,30 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 	root.Flags().BoolVarP(&showVersion, "version", "v", false,
 		"print the version to stdout")
 
+	var (
+		scanWorkers int
+		scanOneFS   bool
+	)
+
+	scanCmd := &cobra.Command{
+		Use:   cmdScan + " [--workers N] [-x] PATH...",
+		Short: "Walk trees and synchronize the scan database",
+		Args:  cobra.MinimumNArgs(1),
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			return checkScanWorkers(cmd, scanWorkers)
+		},
+		RunE: runE(func(ctx context.Context, args []string) error {
+			ctx, stop := interruptContext(ctx)
+			defer stop()
+
+			return runScan(ctx, args, scanWorkers, scanOneFS)
+		}),
+	}
+	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
+		"concurrent workers for the walk, hash, and content phases")
+	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
+		"do not cross filesystem boundaries")
+
 	reportCmd := &cobra.Command{
 		Use:   cmdReport,
 		Short: "Read the scan database and print the file-level duplicates report",
@@ -192,59 +190,11 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command {
 		}),
 	}
 
-	infoCmd := &cobra.Command{
+	root.AddCommand(scanCmd, reportCmd, treesCmd)
-		Use:   cmdInfo,
-		Short: "Read the scan database and print its details",
-		Args:  cobra.NoArgs,
-		RunE: runE(func(ctx context.Context, _ []string) error {
-			return runInfo(ctx, stdout)
-		}),
-	}
-
-	root.AddCommand(newScanCommand(), reportCmd, treesCmd, infoCmd)
 
 	return root
 }
 
-// newScanCommand builds the scan subcommand, with its --workers, -x and
-// --no-default-excludes flags. Its help lists the default excludes of
-// the system it runs on, and the filesystem types it never records
-// anything on.
-func newScanCommand() *cobra.Command {
-	var (
-		scanWorkers           int
-		scanOneFS             bool
-		scanNoDefaultExcludes bool
-	)
-
-	short := "Walk trees and synchronize the scan database"
-
-	scanCmd := &cobra.Command{
-		Use:   cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
-		Short: short,
-		Long:  short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
-		Args:  cobra.MinimumNArgs(1),
-		PreRunE: func(cmd *cobra.Command, _ []string) error {
-			return checkScanWorkers(cmd, scanWorkers)
-		},
-		RunE: runE(func(ctx context.Context, args []string) error {
-			ctx, stop := interruptContext(ctx)
-			defer stop()
-
-			return runScan(ctx, args, scanWorkers, scanOneFS,
-				scanNoDefaultExcludes)
-		}),
-	}
-	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
-		"concurrent workers for the walk and hash phases")
-	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
-		"do not cross filesystem boundaries")
-	scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes",
-		false, "enter the default excludes, the paths listed above")
-
-	return scanCmd
-}
-
 // checkScanWorkers rejects a scan --workers value below 1. That is a
 // usage error reported in one line: cobra prints the returned message
 // without the usage text, and run exits 2.

main_test.go

@@ -4,12 +4,10 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
-	"runtime/debug"
 	"slices"
 	"strconv"
 	"strings"
@@ -20,36 +18,6 @@ import (
 // error prints and a runtime failure must not.
 const usageMarker = "Usage:"
 
-// TestMain stops the tests at once, saying why, when their temporary
-// directory is not on a persistent filesystem: scan records nothing
-// there (README "Rules for the walk"), so most scan tests would fail
-// without saying why.
-func TestMain(m *testing.M) {
-	nonPersistent, err := nonPersistentFilesystems()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "list the mounted filesystems: %v\n", err)
-		os.Exit(1)
-	}
-
-	dir := os.TempDir()
-
-	fi, err := os.Stat(dir)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-
-	dev, ok := deviceOfInfo(fi)
-	if fsType, found := nonPersistent[dev]; ok && found {
-		fmt.Fprintf(os.Stderr, "%s is on %s, not a persistent filesystem, "+
-			"and scan records nothing there: set TMPDIR to a directory on a "+
-			"persistent filesystem\n", dir, fsType)
-		os.Exit(1)
-	}
-
-	os.Exit(m.Run())
-}
-
 // walSuffixes are the SQLite sidecar files a WAL-mode database keeps
 // while it is open. A clean close checkpoints the WAL and removes
 // both; finding either afterwards means the database was never closed.
@@ -180,9 +148,9 @@ func capture(t *testing.T, std **os.File) func() string {
 }
 
 // brokenDatabase writes a database that opens cleanly and passes the
-// schema-version check but has no tables, so the first query fails
+// schema-version check but has no files table, so the first query
-// with the database already open: a fatal error on a path that owns an
+// fails with the database already open: a fatal error on a path that
-// open database. It closes the database the way scan does.
+// owns an open database. It closes the database the way scan does.
 func brokenDatabase(t *testing.T) string {
 	t.Helper()
 
@@ -265,9 +233,8 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
 			assertFatalOutput(t, stderr.String(), stdout())
 
 			// Proof that the failure happened after the open: only a
-			// query against the opened database can report this. Each
+			// query against the opened database can report this.
-			// subcommand's first query reads totals.
+			if !strings.Contains(stderr.String(), "no such table: files") {
-			if !strings.Contains(stderr.String(), "no such table: totals") {
 				t.Errorf("stderr = %q, want the failure to come from a "+
 					"query on the open database", stderr.String())
 			}
@@ -317,16 +284,10 @@ func assertFatalOutput(t *testing.T, stderr, stdout string) {
 }
 
 func TestRunMissingDatabaseIsFatal(t *testing.T) {
-	// README §Database: report, trees and info need an existing
+	// README §Database: report and trees need an existing database; a
-	// database; a missing one exits 1 with a message telling the user to
+	// missing one exits 1 with a message telling the user to run scan,
-	// run scan, after the line naming the database.
+	// after the line naming the database.
-	openLines := map[string]string{
+	for _, name := range []string{cmdReport, cmdTrees} {
-		cmdReport: reportOpenLine,
-		cmdTrees:  treesOpenLine,
-		cmdInfo:   infoOpenLine,
-	}
-
-	for name, openLine := range openLines {
 		t.Run(name, func(t *testing.T) {
 			path := testDBPath(t)
 			t.Setenv(databaseEnv, path)
@@ -342,10 +303,8 @@ func TestRunMissingDatabaseIsFatal(t *testing.T) {
 				t.Errorf("run(%s) = %d, want %d", name, code, exitFatal)
 			}
 
-			// Each shows its open phase, as it starts and as it ends,
-			// before the error.
 			want := name + ": database " + path + " (from " + databaseEnv +
-				")\n" + openLine + "\n" + openLine + "\n" +
+				")\n" +
 				"sfdupes: " + path + ": no database (run \"sfdupes " +
 				"scan\" first, or set " + databaseEnv + ")\n"
 			if got := stderr(); got != want {
@@ -546,9 +505,8 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
 	// exist at the default path on the machine running the test, so
 	// nothing here may create or read one: scan gets an operand that
 	// does not exist, which stops it before it takes the lock, and
-	// report, trees and info a cancelled context, which stops them
+	// report and trees a cancelled context, which stops them before
-	// before they read anything. They show their open phase after the
+	// they read anything.
-	// line.
 	t.Setenv(databaseEnv, "")
 
 	// Whether scan calls the database new depends on that machine too.
@@ -568,7 +526,7 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
 	}{
 		cmdScan: {
 			run: func() error {
-				return runScan(t.Context(), []string{missing}, 1, false, false)
+				return runScan(t.Context(), []string{missing}, 1, false)
 			},
 			want: scanWant,
 		},
@@ -576,22 +534,13 @@ func TestRunNamesDefaultDatabase(t *testing.T) {
 			run: func() error {
 				return runReport(cancelledContext(t), io.Discard)
 			},
-			want: "report: database " + defaultDatabasePath + " (default)\n" +
+			want: "report: database " + defaultDatabasePath + " (default)\n",
-				reportOpenLine + "\n" + reportOpenLine + "\n",
 		},
 		cmdTrees: {
 			run: func() error {
 				return runTrees(cancelledContext(t), io.Discard)
 			},
-			want: "trees: database " + defaultDatabasePath + " (default)\n" +
+			want: "trees: database " + defaultDatabasePath + " (default)\n",
-				treesOpenLine + "\n" + treesOpenLine + "\n",
-		},
-		cmdInfo: {
-			run: func() error {
-				return runInfo(cancelledContext(t), io.Discard)
-			},
-			want: "info: database " + defaultDatabasePath + " (default)\n" +
-				infoOpenLine + "\n" + infoOpenLine + "\n",
 		},
 	}
 
@@ -619,7 +568,6 @@ func TestRunUsageErrors(t *testing.T) {
 		"scan without paths": {[]string{cmdScan}, usageMarker},
 		"report with args":   {[]string{cmdReport, "x"}, usageMarker},
 		"trees with args":    {[]string{cmdTrees, "x"}, usageMarker},
-		"info with args":     {[]string{cmdInfo, "x"}, usageMarker},
 		"unknown flag":       {[]string{cmdScan, "--nope", "/"}, usageMarker},
 		"unknown subcommand": {[]string{"nope"}, "unknown command"},
 	}
@@ -716,8 +664,6 @@ func TestRunVersion(t *testing.T) {
 
 	// README §Subcommands: the version is one line on stdout, with
 	// nothing on stderr, and exits 0.
-	info, _ := debug.ReadBuildInfo()
-
 	for _, arg := range []string{"--version", "-v"} {
 		var stdout, stderr bytes.Buffer
 
@@ -726,7 +672,7 @@ func TestRunVersion(t *testing.T) {
 			t.Errorf("run(%s) = %d, want %d", arg, code, exitOK)
 		}
 
-		want := "sfdupes " + buildVersion(Version, info) + "\n"
+		want := "sfdupes " + Version + "\n"
 		if got := stdout.String(); got != want {
 			t.Errorf("run(%s) stdout = %q, want %q", arg, got, want)
 		}
@@ -737,40 +683,6 @@ func TestRunVersion(t *testing.T) {
 	}
 }
 
-func TestBuildVersion(t *testing.T) {
-	t.Parallel()
-
-	// README "Install": the linker's version wins; otherwise the main
-	// module's version that Go recorded; "dev" when neither is known.
-	tagged := &debug.BuildInfo{Main: debug.Module{Version: "v0.0.2"}}
-	pseudo := &debug.BuildInfo{Main: debug.Module{
-		Version: "v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1",
-	}}
-	devel := &debug.BuildInfo{Main: debug.Module{Version: "(devel)"}}
-
-	cases := []struct {
-		name   string
-		linker string
-		info   *debug.BuildInfo
-		want   string
-	}{
-		{"linker set", "v0.0.2-3-g4e4a94d", tagged, "v0.0.2-3-g4e4a94d"},
-		{"linker set, no build info", "4e4a94d", nil, "4e4a94d"},
-		{"build info with a tag", "", tagged, "v0.0.2"},
-		{"build info with a pseudo-version", "", pseudo,
-			"v1.0.0-rc2.0.20261016120000-4e4a94d0c2a1"},
-		{"(devel)", "", devel, "dev"},
-		{"no build info", "", nil, "dev"},
-	}
-
-	for _, c := range cases {
-		if got := buildVersion(c.linker, c.info); got != c.want {
-			t.Errorf("%s: buildVersion(%q, ...) = %q, want %q",
-				c.name, c.linker, got, c.want)
-		}
-	}
-}
-
 func TestRunVersionWriteFailureIsFatal(t *testing.T) {
 	t.Parallel()
 
@@ -818,23 +730,6 @@ func scanFixture(t *testing.T) []string {
 	return dupes
 }
 
-// storedDiskSize returns the disk_size of the record for path in the
-// database at dbPath. A test of a trees run compares with this rather
-// than with a new lstat, which on ZFS can differ from the scan's (see
-// diskSizeOf).
-func storedDiskSize(t *testing.T, dbPath, path string) int64 {
-	t.Helper()
-
-	db, err := openReportDatabase(t.Context(), dbPath)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = db.Close() }()
-
-	return recordByPath(t, dbRecords(t, db), path).diskSize
-}
-
 // scanOK runs scan over operands, fails the test unless it exits 0 with
 // nothing on stdout, and returns everything it printed to stderr.
 func scanOK(t *testing.T, operands ...string) string {
@@ -946,21 +841,24 @@ func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
 }
 
 func TestRunScanSkipsZFSOperand(t *testing.T) {
-	zfs := filepath.Join(t.TempDir(), ".zfs")
+	path := testDBPath(t)
-	f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
-
-	// No scan records a file inside a .zfs directory, so the record
-	// beneath the operand is written straight into the database.
-	path := seedDatabase(t, []scanRec{{size: 10, path: f}})
 	t.Setenv(databaseEnv, path)
 
+	zfs := filepath.Join(t.TempDir(), ".zfs")
+	snapshot := filepath.Join(zfs, "snapshot", "hourly")
+	f := writeFile(t, snapshot, "f", pattern(1, 10))
+
+	// An operand beneath a .zfs directory is walked, because it is not
+	// itself named .zfs.
+	scanOK(t, snapshot)
+
 	assertOperandSkipped(t, path, zfs, ".zfs directory", f)
 }
 
 // assertOperandSkipped scans operand alone and checks that it is skipped
 // as kind: a warning naming it, one skip in the summary, exit 0, and the
-// record for kept, stored beneath operand before this scan, still in the
+// record for kept, which an earlier scan stored beneath operand, still
-// database at dbPath.
+// in the database at dbPath.
 func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	kept string,
 ) {
@@ -989,121 +887,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	recordByPath(t, dbRecords(t, db), kept)
 }
 
-// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
-// directory, first alone and then under another operand. Each time it
-// is warned about and counted as skipped, and nothing in it is walked.
-// The other operand is scanned as usual.
-func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
-	writeFile(t, hourly, "f", pattern(1, 10))
-	f := writeFile(t, dir, "f", pattern(2, 10))
-
-	warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
-
-	for _, tc := range []struct {
-		operands []string
-		summary  string
-	}{
-		{
-			[]string{hourly},
-			"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-		{
-			[]string{dir, hourly},
-			"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-	} {
-		stderr := scanOK(t, tc.operands...)
-		if !strings.Contains(stderr, warning) ||
-			!strings.Contains(stderr, tc.summary) {
-			t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
-				stderr, warning, tc.summary)
-		}
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
-		t.Errorf("records = %q, want %q", got, []string{f})
-	}
-}
-
-// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
-// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
-// by the path given, counted as skipped, and not walked.
-func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
-	t.Setenv(databaseEnv, testDBPath(t))
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
-
-	snaps := filepath.Join(dir, "snaps")
-
-	err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(snaps, "daily")
-	stderr := scanOK(t, daily)
-
-	warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
-	summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-		"0 unchanged), 1 skipped\n"
-
-	if !strings.Contains(stderr, warning) ||
-		!strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
-	}
-}
-
-// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
-// a symlink to a directory outside any .zfs directory. The operand is
-// scanned as usual, and its file is recorded by the path given.
-func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
-
-	data := filepath.Join(dir, "data")
-
-	err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(data, "daily")
-	stderr := scanOK(t, daily)
-
-	summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-		"0 unchanged), 0 skipped\n"
-	if !strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q", stderr, summary)
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
-}
-
 func TestRunReportSucceeds(t *testing.T) {
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
@@ -1125,16 +908,11 @@ func TestRunReportSucceeds(t *testing.T) {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
 
-	// Its phases come between the two lines; see
+	want = "report: database " + path + " (from " + databaseEnv + ")\n" +
-	// TestRunReportProgressNamesEachPhase.
+		"report: 2 records read, 1 duplicate groups, 1 dupe files, " +
-	first := "report: database " + path + " (from " + databaseEnv + ")\n"
-	last := "\nreport: 2 records read, 1 duplicate groups, 1 dupe files, " +
 		"300 B reclaimable\n"
-
+	if got := stderr(); got != want {
-	if got := stderr(); !strings.HasPrefix(got, first) ||
+		t.Errorf("stderr = %q, want %q", got, want)
-		!strings.HasSuffix(got, last) {
-		t.Errorf("stderr = %q, want it to start %q and end %q",
-			got, first, last)
 	}
 
 	assertNoSidecars(t, path)
@@ -1158,23 +936,17 @@ func TestRunTreesSucceeds(t *testing.T) {
 
 	// The two directories holding the duplicate pair are duplicate
 	// trees of each other.
-	want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
+	want := "first\tdupe\tfiles\tsize\n" +
-		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
+		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
-		strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
 	if got := stdout.String(); got != want {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
 
-	// Its phases come between the two lines; see
+	want = "trees: database " + path + " (from " + databaseEnv + ")\n" +
-	// TestRunTreesProgressNamesEachPhase.
+		"trees: 2 records read, 1 duplicate tree groups, 1 dupe trees, " +
-	first := "trees: database " + path + " (from " + databaseEnv + ")\n"
+		"300 B reclaimable\n"
-	last := "\ntrees: 2 records read, 1 duplicate tree groups, " +
+	if got := stderr(); got != want {
-		"1 dupe trees, 300 B reclaimable\n"
+		t.Errorf("stderr = %q, want %q", got, want)
-
-	if got := stderr(); !strings.HasPrefix(got, first) ||
-		!strings.HasSuffix(got, last) {
-		t.Errorf("stderr = %q, want it to start %q and end %q",
-			got, first, last)
 	}
 
 	assertNoSidecars(t, path)
@@ -1194,10 +966,9 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
 	cases := map[string]string{
 		cmdReport: "first\tdupe\tsize\n" +
 			dupes[0] + "\t" + dupes[1] + "\t300\n",
-		cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
+		cmdTrees: "first\tdupe\tfiles\tsize\n" +
 			filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
-			"\t1\t300\t" +
+			"\t1\t300\n",
-			strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
 	}
 
 	for name, want := range cases {
@@ -1232,11 +1003,8 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 		t.Errorf("report stdout = %q, want %q", got, want)
 	}
 
-	diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
+	want = "first\tdupe\tfiles\tsize\n" +
-
+		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
-	want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
-		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
-		strconv.FormatInt(diskSize, 10) + "\n"
 	if got := runStdout(t, cmdTrees); got != want {
 		t.Errorf("trees stdout = %q, want %q", got, want)
 	}
@@ -1470,7 +1238,7 @@ func TestRunReportsDuringScan(t *testing.T) {
 func TestRunStdoutClosedIsFatal(t *testing.T) {
 	// README §Error handling: a stdout write failure exits 1, reported
 	// in one line on stderr.
-	for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
+	for _, name := range []string{cmdReport, cmdTrees} {
 		t.Run(name, func(t *testing.T) {
 			t.Setenv(databaseEnv, testDBPath(t))
 
@@ -1520,7 +1288,6 @@ func TestStdoutWriteErrorPropagates(t *testing.T) {
 	cases := map[string]func(context.Context, io.Writer) error{
 		cmdReport: runReport,
 		cmdTrees:  runTrees,
-		cmdInfo:   runInfo,
 	}
 
 	for name, fn := range cases {

progress.go

@@ -2,14 +2,10 @@ package main
 
 import (
 	"fmt"
-	"io"
-	"math"
 	"os"
 	"slices"
-	"strings"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	"github.com/schollz/progressbar/v3"
 	"golang.org/x/term"
@@ -23,54 +19,22 @@ const plainInterval = 5 * time.Second
 // a TTY.
 const redrawInterval = 100 * time.Millisecond
 
-// rateWindow is the time the rate a display shows covers: the items
-// completed over the last rateWindow, which is the time since the line
-// before when stderr is not a TTY.
-const rateWindow = 5 * time.Second
-
-// stallTime is how long a phase goes with no item completed before its
-// line says it has stalled, and where it waits, in place of the rate.
-const stallTime = 10 * time.Second
-
 // spinnerType selects the progressbar library's spinner style.
 const spinnerType = 14
 
-// barWidth is the width of a phase's bar between its two ends, as in
-// the example line in README "Progress".
-const barWidth = 10
-
-// barCell is one column of the filled part of a bar.
-const barCell = "█"
-
-// barColumns is the width of what percentBar returns: the percent in
-// four columns and its sign, a space, and the bar with its two ends.
-const barColumns = barWidth + 8
-
 // percentScale converts a fraction to a percentage.
 const percentScale = 100
 
-// fallbackWidth is the width taken for a terminal whose width cannot be
+// fallbackWidth is the width the progressbar library takes a terminal
-// read, as the progressbar library takes it.
+// to have when it cannot read it, as when stderr is not a terminal.
 const fallbackWidth = 80
 
-// spinnerMargin is the columns of a line besides its text: the spinner
+// spinnerMargin is the columns of a spinner's line besides its text:
-// and the three spaces the library draws around the text, and the
+// the spinner and the three spaces the library draws around the text,
-// terminal's last column, which every line stays clear of so that no
+// and the terminal's last column, which the line stays clear of, as the
-// terminal wraps it.
+// library's bars do.
 const spinnerMargin = 5
 
-// countingRecords is what the display of a phase that reads the records
-// says until it has the number of records, its total.
-const countingRecords = "counting the records"
-
-// openPhase is the name of the phase in which report, trees and info
-// open the database, and openingDatabase what its display says while
-// it does.
-const (
-	openPhase       = "open"
-	openingDatabase = "opening the database"
-)
-
 // stderrIsTTY reports whether stderr is attached to a terminal.
 func stderrIsTTY() bool {
 	return term.IsTerminal(int(os.Stderr.Fd()))
@@ -88,7 +52,7 @@ func terminalWidth() int {
 	return width
 }
 
-// phaseLabel names phase with its place among phases, as "[2/4] walk".
+// phaseLabel names phase with its place among phases, as "[2/5] walk".
 func phaseLabel(phases []string, phase string) string {
 	return fmt.Sprintf("[%d/%d] %s",
 		slices.Index(phases, phase)+1, len(phases), phase)
@@ -97,85 +61,73 @@ func phaseLabel(phases []string, phase string) string {
 // progressParams describe one phase's progress display as it starts.
 type progressParams struct {
 	label      string // the phase and its place, from phaseLabel
-	unit       string // what the phase counts, as "files"
 	total      int64  // the items the phase will complete; -1 when not known
 	estimate   bool   // total is an estimate, which the count may pass
 	waitingFor string // what the phase waits for until setTotal is called
-	stalledOn  string // where the phase waits, as "disk/db"; see stallTime
-}
-
-// drawnCount is the count of items completed as of one draw, and when
-// that draw was.
-type drawnCount struct {
-	at    time.Time
-	count int64
 }
 
 // progress renders one phase's progress on stderr. On a TTY the
-// progressbar library draws its spinner and then line, fitted to the
+// progressbar library draws it. A phase with an exact total from its
-// terminal's width. A phase with an exact total ends the line with its
+// start gets the library's bar, with the phase and its place as the
-// percent and bar, at the terminal's right edge, when the terminal has
+// bar's description and the library's counts, rate, elapsed time and
-// room for them. It is drawn again every redrawInterval, so its elapsed
+// ETA beside it; the bar takes the room the rest leaves on the line. Any
-// time keeps moving while the phase waits for its next item. When
+// other phase gets the library's spinner, with line as its text, cut to
-// stderr is not a TTY it emits no ANSI redraws: it prints line as the
+// the terminal's width. Either is drawn again every redrawInterval, so
-// phase starts, every plainInterval while it runs, and as it ends.
+// its elapsed time keeps moving while the phase waits for its next
+// item. When stderr is not a TTY it emits no ANSI redraws: it prints
+// line as the phase starts, every plainInterval while it runs, and as it
+// ends.
 //
 // The draws come from a goroutine of their own, so mu guards the fields
-// below it. Every method must be called from the main goroutine. A nil
+// below it and every call on bar. Every method must be called from the
-// *progress is a valid no-display receiver: every method is a no-op, so
+// main goroutine. A nil *progress is a valid no-display receiver: every
-// batched database flushes during the hash phase can reuse the update
+// method is a no-op, so batched database flushes during the hash phase
-// phase's helpers without rendering anything, and its writer writes
+// can reuse the update phase's helpers without rendering anything.
-// with no display to clear.
 type progress struct {
-	now   func() time.Time // reads the time
+	bar     *progressbar.ProgressBar // nil when stderr is not a TTY
-	width func() int       // reads the terminal's width; nil when not a TTY
+	spinner bool                     // bar is a spinner, with line as its text
-	stop  chan struct{}    // closed by finish to end the draws
+	stop    chan struct{}            // closed by finish to end the draws
-	done  chan struct{}    // closed once the draws have ended
+	done    chan struct{}            // closed once the draws have ended
 
 	mu         sync.Mutex
-	bar        *progressbar.ProgressBar // what the library draws; nil until then
-	widest     int                      // the widest line bar has drawn, in columns
 	label      string
-	unit       string
 	total      int64
 	estimate   bool
 	waitingFor string
-	stalledOn  string
 	count      int64
+	drawn      int64     // the items bar has been given, unless a spinner
 	start      time.Time // when the phase started, for the elapsed time
-	rateStart  time.Time // when the count started, for the time remaining
+	rateStart  time.Time // when the count started, for the rate
-	lastItem   time.Time // when the last item was completed, or the count started
-	// drawn holds the count at each draw over the last rateWindow, oldest
-	// first; the rate counts from the first. It keeps the draw before the
-	// latest even when that is older, as the line before is when lines
-	// are plainInterval apart and a tick comes late.
-	drawn []drawnCount
 }
 
 // newProgress starts the display of one phase on stderr.
 func newProgress(params progressParams) *progress {
 	if stderrIsTTY() {
-		return startProgress(params, terminalWidth, redrawInterval, time.Now)
+		return startProgress(params, true, redrawInterval)
 	}
 
-	return startProgress(params, nil, plainInterval, time.Now)
+	return startProgress(params, false, plainInterval)
 }
 
-// startProgress shows the phase params describes at once, and again
+// startProgress shows the phase params describes at once, with the
-// every interval until finish: with the progressbar library on a
+// progressbar library when onTTY is set and as a plain line when not,
-// terminal, whose width each draw reads by calling width, or as a plain
+// and again every interval until finish.
-// line when width is nil. The display reads the time by calling now.
+func startProgress(params progressParams, onTTY bool,
-func startProgress(params progressParams, width func() int,
+	interval time.Duration,
-	interval time.Duration, now func() time.Time,
 ) *progress {
-	start := now()
+	now := time.Now()
 	p := &progress{
-		now: now, width: width,
 		stop: make(chan struct{}), done: make(chan struct{}),
-		label: params.label, unit: params.unit, total: params.total,
+		label: params.label, total: params.total, estimate: params.estimate,
-		estimate: params.estimate, waitingFor: params.waitingFor,
+		waitingFor: params.waitingFor, start: now, rateStart: now,
-		stalledOn: params.stalledOn, start: start, rateStart: start,
+	}
-		lastItem: start, drawn: []drawnCount{{at: start}},
+
+	switch {
+	case onTTY && (params.total < 0 || params.estimate):
+		// The library's bar needs an exact total.
+		p.bar, p.spinner = newSpinner(), true
+	case onTTY:
+		p.bar = newBar(params.label, params.total)
 	}
 
 	// Show the zero state at once: the first item may take minutes,
@@ -201,101 +153,80 @@ func startProgress(params progressParams, width func() int,
 	return p
 }
 
-// draw shows the phase's progress: on a TTY in place of the line
+// newBar builds the library's bar for a phase with an exact total, with
-// before, and otherwise as a plain line. It first adds the count to
+// label as its description and the library's counts, rate, elapsed time
-// drawn, then drops from drawn the draws older than rateWindow, except
+// and ETA beside it. The bar is as wide as the rest of the line leaves
-// the one before this draw.
+// room for on the terminal.
-func (p *progress) draw() {
+func newBar(label string, total int64) *progressbar.ProgressBar {
-	p.mu.Lock()
+	return progressbar.NewOptions64(total,
-	defer p.mu.Unlock()
+		progressbar.OptionSetWriter(os.Stderr),
-
+		progressbar.OptionSetDescription(label),
-	now := p.now()
+		progressbar.OptionShowCount(),
-
+		progressbar.OptionShowIts(),
-	p.drawn = append(p.drawn, drawnCount{at: now, count: p.count})
+		progressbar.OptionSetItsString("files"),
-	for len(p.drawn) > 2 && now.Sub(p.drawn[0].at) > rateWindow {
+		progressbar.OptionSetElapsedTime(true),
-		p.drawn = p.drawn[1:]
+		progressbar.OptionSetPredictTime(true),
-	}
+		progressbar.OptionShowElapsedTimeOnFinish(),
-
+		progressbar.OptionFullWidth(),
-	if p.width == nil {
+	)
-		fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, false))
-
-		return
-	}
-
-	width := p.width()
-	text := p.line(now, width-spinnerMargin, true)
-
-	// Before each draw the library erases its line with as many spaces as
-	// the widest line it has drawn. A display gets a new one when the
-	// terminal has been made too narrow for that erase, which would wrap
-	// and leave a line behind at every draw.
-	if p.bar == nil || p.widest >= width {
-		p.newBar(width)
-	}
-
-	p.bar.Describe(text)
-
-	// The library drew the spinner, the text and the spaces around it,
-	// less the terminal's last column, which spinnerMargin keeps clear. A
-	// bar's cells take one column each but more than one byte.
-	p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
 }
 
-// newBar has the library make what draw draws: its spinner, followed by
+// newSpinner builds the library's spinner for a phase without an exact
-// the text draw gives it and nothing of the library's own. p.mu must be
+// total, followed by the text draw gives it.
-// held.
+func newSpinner() *progressbar.ProgressBar {
-func (p *progress) newBar(width int) {
+	return progressbar.NewOptions64(-1,
-	if p.bar != nil {
-		// Erase what was drawn before, short of the last column of a
-		// terminal that may have been made narrower since: the library's
-		// own erase is as wide as the widest line it drew.
-		blank := strings.Repeat(" ", min(p.widest, width-1))
-		fmt.Fprint(os.Stderr, "\r"+blank+"\r")
-	}
-
-	// A total of -1 gives the library's spinner.
-	p.bar = progressbar.NewOptions64(-1,
 		progressbar.OptionSetWriter(os.Stderr),
 		progressbar.OptionSpinnerType(spinnerType),
 		// The spinner turns at each draw, not on a timer of the
 		// library's own.
 		progressbar.OptionSetSpinnerChangeInterval(0),
-		// The text carries the elapsed time and the time remaining.
+		// The text carries the elapsed time.
 		progressbar.OptionSetElapsedTime(false),
-		progressbar.OptionSetPredictTime(false),
 	)
-	p.widest = 0
 }
 
-// percentBar returns the percent and the bar that end the text of a
+// draw shows the phase's progress: on a TTY in place of the line
-// phase with an exact total, as "  25% |██        |": the bar's
+// before, and otherwise as a plain line.
-// barWidth columns fill in proportion to the items done. A count past
+func (p *progress) draw() {
-// the total, as when a scan adds records while trees reads them, shows
+	p.mu.Lock()
-// as 100%. p.mu must be held.
+	defer p.mu.Unlock()
-func (p *progress) percentBar() string {
-	done := min(float64(p.count)/float64(p.total), 1)
-	filled := strings.Repeat(barCell, int(done*barWidth))
 
-	return fmt.Sprintf("%4.0f%% |%-*s|", done*percentScale, barWidth, filled)
+	switch {
+	case p.bar == nil:
+		fmt.Fprintln(os.Stderr, p.line())
+	case p.spinner:
+		text := p.line()
+		if room := terminalWidth() - spinnerMargin; len(text) > room {
+			text = text[:max(room, 0)]
+		}
+
+		p.bar.Describe(text)
+	case p.count > p.drawn:
+		// The library measures its rate over the items it is given,
+		// so it is given items only when there are new ones: an
+		// empty add would count a wait as a rate of zero.
+		_ = p.bar.Add64(p.count - p.drawn)
+		p.drawn = p.count
+	default:
+		// No new items: the bar is drawn again all the same, with
+		// its elapsed time as of now.
+		p.bar.Describe(p.label)
+	}
 }
 
 // setTotal gives a phase that started without a total its total, once
-// it is known. The rate, the time remaining and the time with no item
+// it is known. The rate, and so the ETA, count from then on: the time
-// completed count from then on: the time spent waiting for the total
+// spent waiting for the total says nothing about how fast items go.
-// says nothing about how fast items go.
 func (p *progress) setTotal(total int64) {
 	if p == nil {
 		return
 	}
 
-	now := p.now()
-
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
 	p.total = total
-	p.rateStart, p.lastItem = now, now
+	p.rateStart = time.Now()
-	p.drawn = []drawnCount{{at: now, count: p.count}}
 }
 
 // increment records one completed item, which the display shows at its
@@ -305,13 +236,10 @@ func (p *progress) increment() {
 		return
 	}
 
-	now := p.now()
-
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
 	p.count++
-	p.lastItem = now
 }
 
 // warnf prints a one-line warning to stderr without corrupting the
@@ -336,73 +264,6 @@ func (p *progress) warnf(format string, args ...any) {
 	fmt.Fprintln(os.Stderr, msg)
 }
 
-// progressWriter writes a phase's output to out, which may be the file
-// the phase's display is drawn on, as stdout is in a plain "sfdupes
-// trees" on a terminal. It holds what is written to it and writes that
-// to out in pieces of about ioBufSize bytes. When out is the file
-// stderr is, each piece is written under the display's lock with the
-// display cleared first, as warnf does for a warning: no draw lands in
-// the middle of a piece, and the next draw shows the display again, on
-// the line below. Every write to it must end with a newline, as a row
-// written with one fmt.Fprintf does, so that each piece ends a line.
-// Flush writes what it holds once the last row is written.
-type progressWriter struct {
-	// p is the display to clear, nil unless out is the file stderr is.
-	p   *progress
-	out io.Writer
-	buf []byte
-}
-
-// writer returns a progressWriter writing to out around p's display.
-// It clears the display and takes its lock only when out is the file
-// stderr is: the same device and inode, as os.SameFile compares them,
-// which is one terminal, or one pipe or file that both are sent to.
-// Otherwise it writes to out without the lock, so the display keeps
-// being drawn while a write to out waits for a slow reader.
-func (p *progress) writer(out io.Writer) *progressWriter {
-	w := &progressWriter{out: out}
-
-	if f, ok := out.(*os.File); ok {
-		outInfo, outErr := f.Stat()
-		errInfo, errErr := os.Stderr.Stat()
-
-		if outErr == nil && errErr == nil && os.SameFile(outInfo, errInfo) {
-			w.p = p
-		}
-	}
-
-	return w
-}
-
-// Write holds b, and writes what it holds to out once that comes to
-// ioBufSize bytes.
-func (w *progressWriter) Write(b []byte) (int, error) {
-	w.buf = append(w.buf, b...)
-	if len(w.buf) < ioBufSize {
-		return len(b), nil
-	}
-
-	return len(b), w.Flush()
-}
-
-// Flush writes what w holds to out. When out is the file stderr is, the
-// display is cleared first and not drawn again until it is written.
-func (w *progressWriter) Flush() error {
-	if w.p != nil {
-		w.p.mu.Lock()
-		defer w.p.mu.Unlock()
-
-		if w.p.bar != nil {
-			_ = w.p.bar.Clear()
-		}
-	}
-
-	_, err := w.out.Write(w.buf)
-	w.buf = w.buf[:0]
-
-	return err
-}
-
 // finish ends the phase's display, drawn once more as the phase ends.
 // A phase stopped short of its total, as an interrupted one is, shows
 // the count it stopped at rather than a full bar.
@@ -416,132 +277,57 @@ func (p *progress) finish() {
 
 	p.draw()
 
-	if p.width != nil {
+	if p.bar != nil {
 		// The display stays on the TTY as drawn; what follows starts
 		// on a line of its own.
 		fmt.Fprintln(os.Stderr)
 	}
 }
 
-// line formats the phase's progress as of now in at most room columns,
+// line formats the phase's progress, as
-// as
 //
-//	[3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, remain 13s
+//	[3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
 //
-// where "files" is the phase's unit. A phase with no total shows the
+// with "~" before a total that is an estimate. A phase with no total
-// items done instead, or what it waits for. The rate is that since the
+// shows the items done instead, or what it waits for. p.mu must be
-// first draw in drawn, and the time remaining comes from the rate since
+// held.
-// the count started. Once no item has been completed for stallTime, the
+func (p *progress) line() string {
-// rate, which would read 0, gives way to the word "stalled" and where
+	elapsed := time.Since(p.start).Round(time.Second)
-// the phase waits, as "stalled disk/db".
-//
-// When bar is true, as on a terminal, a phase with an exact total ends
-// the line with its percent and bar, in place of the percent after its
-// count. A total of 0 leaves the bar nothing to fill.
-//
-// The phase and its place, the progress, the elapsed time and the time
-// remaining are kept whole. A line wider than room first leaves out the
-// rate, the words of a stalled phase, or what the phase waits for. Next
-// it leaves out the bar. Only if it is still wider is it cut at room.
-// Every part but the bar is ASCII, so its length is its width in
-// columns. p.mu must be held.
-func (p *progress) line(now time.Time, room int, bar bool) string {
-	elapsed := now.Sub(p.start).Round(time.Second)
-
-	// head and tail hold the parts kept whole; extra, between them, is
-	// the part left out.
-	var head, extra, tail string
 
 	if p.total < 0 && p.waitingFor != "" {
-		head, extra = p.label+": ", p.waitingFor+", "
+		return fmt.Sprintf("%s: %s, elapsed %s, eta unknown",
-		tail = fmt.Sprintf("elapsed %s, remain unknown", elapsed)
+			p.label, p.waitingFor, elapsed)
-	} else {
-		since := p.drawn[0]
-		rate := perSecond(p.count-since.count, now.Sub(since.at))
-		average := perSecond(p.count, now.Sub(p.rateStart))
-
-		head = p.label + ": " + p.itemsDone(true) + ", "
-		extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
-		tail = fmt.Sprintf("elapsed %s, remain %s", elapsed, p.remain(average))
-
-		if now.Sub(p.lastItem) >= stallTime {
-			extra = "stalled " + p.stalledOn + ", "
-		}
 	}
 
-	if bar && p.total > 0 && !p.estimate {
+	rate := 0.0
-		// Beside the bar, which shows the percent, the count shows none.
+	if s := time.Since(p.rateStart).Seconds(); s > 0 {
-		// The line is padded to the room the bar leaves it, so the bar
+		rate = float64(p.count) / s
-		// stays at room's right edge however long the line is.
+	}
-		beside := room - barColumns
-		barHead := p.label + ": " + p.itemsDone(false) + ", "
 
-		switch {
+	done := fmt.Sprintf("%d files", p.count)
-		case len(barHead+extra+tail) <= beside:
+
-			return fmt.Sprintf("%-*s%s", beside, barHead+extra+tail,
+	if p.total >= 0 {
-				p.percentBar())
+		approx := ""
-		case len(barHead+tail) <= beside:
+		if p.estimate {
-			return fmt.Sprintf("%-*s%s", beside, barHead+tail, p.percentBar())
+			approx = "~"
 		}
 
-		// The bar gives way. The rate, or the words of a stalled phase,
+		pct := float64(percentScale)
-		// gave way before it and stay out.
+		if p.total > 0 {
-		extra = ""
+			pct = percentScale * float64(p.count) / float64(p.total)
+		}
+
+		done = fmt.Sprintf("[%d/%s%d] %.0f%%", p.count, approx, p.total, pct)
 	}
 
-	full, kept := head+extra+tail, head+tail
+	return fmt.Sprintf("%s: %s, %.0f files/s, elapsed %s, eta %s",
-
+		p.label, done, rate, elapsed, p.eta(rate))
-	switch {
-	case len(full) <= room:
-		return full
-	case len(kept) <= room:
-		return kept
-	}
-
-	return kept[:max(room, 0)]
 }
 
-// perSecond returns count items over d as items a second, or 0 when no
+// eta returns the time the phase has left at rate items a second, or
-// time has passed.
-func perSecond(count int64, d time.Duration) float64 {
-	if d <= 0 {
-		return 0
-	}
-
-	return float64(count) / d.Seconds()
-}
-
-// itemsDone formats the items done, as "[1234/5000] 25%", with "~"
-// before a total that is an estimate and without the percent when
-// percent is false, or as "1234 files", in the phase's unit, when the
-// phase has no total. p.mu must be held.
-func (p *progress) itemsDone(percent bool) string {
-	if p.total < 0 {
-		return fmt.Sprintf("%d %s", p.count, p.unit)
-	}
-
-	approx := ""
-	if p.estimate {
-		approx = "~"
-	}
-
-	done := fmt.Sprintf("[%d/%s%d]", p.count, approx, p.total)
-	if !percent {
-		return done
-	}
-
-	pct := float64(percentScale)
-	if p.total > 0 {
-		pct = percentScale * float64(p.count) / float64(p.total)
-	}
-
-	return fmt.Sprintf("%s %.0f%%", done, pct)
-}
-
-// remain returns the time the phase has left at rate items a second, or
 // "unknown": with no total, before the first item, and once the count
 // passes an estimate. p.mu must be held.
-func (p *progress) remain(rate float64) string {
+func (p *progress) eta(rate float64) string {
 	switch {
 	case p.total < 0, p.estimate && p.count > p.total:
 		return "unknown"
@@ -551,8 +337,8 @@ func (p *progress) remain(rate float64) string {
 		return "unknown"
 	}
 
-	left := time.Duration(
+	remaining := time.Duration(
 		float64(p.total-p.count) / rate * float64(time.Second))
 
-	return left.Round(time.Second).String()
+	return remaining.Round(time.Second).String()
 }

report.go

@@ -1,13 +1,11 @@
 package main
 
 import (
+	"bufio"
 	"context"
-	"crypto/sha256"
-	"database/sql"
 	"fmt"
 	"io"
 	"os"
-	"slices"
 	"strings"
 	"time"
 )
@@ -21,37 +19,23 @@ const minGroupSize = 2
 
 // scanRec is one file record from the database. The signature (size,
 // head, tail, content) is the duplicate key; mtime is informational
-// only and used by scan for change detection. diskSize is the bytes the
+// only and used by scan for change detection.
-// file takes on disk, which trees adds up.
 type scanRec struct {
-	size     int64
+	size    int64
-	diskSize int64
+	mtime   time.Time
-	mtime    time.Time
+	head    string
-	head     [sha256.Size]byte
+	tail    string
-	tail     [sha256.Size]byte
+	content string
-	content  [sha256.Size]byte
+	path    string
-	path     string
-}
-
-// reportPhase names one phase of report with its place among all of
-// them, as "[2/2] read", for the phase's progress display. The list
-// here is the one place the phases are counted.
-func reportPhase(name string) string {
-	return phaseLabel([]string{openPhase, "read"}, name)
 }
 
 // runReport implements the report subcommand: it prints the file-level
-// duplicates report as TSV on stdout. It never touches the scanned
+// duplicates report as TSV on stdout. SQLite groups and orders the
-// filesystem; its only I/O is the database, stdout, and stderr. It
+// records, and each row is written as it is read, so no group is held
-// names the database on stderr before it opens it. Any database
+// in memory. It never touches the scanned filesystem; its only I/O is
+// the database (with SQLite's temporary sort file), stdout, and stderr.
+// It names the database on stderr before it opens it. Any database
 // problem, including a missing database, is fatal.
-//
-// The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and read (read the number of
-// records from the totals table, then read the records in signature
-// order, writing each size's duplicate groups to stdout once that size
-// is read; see readReport). Each display ends before the next phase
-// starts, and before an error is returned to be printed.
 func runReport(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -61,190 +45,65 @@ func runReport(ctx context.Context, stdout io.Writer) error {
 	fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
 		escapePath(dbPath), source)
 
-	// Opening can wait on a busy database for as long as the busy
-	// timeout.
-	prog := newProgress(progressParams{
-		label: reportPhase(openPhase), total: -1,
-		waitingFor: openingDatabase,
-	})
 	db, err := openReportDatabase(ctx, dbPath)
-
-	prog.finish()
-
 	if err != nil {
 		return err
 	}
 
 	defer func() { _ = db.Close() }()
 
-	// The read phase writes the report to stdout while it reads the
+	out := bufio.NewWriterSize(stdout, ioBufSize)
-	// records, so it may wait on either.
-	prog = newProgress(progressParams{
-		label: reportPhase("read"), unit: "records", total: -1,
-		waitingFor: countingRecords, stalledOn: "db/out",
-	})
-	report, err := readReport(ctx, db, dbPath, stdout, prog)
 
-	prog.finish()
+	_, err = fmt.Fprintln(out, "first\tdupe\tsize")
+	if err != nil {
+		return fmt.Errorf("write stdout: %w", err)
+	}
+
+	var (
+		groups, dupeFiles int
+		reclaimable       int64
+		writeErr          error
+	)
+
+	records, err := loadDupeRows(ctx, db,
+		func(first, path string, size int64) error {
+			// A group's first path is its first row; every other
+			// path is a dupe.
+			if path == first {
+				groups++
+
+				return nil
+			}
+
+			_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
+				escapePath(first), escapePath(path), size)
+			dupeFiles++
+			reclaimable += size
+
+			return writeErr
+		})
+
+	if writeErr != nil {
+		return fmt.Errorf("write stdout: %w", writeErr)
+	}
 
 	if err != nil {
-		return err
+		return fmt.Errorf("database %s: %w", dbPath, err)
+	}
+
+	err = out.Flush()
+	if err != nil {
+		return fmt.Errorf("write stdout: %w", err)
 	}
 
 	fmt.Fprintf(os.Stderr,
 		"report: %d records read, %d duplicate groups, %d dupe files, "+
 			"%s reclaimable\n",
-		report.records, report.dupeGroups, report.dupeFiles,
+		records, groups, dupeFiles, humanBytes(reclaimable))
-		humanBytes(report.reclaimable))
 
 	return nil
 }
 
-// readReport gives prog the number of records in db, the database at
-// dbPath, as its total, then reads every record in signature order,
-// counting each on prog, and writes the report to stdout: the header
-// line, then the rows of each size's duplicate groups once that size is
-// read. stdout may be the terminal prog is drawn on, so the lines go
-// through prog's writer. It returns what it wrote, with the totals for
-// the summary line.
-func readReport(ctx context.Context, db *sql.DB, dbPath string,
-	stdout io.Writer, prog *progress,
-) (*reportWriter, error) {
-	out := prog.writer(stdout)
-	report := &reportWriter{out: out}
-
-	_, err := fmt.Fprintln(out, "first\tdupe\tsize")
-	if err != nil {
-		return nil, fmt.Errorf("write stdout: %w", err)
-	}
-
-	var writeErr error
-
-	err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
-		prog.increment()
-
-		writeErr = report.add(r)
-
-		return writeErr
-	})
-
-	if writeErr != nil {
-		return nil, fmt.Errorf("write stdout: %w", writeErr)
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("database %s: %w", dbPath, err)
-	}
-
-	err = report.finish()
-	if err == nil {
-		err = out.Flush()
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("write stdout: %w", err)
-	}
-
-	return report, nil
-}
-
-// reportWriter writes the report's rows to out from the records, added
-// in signature order, largest size first, as loadSignatureRows reads
-// them. The records sharing a signature come together, and so do the
-// records of one size, so it holds only the duplicate groups of the
-// size being read, and writes them, ordered by first path, once a
-// record of a smaller size comes. It counts what the summary line
-// reports.
-type reportWriter struct {
-	out io.Writer
-
-	sig    scanRec    // the signature being read; its path is not used
-	paths  []string   // the paths read so far with sig
-	groups [][]string // the duplicate groups of sig.size, each sorted
-
-	records, dupeGroups, dupeFiles int
-	reclaimable                    int64
-}
-
-// add takes the next record. One with the signature being read joins
-// its paths; any other ends them, and one of a smaller size first
-// writes the groups of the size before.
-func (w *reportWriter) add(r scanRec) error {
-	w.records++
-
-	if r.size == w.sig.size && r.head == w.sig.head &&
-		r.tail == w.sig.tail && r.content == w.sig.content {
-		w.paths = append(w.paths, r.path)
-
-		return nil
-	}
-
-	w.endGroup()
-
-	if r.size != w.sig.size {
-		err := w.writeGroups()
-		if err != nil {
-			return err
-		}
-	}
-
-	w.sig = r
-	w.paths = append(w.paths, r.path)
-
-	return nil
-}
-
-// endGroup ends the paths read with w.sig: two or more are a duplicate
-// group, held with its paths sorted until writeGroups writes it.
-func (w *reportWriter) endGroup() {
-	if len(w.paths) < minGroupSize {
-		// A path with no duplicate is dropped, and its slice reused.
-		w.paths = w.paths[:0]
-
-		return
-	}
-
-	slices.Sort(w.paths)
-	w.groups = append(w.groups, w.paths)
-	w.paths = nil
-}
-
-// writeGroups writes the duplicate groups held, all of size w.sig.size,
-// ordered by first path: for each group, a row for every path but the
-// first. No two groups share a first path, since no two records share
-// a path.
-func (w *reportWriter) writeGroups() error {
-	slices.SortFunc(w.groups, func(a, b []string) int {
-		return strings.Compare(a[0], b[0])
-	})
-
-	for _, g := range w.groups {
-		for _, path := range g[1:] {
-			_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
-				escapePath(g[0]), escapePath(path), w.sig.size)
-			if err != nil {
-				return err
-			}
-
-			w.dupeFiles++
-			w.reclaimable += w.sig.size
-		}
-
-		w.dupeGroups++
-	}
-
-	w.groups = nil
-
-	return nil
-}
-
-// finish writes the groups of the last size read.
-func (w *reportWriter) finish() error {
-	w.endGroup()
-
-	return w.writeGroups()
-}
-
 // escapePath returns a path as it is written in a report column (README
 // "Report output format"): a backslash, tab, newline or carriage return
 // becomes \\, \t, \n or \r, and every other byte is kept as it is.

report_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -10,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -19,32 +17,18 @@ import (
 // awkwardDir is a directory name holding every byte the reports escape.
 const awkwardDir = "/d/\tone\ntwo\rthree\\four"
 
-// testHash returns a stand-in hash for a test record: the bytes of s,
-// then zeros. Stand-ins sort as their strings do, so a test can choose
-// the order in which report reads its records.
-func testHash(s string) [sha256.Size]byte {
-	var h [sha256.Size]byte
-
-	copy(h[:], s)
-
-	return h
-}
-
 // awkwardPairRecs is a duplicate pair in sibling directories /d/A and
 // awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
 // sorts after it, so awkwardDir coming first shows that sorting uses
 // the raw path.
 func awkwardPairRecs() []scanRec {
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
-
 	return []scanRec{
-		{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"},
+		{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
-		{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
+		{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
 	}
 }
 
-// seedDatabase writes recs into a fresh database, closes it as scan
+// seedDatabase writes recs into a fresh database and returns its path.
-// does, out of WAL mode, and returns its path.
 func seedDatabase(t *testing.T, recs []scanRec) string {
 	t.Helper()
 
@@ -60,7 +44,10 @@ func seedDatabase(t *testing.T, recs []scanRec) string {
 		t.Fatal(err)
 	}
 
-	closeScanDatabase(t.Context(), db, path)
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	return path
 }
@@ -73,39 +60,25 @@ type dupeGroup struct {
 }
 
 // dupeGroups returns the duplicate groups report reads from db, in
-// report order, from the rows it writes: a group's rows come together,
+// report order.
-// each naming the group's first path.
 func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup {
 	t.Helper()
 
-	var out bytes.Buffer
-
-	_, err := readReport(t.Context(), db, "", &out, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	var groups []dupeGroup
 
-	rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
+	_, err := loadDupeRows(t.Context(), db,
-	for _, row := range rows[1:] { // after the header
+		func(first, path string, size int64) error {
-		fields := strings.Split(row, "\t")
+			if path == first {
-		if len(fields) != 3 {
+				groups = append(groups, dupeGroup{size: size})
-			t.Fatalf("row %q, want first, dupe and size", row)
+			}
-		}
 
-		size, err := strconv.ParseInt(fields[2], 10, 64)
+			g := &groups[len(groups)-1]
-		if err != nil {
+			g.paths = append(g.paths, path)
-			t.Fatal(err)
-		}
 
-		if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] {
+			return nil
-			groups = append(groups,
+		})
-				dupeGroup{size: size, paths: []string{fields[0]}})
+	if err != nil {
-		}
+		t.Fatal(err)
-
-		g := &groups[len(groups)-1]
-		g.paths = append(g.paths, fields[1])
 	}
 
 	return groups
@@ -145,160 +118,38 @@ func TestRunReportEscapesPaths(t *testing.T) {
 }
 
 func TestReportStdoutFailsWhileReading(t *testing.T) {
-	t.Parallel()
+	// Each row holds two paths longer than dir, so the report is more
-
+	// than twice the stdout buffer and stdout fails while rows are
-	// Every two records are a duplicate pair of a size of their own, and
+	// still being read, not at the final flush.
-	// a pair's row is written once a smaller size is read. Each row
-	// holds two paths longer than dir, so the rows come to more than
-	// twice the stdout buffer, and stdout fails while records are still
-	// being read, not once every record is.
 	dir := "/" + strings.Repeat("d", 4096)
 
-	recs := make([]scanRec, 2*ioBufSize/len(dir))
+	recs := make([]scanRec, ioBufSize/len(dir))
 	for i := range recs {
 		recs[i] = scanRec{
-			size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
+			size: 1, head: "h", tail: "t", content: "c",
-			content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
+			path: fmt.Sprintf("%s/%d", dir, i),
 		}
 	}
 
-	db := openTestDB(t)
+	t.Setenv(databaseEnv, seedDatabase(t, recs))
 
-	err := applyChanges(t.Context(), db, recs, nil, nil)
+	err := runReport(t.Context(), failingWriter{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Counts the records read, and draws nothing.
-	prog := &progress{now: time.Now}
-
-	_, err = readReport(t.Context(), db, "", failingWriter{}, prog)
 	if !errors.Is(err, errWriteFailed) ||
 		!strings.HasPrefix(err.Error(), "write stdout: ") {
 		t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed)
 	}
-
-	if prog.count >= int64(len(recs)) {
-		t.Errorf("stdout failed after all %d records were read, want it "+
-			"to stop the read", prog.count)
-	}
-}
-
-// reportOpenLine is the plain line of report's open phase when it opens
-// the database at once; it prints it as the phase starts and as it ends.
-const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
-	"remain unknown"
-
-// smokeReport is report's stdout for the smoke-test records.
-const smokeReport = "first\tdupe\tsize\n" +
-	"/d/t1/f1\t/d/t2/f1\t3000\n" +
-	"/d/t1/f1\t/d/t3/f1\t3000\n" +
-	"/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" +
-	"/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n"
-
-// TestRunReportProgressNamesEachPhase runs report on the smoke-test
-// records with stderr not a terminal, and checks every line it prints
-// there. After the line naming the database, each phase prints a line
-// naming it with its place among the two as it starts and another as
-// it ends; read counts the 6 records before it reads them. The summary
-// comes last, and stdout holds the report alone.
-func TestRunReportProgressNamesEachPhase(t *testing.T) {
-	path := seedDatabase(t, smokeTreeRecs())
-	t.Setenv(databaseEnv, path)
-
-	stderr := captureStderr(t)
-
-	var stdout bytes.Buffer
-
-	code := run([]string{cmdReport}, &stdout, os.Stderr)
-	if code != exitOK {
-		t.Fatalf("run(report) = %d, want %d; stderr: %s",
-			code, exitOK, stderr())
-	}
-
-	if got := stdout.String(); got != smokeReport {
-		t.Errorf("stdout = %q, want %q", got, smokeReport)
-	}
-
-	// How each line starts; the rate at the end of a phase varies.
-	want := []string{
-		"report: database " + path + " (from " + databaseEnv + ")",
-		reportOpenLine,
-		reportOpenLine,
-		"[2/2] read: counting the records, elapsed 0s, remain unknown",
-		"[2/2] read: [6/6] 100%, ",
-		"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
-			"6.1 KiB reclaimable",
-	}
-
-	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
-	if len(lines) != len(want) {
-		t.Fatalf("stderr lines = %q, want %d lines starting %q",
-			lines, len(want), want)
-	}
-
-	for i := range want {
-		if !strings.HasPrefix(lines[i], want[i]) {
-			t.Errorf("stderr line %d = %q, want it to start %q",
-				i+1, lines[i], want[i])
-		}
-	}
-}
-
-// TestReadReportOnTerminal writes the smoke-test report to the terminal
-// its read phase's display is drawn on, as a plain "sfdupes report"
-// does: the terminal must show the header and every row whole, each on
-// a line of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestReadReportOnTerminal(t *testing.T) {
-	stderr := captureStderr(t)
-	db := openTestDB(t)
-
-	err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// No timed draws: the display is drawn as it starts and as it ends.
-	p := startProgress(progressParams{
-		label: reportPhase("read"), unit: "records", total: -1,
-		waitingFor: countingRecords,
-	}, terminalWidth, time.Hour, time.Now)
-
-	_, err = readReport(t.Context(), db, "", os.Stderr, p)
-
-	p.finish()
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// A terminal shows each line as the text after its last carriage
-	// return.
-	var shown []string
-
-	for line := range strings.SplitSeq(stderr(), "\n") {
-		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
-	}
-
-	want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n")
-	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
-		t.Errorf("terminal shows lines %q, want %q first", shown, want)
-	}
 }
 
 func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
 	// README §Constraints: identical database contents give identical
 	// output, whatever order the records were inserted in.
-	a, b := testHash("a"), testHash("b")
-
 	recs := append(smokeTreeRecs(), awkwardPairRecs()...)
 	recs = append(recs,
-		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
+		scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
-		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
+		scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
-		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
+		scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
-		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
+		scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
+		scanRec{size: 50, path: "/x/unhashed"},
 	)
 
 	reversed := slices.Clone(recs)
@@ -401,19 +252,16 @@ func TestWarnfEscapes(t *testing.T) {
 func TestDupeGroups(t *testing.T) {
 	t.Parallel()
 
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
-	bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
-	u := testHash("u")
 	recs := []scanRec{
-		{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
-		{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
-		{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
-		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
+		{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
-		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
+		{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
 		// Same size as the /z group but a different head hash.
-		{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
+		{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
 		// A singleton signature must not form a group.
-		{size: 7, head: u, tail: u, content: u, path: "/lonely"},
+		{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -437,13 +285,16 @@ func TestDupeGroups(t *testing.T) {
 func TestDupeGroupsContentSeparates(t *testing.T) {
 	t.Parallel()
 
-	// Same size, head, and tail: /b differs from the others only in its
+	// Same size, head, and tail, but different content hashes: the final
-	// content hash, so it joins no group, while /a and /c group.
+	// rung keeps them apart, so no group forms. Matching content groups.
-	h, tl := testHash("h"), testHash("t")
+	// Records without a content hash never group, not even with each
+	// other.
 	recs := []scanRec{
-		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
+		{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
-		{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
+		{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
-		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
+		{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
+		{size: 100, head: "h", tail: "t", path: "/d"},
+		{size: 100, head: "h", tail: "t", path: "/e"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -462,15 +313,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
 
 	// mtime is informational only; records differing only in mtime
 	// still group together.
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 	recs := []scanRec{
 		{
-			size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
+			size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
-			content: c, path: "/m/1",
+			content: "c", path: "/m/1",
 		},
 		{
-			size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
+			size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
-			content: c, path: "/m/2",
+			content: "c", path: "/m/2",
 		},
 	}
 
@@ -485,12 +335,11 @@ func TestDupeGroupsTieBreak(t *testing.T) {
 
 	// The hashes sort opposite to the first paths, so ordering the
 	// groups by hash instead of by first path fails this test.
-	a, b := testHash("a"), testHash("b")
 	recs := []scanRec{
-		{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
-		{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
-		{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -505,47 +354,14 @@ func TestDupeGroupsTieBreak(t *testing.T) {
 	}
 }
 
-func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
-	t.Parallel()
-
-	// report reads the groups of size 50 in signature order, here /a,
-	// /z, /k, which is neither the order of their first paths nor its
-	// reverse, and must write them ordered by first path. The group of
-	// size 7 comes after them all, though its first path sorts first.
-	a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
-	recs := []scanRec{
-		{size: 50, head: a, tail: a, content: a, path: "/k/2"},
-		{size: 50, head: a, tail: a, content: a, path: "/k/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/z/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/z/2"},
-		{size: 50, head: c, tail: c, content: c, path: "/a/2"},
-		{size: 50, head: c, tail: c, content: c, path: "/a/1"},
-		{size: 7, head: d, tail: d, content: d, path: "/0/2"},
-		{size: 7, head: d, tail: d, content: d, path: "/0/1"},
-	}
-
-	groups := dupeGroupsOf(t, recs)
-
-	got := make([]string, 0, len(groups))
-	for _, g := range groups {
-		got = append(got, g.paths[0])
-	}
-
-	want := []string{"/a/1", "/k/1", "/z/1", "/0/1"}
-	if !slices.Equal(got, want) {
-		t.Errorf("groups' first paths = %q, want %q", got, want)
-	}
-}
-
 func TestDupeGroupsDeterministic(t *testing.T) {
 	t.Parallel()
 
-	a, b := testHash("a"), testHash("b")
 	recs := []scanRec{
-		{size: 1, head: a, tail: a, content: a, path: "/p/1"},
+		{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
-		{size: 1, head: a, tail: a, content: a, path: "/p/2"},
+		{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
-		{size: 2, head: b, tail: b, content: b, path: "/q/1"},
+		{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
-		{size: 2, head: b, tail: b, content: b, path: "/q/2"},
+		{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
 	}
 
 	forward := dupeGroupsOf(t, recs)

script/cibuild

@@ -2,12 +2,11 @@
 # script/cibuild: run the CI build. It bootstraps first: a CI runner
 # checks out and runs this and nothing else, and script/fmt-check runs
 # the formatter on the host, which a pristine checkout cannot do.
-# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
+# A new CACHEBUST value reruns every RUN step, since a cached gate phase
-# line, since a cached gate phase is a check that did not run; unlike
+# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
-# --no-cache, it keeps the layers above that line and the cache mounts.
 # This relies on every stage of the Dockerfile declaring ARG CACHEBUST
-# before every step that lints, tests or compiles; a stage without that
+# first; a stage without that line is served from cache on an unchanged
-# line is served from cache on an unchanged tree, with no warning.
+# tree, with no warning.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"

script/docker

@@ -1,12 +1,11 @@
 #!/bin/sh
 # script/docker: build the Docker image tagged with the project name.
 # Identical in all repos; the tag comes from script/projectname.
-# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST
+# A new CACHEBUST value reruns every RUN step, since a cached gate phase
-# line, since a cached gate phase is a check that did not run; unlike
+# is a check that did not run; unlike --no-cache, it keeps Go's build cache.
-# --no-cache, it keeps the layers above that line and the cache mounts.
 # This relies on every stage of the Dockerfile declaring ARG CACHEBUST
-# before every step that lints, tests or compiles; a stage without that
+# first; a stage without that line is served from cache on an unchanged
-# line is served from cache on an unchanged tree, with no warning.
+# tree, with no warning.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"

script/lint

@@ -5,13 +5,11 @@
 # make its answer untrustworthy.
 #
 # The phase is not the last stage in the file, so it is built only when
-# --target names it. A new CACHEBUST value reruns every RUN step after
+# --target names it. A new CACHEBUST value reruns every RUN step, since
-# the ARG CACHEBUST line, since a cached lint layer is a lint that did
+# a cached lint layer is a lint that did not run; unlike --no-cache, it
-# not run; unlike --no-cache, it keeps the layers above that line and
+# keeps Go's build cache. This relies on every stage of the Dockerfile
-# the cache mounts. This relies on every stage of the Dockerfile
+# declaring ARG CACHEBUST first; a stage without that line is served
-# declaring ARG CACHEBUST before every step that lints, tests or
+# from cache on an unchanged tree, with no warning.
-# compiles; a stage without that line is served from cache on an
-# unchanged tree, with no warning.
 # --output type=cacheonly writes no image, since nothing uses one.
 set -eu
 

script/test

@@ -3,12 +3,11 @@
 # and this builds that phase alone, on the same terms as script/lint:
 # --target because a phase that is not the last stage is built only when
 # named, and a new CACHEBUST value because a cached test layer is a test
-# that did not run; unlike --no-cache, it keeps the layers above the ARG
+# that did not run; unlike --no-cache, it keeps Go's build cache. This
-# CACHEBUST line and the cache mounts. This relies on every stage of the
+# relies on every stage of the Dockerfile declaring ARG CACHEBUST first;
-# Dockerfile declaring ARG CACHEBUST before every step that lints, tests
+# a stage without that line is served from cache on an unchanged tree,
-# or compiles; a stage without that line is served from cache on an
+# with no warning. --output type=cacheonly writes no image, since
-# unchanged tree, with no warning. --output type=cacheonly writes no
+# nothing uses one.
-# image, since nothing uses one.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"

trees.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"crypto/sha256"
-	"database/sql"
 	"fmt"
 	"io"
 	"os"
@@ -22,17 +22,6 @@ type treeNode struct {
 	digest    [sha256.Size]byte
 	fileCount int64
 	totalSize int64
-	// diskSize is the bytes the directory's files take on disk. Two
-	// copies of one tree have the same totalSize, but their diskSize
-	// can differ.
-	diskSize int64
-}
-
-// treesPhase names one phase of trees with its place among all of them,
-// as "[2/4] read", for the phase's progress display. The list here is
-// the one place the phases are counted.
-func treesPhase(name string) string {
-	return phaseLabel([]string{openPhase, "read", "group", "write"}, name)
 }
 
 // runTrees implements the trees subcommand: it reads every record from
@@ -42,13 +31,6 @@ func treesPhase(name string) string {
 // touches the scanned filesystem; its only I/O is the database, stdout,
 // and stderr. It names the database on stderr before it opens it. Any
 // database problem, including a missing database, is fatal.
-//
-// The work runs in four phases, each under a progress display of its
-// own on stderr: open (open the database), read (read the number of
-// records from the totals table, then read the records into the
-// hierarchy), group (group the directories by digest) and write (write
-// the groups to stdout). Each display ends before the next phase
-// starts, and before an error is returned to be printed.
 func runTrees(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -58,103 +40,34 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 	fmt.Fprintf(os.Stderr, "trees: database %s (%s)\n",
 		escapePath(dbPath), source)
 
-	// Opening can wait on a busy database for as long as the busy
-	// timeout.
-	prog := newProgress(progressParams{
-		label: treesPhase(openPhase), total: -1,
-		waitingFor: openingDatabase,
-	})
 	db, err := openReportDatabase(ctx, dbPath)
-
-	prog.finish()
-
 	if err != nil {
 		return err
 	}
 
 	defer func() { _ = db.Close() }()
 
-	prog = newProgress(progressParams{
+	records := 0
-		label: treesPhase("read"), unit: "records", total: -1,
+	tree := newTreeBuilder()
-		waitingFor: countingRecords, stalledOn: "db",
+
+	err = loadFileRows(ctx, db, func(r scanRec) {
+		records++
+
+		tree.add(r)
 	})
-	records, super, allDirs, err := readTree(ctx, db, prog)
-
-	prog.finish()
-
 	if err != nil {
 		return fmt.Errorf("database %s: %w", dbPath, err)
 	}
 
-	// Grouping works in memory, so it waits on nothing but the
-	// processor.
-	prog = newProgress(progressParams{
-		label: treesPhase("group"), unit: "directories",
-		total: int64(len(allDirs)), stalledOn: "cpu",
-	})
-	dupes := collectTreeGroups(allDirs, super, prog)
-
-	prog.finish()
-
-	prog = newProgress(progressParams{
-		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
-		stalledOn: "out",
-	})
-	dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
-
-	prog.finish()
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Fprintf(os.Stderr,
-		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
-			"%s reclaimable\n",
-		records, len(dupes), dupeTrees, humanBytes(reclaimable))
-
-	return nil
-}
-
-// readTree gives prog the number of records in db as its total, then
-// reads every record in path order into the directory hierarchy,
-// counting each on prog. It returns the number of records read, the
-// super-root and every directory.
-func readTree(ctx context.Context, db *sql.DB,
-	prog *progress,
-) (int, *treeNode, []*treeNode, error) {
-	records := 0
-	tree := newTreeBuilder()
-
-	err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
-		records++
-
-		prog.increment()
-		tree.add(r)
-	})
-	if err != nil {
-		return 0, nil, nil, err
-	}
-
 	super, allDirs := tree.finish()
 
-	return records, super, allDirs, nil
+	dupes := collectTreeGroups(allDirs, super)
-}
 
-// writeTreeGroups writes dupes to stdout as TSV, a header line and then
+	out := bufio.NewWriterSize(stdout, ioBufSize)
-// a row for every member of each group but its first, counting each
-// group on prog once its rows are written. The disk_size column is the
-// dupe's own. stdout may be the terminal prog is drawn on, so the lines
-// go through prog's writer. It returns the number of rows (dupe trees)
-// and the bytes they could reclaim.
-func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
-	prog *progress,
-) (int, int64, error) {
-	out := prog.writer(stdout)
 
-	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
+	_, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
 	if err != nil {
-		return 0, 0, fmt.Errorf("write stdout: %w", err)
+		return fmt.Errorf("write stdout: %w", err)
 	}
 
 	dupeTrees := 0
@@ -164,26 +77,29 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 	for _, g := range dupes {
 		first := g[0]
 		for _, n := range g[1:] {
-			_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n",
+			_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
 				escapePath(first.path), escapePath(n.path),
-				first.fileCount, first.totalSize, n.diskSize)
+				first.fileCount, first.totalSize)
 			if err != nil {
-				return 0, 0, fmt.Errorf("write stdout: %w", err)
+				return fmt.Errorf("write stdout: %w", err)
 			}
 
 			dupeTrees++
 			reclaimable += first.totalSize
 		}
-
-		prog.increment()
 	}
 
 	err = out.Flush()
 	if err != nil {
-		return 0, 0, fmt.Errorf("write stdout: %w", err)
+		return fmt.Errorf("write stdout: %w", err)
 	}
 
-	return dupeTrees, reclaimable, nil
+	fmt.Fprintf(os.Stderr,
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
+			"%s reclaimable\n",
+		records, len(dupes), dupeTrees, humanBytes(reclaimable))
+
+	return nil
 }
 
 // treeBuilder reconstructs the directory hierarchy from records added
@@ -237,7 +153,6 @@ func (b *treeBuilder) add(r scanRec) {
 	dir.entries = append(dir.entries, fileEntry(name, r))
 	dir.fileCount++
 	dir.totalSize += r.size
-	dir.diskSize += r.diskSize
 }
 
 // openDir starts the directory called name inside the innermost open
@@ -276,7 +191,6 @@ func (b *treeBuilder) closeTo(n int) {
 			"d\x00"+name+"\x00"+string(dir.digest[:]))
 		dir.parent.fileCount += dir.fileCount
 		dir.parent.totalSize += dir.totalSize
-		dir.parent.diskSize += dir.diskSize
 		b.dirs = append(b.dirs, dir)
 	}
 }
@@ -293,23 +207,28 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
 // name and its signature (size, head, tail, content); mtime is
 // excluded.
 func fileEntry(name string, r scanRec) string {
+	content := r.content
+
+	// A record without a content hash has unknown content (README
+	// "Database"): give it a signature no other file can share, so
+	// trees containing it never compare equal. Real hashes are hex, so
+	// the NUL-prefixed form cannot collide.
+	if content == "" {
+		content = "unhashed\x00" + r.path
+	}
+
 	return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
-		"\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) +
+		"\x00" + r.head + "\x00" + r.tail + "\x00" + content
-		"\x00" + string(r.content[:])
 }
 
-// collectTreeGroups groups directories by digest, counting each one on
+// collectTreeGroups groups directories by digest and returns every
-// prog as it is grouped, and returns every maximal group with two or
+// maximal group with two or more members, each group's members sorted
-// more members, each group's members sorted by path, groups ordered by
+// by path, groups ordered by tree size descending then by first path
-// tree size descending then by first path ascending.
+// ascending.
-func collectTreeGroups(allDirs []*treeNode, super *treeNode,
+func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
-	prog *progress,
-) [][]*treeNode {
 	groups := make(map[[sha256.Size]byte][]*treeNode)
 	for _, d := range allDirs {
 		groups[d.digest] = append(groups[d.digest], d)
-
-		prog.increment()
 	}
 
 	var dupes [][]*treeNode
@@ -345,8 +264,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 // digest is the SHA-256 of its child entries — files serialized with
 // name and signature, subdirectories with name and recursive digest —
 // sorted byte-lexicographically. Filenames cannot contain NUL or "/",
-// and every hash and digest is 32 bytes, so NUL delimiters are
+// so NUL delimiters are unambiguous.
-// unambiguous.
 func (n *treeNode) computeDigest() {
 	slices.Sort(n.entries)
 

trees_test.go

@@ -3,38 +3,31 @@ package main
 import (
 	"bytes"
 	"database/sql"
-	"os"
 	"slices"
-	"strings"
 	"testing"
-	"time"
 )
 
-// treesOpenLine is the plain line of trees' open phase when it opens
+// Signature hashes shared by the smoke-test records.
-// the database at once; it prints it as the phase starts and as it ends.
+const (
-const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
+	f1Head    = "f1h"
-	"remain unknown"
+	f1Tail    = "f1t"
+	f1Content = "f1c"
+	f2Head    = "f2h"
+	f2Tail    = "f2t"
+	f2Content = "f2c"
+)
 
 // smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
 // /d/t2 are identical, /d/t3 differs from them only by one filename.
-// Each file takes one 4 KiB block on disk.
 func smokeTreeRecs() []scanRec {
-	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
-	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
-
 	return []scanRec{
-		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
+		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
-			content: f1Content, path: "/d/t1/f1"},
+		{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
-		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
+		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
-			content: f2Content, path: "/d/t1/sub/f2"},
+		{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
-		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
+		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
-			content: f1Content, path: "/d/t2/f1"},
+		{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
-		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
+			path: "/d/t3/sub/f2renamed"},
-			content: f2Content, path: "/d/t2/sub/f2"},
-		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
-			content: f1Content, path: "/d/t3/f1"},
-		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
-			content: f2Content, path: "/d/t3/sub/f2renamed"},
 	}
 }
 
@@ -45,7 +38,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
 
 	tree := newTreeBuilder()
 
-	err := loadFileRows(t.Context(), db, func(int64) {}, tree.add)
+	err := loadFileRows(t.Context(), db, tree.add)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -152,14 +145,13 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
 
 	for _, top := range []string{"/a", "/c"} {
 		for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
-			content := testHash("c")
+			content := "c"
 			if p == "/b-x/f" {
-				content = testHash("other")
+				content = "other"
 			}
 
 			recs = append(recs, scanRec{
-				size: 1, head: testHash("h"), tail: testHash("t"),
+				size: 1, head: "h", tail: "t", content: content, path: top + p,
-				content: content, path: top + p,
 			})
 		}
 	}
@@ -178,7 +170,7 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
 		t.Fatalf("directory paths = %q, want %q", got, want)
 	}
 
-	groups := collectTreeGroups(dirs, super, nil)
+	groups := collectTreeGroups(dirs, super)
 
 	gotGroups := groupPaths(groups)
 	wantGroups := [][]string{{"/a", "/c"}}
@@ -204,128 +196,13 @@ func TestRunTreesEscapesPaths(t *testing.T) {
 			code, exitOK, stderr.String())
 	}
 
-	want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
+	want := "first\tdupe\tfiles\tsize\n" +
-		`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\t4096\n"
+		`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n"
 	if got := stdout.String(); got != want {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
 }
 
-// TestRunTreesDiskSize runs trees on two copies of a tree whose files
-// take different space on disk. The row's disk_size is the sum over the
-// dupe copy's files, those in its subdirectory included.
-func TestRunTreesDiskSize(t *testing.T) {
-	f, g := testHash("f"), testHash("g")
-	recs := []scanRec{
-		{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
-		{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
-		{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
-		{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
-	}
-
-	t.Setenv(databaseEnv, seedDatabase(t, recs))
-
-	want := "first\tdupe\tfiles\tsize\tdisk_size\n/a\t/b\t2\t30\t1536\n"
-	if got := runStdout(t, cmdTrees); got != want {
-		t.Errorf("stdout = %q, want %q", got, want)
-	}
-}
-
-// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
-// records with stderr not a terminal, and checks every line it prints
-// there. After the line naming the database, each phase prints a line
-// naming it with its place among the four as it starts and another as
-// it ends. read counts the 6 records before it reads them, group counts
-// the 8 directories, and write the 1 group. The summary comes last, and
-// stdout holds the report alone.
-func TestRunTreesProgressNamesEachPhase(t *testing.T) {
-	path := seedDatabase(t, smokeTreeRecs())
-	t.Setenv(databaseEnv, path)
-
-	stderr := captureStderr(t)
-
-	var stdout bytes.Buffer
-
-	code := run([]string{cmdTrees}, &stdout, os.Stderr)
-	if code != exitOK {
-		t.Fatalf("run(trees) = %d, want %d; stderr: %s",
-			code, exitOK, stderr())
-	}
-
-	report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
-		"/d/t1\t/d/t2\t2\t3100\t8192\n"
-	if got := stdout.String(); got != report {
-		t.Errorf("stdout = %q, want %q", got, report)
-	}
-
-	// How each line starts; the rate at the end of a phase varies.
-	want := []string{
-		"trees: database " + path + " (from " + databaseEnv + ")",
-		treesOpenLine,
-		treesOpenLine,
-		"[2/4] read: counting the records, elapsed 0s, remain unknown",
-		"[2/4] read: [6/6] 100%, ",
-		"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
-		"[3/4] group: [8/8] 100%, ",
-		"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
-		"[4/4] write: [1/1] 100%, ",
-		"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
-	}
-
-	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
-	if len(lines) != len(want) {
-		t.Fatalf("stderr lines = %q, want %d lines starting %q",
-			lines, len(want), want)
-	}
-
-	for i := range want {
-		if !strings.HasPrefix(lines[i], want[i]) {
-			t.Errorf("stderr line %d = %q, want it to start %q",
-				i+1, lines[i], want[i])
-		}
-	}
-}
-
-// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
-// terminal its write phase's display is drawn on, as a plain "sfdupes
-// trees" does: the terminal must show the header and the row whole,
-// each on a line of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestWriteTreeGroupsOnTerminal(t *testing.T) {
-	stderr := captureStderr(t)
-	super, dirs := treeOf(t, smokeTreeRecs())
-	dupes := collectTreeGroups(dirs, super, nil)
-
-	// No timed draws: the display is drawn as it starts and as it ends.
-	p := startProgress(progressParams{
-		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
-	}, terminalWidth, time.Hour, time.Now)
-
-	_, _, err := writeTreeGroups(os.Stderr, dupes, p)
-
-	p.finish()
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// A terminal shows each line as the text after its last carriage
-	// return.
-	var shown []string
-
-	for line := range strings.SplitSeq(stderr(), "\n") {
-		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
-	}
-
-	want := []string{
-		"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
-	}
-	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
-		t.Errorf("terminal shows lines %q, want %q first", shown, want)
-	}
-}
-
 func TestTreeDigests(t *testing.T) {
 	t.Parallel()
 
@@ -355,10 +232,11 @@ func TestTreeDigests(t *testing.T) {
 func TestTreeDigestContentSensitivity(t *testing.T) {
 	t.Parallel()
 
-	same, c := testHash("same"), testHash("c")
+	const sharedTail = "same"
+
 	recs := []scanRec{
-		{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
+		{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
-		{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
+		{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
 	}
 
 	_, dirs := treeOf(t, recs)
@@ -376,7 +254,7 @@ func TestCollectTreeGroupsMaximal(t *testing.T) {
 
 	super, dirs := treeOf(t, smokeTreeRecs())
 
-	groups := collectTreeGroups(dirs, super, nil)
+	groups := collectTreeGroups(dirs, super)
 
 	want := [][]string{{"/d/t1", "/d/t2"}}
 	if got := groupPaths(groups); !slices.EqualFunc(got, want,
@@ -400,14 +278,14 @@ func TestCollectTreeGroupsDeterministic(t *testing.T) {
 
 	super, dirs := treeOf(t, recs)
 
-	forward := groupPaths(collectTreeGroups(dirs, super, nil))
+	forward := groupPaths(collectTreeGroups(dirs, super))
 
 	reversed := slices.Clone(recs)
 	slices.Reverse(reversed)
 
 	superR, dirsR := treeOf(t, reversed)
 
-	backward := groupPaths(collectTreeGroups(dirsR, superR, nil))
+	backward := groupPaths(collectTreeGroups(dirsR, superR))
 	if !slices.EqualFunc(forward, backward, slices.Equal) {
 		t.Fatalf("output depends on record order: %v vs %v",
 			forward, backward)
@@ -419,15 +297,14 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
 
 	// Identical sibling dirs share a parent, so their group cannot be
 	// implied by a parent group and must be reported.
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 	recs := []scanRec{
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
 	}
 
 	super, dirs := treeOf(t, recs)
 
-	got := groupPaths(collectTreeGroups(dirs, super, nil))
+	got := groupPaths(collectTreeGroups(dirs, super))
 
 	want := [][]string{{"/p/x1", "/p/x2"}}
 	if !slices.EqualFunc(got, want, slices.Equal) {
@@ -441,16 +318,15 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 	// /p/a and /q/b contain identical x subtrees, but /p/a has an
 	// extra file, so the parents' digests differ and the x group must
 	// be reported.
-	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
 	recs := []scanRec{
-		{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
-		{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
+		{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
-		{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
 	}
 
 	super, dirs := treeOf(t, recs)
 
-	got := groupPaths(collectTreeGroups(dirs, super, nil))
+	got := groupPaths(collectTreeGroups(dirs, super))
 
 	want := [][]string{{"/p/a/x", "/q/b/x"}}
 	if !slices.EqualFunc(got, want, slices.Equal) {