Merge branch 'size-census': hash only files with shared sizes

Restructure scan into three phases: walk+stat, hash, update.

The stat pass is folded into the walk workers: each regular file is
lstatted as its directory is read, while the metadata is hot. The
walk builds a scan-wide size census (walked files plus records
outside the scan roots), and unchanged already-hashed files resolve
during the walk without further work.

Only files whose size at least one other file shares are ever read:
a size-unique file cannot be a duplicate, so it is recorded without
hashes (head and tail empty). When a later scan makes its size
shared, the file is hashed then, even if otherwise unchanged. report
excludes unhashed records; trees gives them a never-matching
signature so a tree containing one never compares equal to another.

Hashed records are committed in batched transactions while the hash
phase runs, so an interrupted scan keeps everything hashed so far
and the next run resumes cheaply. The hash phase total is exact,
giving a meaningful ETA.

Memory drops accordingly: the existing-record index holds only path,
size, mtime, and a hashed flag (no hash values); the walk carries one
small record per candidate file; overlapping operands are pruned up
front instead of deduplicating every walked path in a scan-wide set.
Files no bigger than one chunk are hashed with a single read.

.claude/settings.json

@@ -0,0 +1,5 @@
+{
+    "worktree": {
+        "bgIsolation": "none"
+    }
+}

.dockerignore

@@ -0,0 +1,8 @@
+.git
+.claude
+.DS_Store
+sfdupes
+files.dat
+*.log
+*.out
+*.test

.gitea/workflows/check.yml

@@ -0,0 +1,9 @@
+name: check
+on: [push]
+jobs:
+    check:
+        runs-on: ubuntu-latest
+        steps:
+            # actions/checkout v4.2.2, 2026-02-22
+            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+            - run: docker build .

.gitignore

@@ -28,6 +28,9 @@ node_modules/
 
 # Local scan data
 files.dat
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
 
 # Agent worktrees
 .claude/worktrees/

.golangci.yml

@@ -0,0 +1,32 @@
+version: "2"
+
+run:
+  timeout: 5m
+  modules-download-mode: readonly
+
+linters:
+  default: all
+  disable:
+    # Genuinely incompatible with project patterns
+    - exhaustruct      # Requires all struct fields
+    - depguard         # Dependency allow/block lists
+    - godot            # Requires comments to end with periods
+    - wsl              # Deprecated, replaced by wsl_v5
+    - wrapcheck        # Too verbose for internal packages
+    - varnamelen       # Short names like db, id are idiomatic Go
+
+linters-settings:
+  lll:
+    line-length: 88
+  funlen:
+    lines: 80
+    statements: 50
+  cyclop:
+    max-complexity: 15
+  dupl:
+    threshold: 100
+
+issues:
+  exclude-use-default: false
+  max-issues-per-linter: 0
+  max-same-issues: 0

Dockerfile

@@ -0,0 +1,38 @@
+# Lint stage — fast feedback on formatting and lint issues
+# golangci/golangci-lint:v2.12.1, 2026-07-23
+FROM golangci/golangci-lint@sha256:c9843d374ca80ecbac86081ec4dd7fe2bb6187b03224f59a0cc2f80759e1845b AS lint
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN make fmt-check
+RUN make lint
+
+# Build stage
+# golang:1.25-alpine, 2026-07-23
+FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS builder
+
+RUN apk add --no-cache make
+
+WORKDIR /src
+
+# Reuse the linter binary from the lint stage; the copy also forces
+# BuildKit to complete linting before this stage proceeds.
+COPY --from=lint /usr/bin/golangci-lint /usr/local/bin/golangci-lint
+
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+
+# Fail the build unless the branch is green.
+RUN make check
+
+RUN make build
+
+# Runtime stage
+# alpine:3.22, 2026-07-23
+FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce
+
+COPY --from=builder /src/sfdupes /usr/local/bin/sfdupes
+
+ENTRYPOINT ["sfdupes"]

Makefile

@@ -2,16 +2,49 @@
 # toolchain.
 export CGO_ENABLED = 0
 
-.PHONY: all build check clean
+BINARY := sfdupes
+VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
+LDFLAGS := -X main.Version=$(VERSION)
 
-all: build
+.PHONY: sfdupes build test lint fmt fmt-check check docker hooks clean
 
-build:
+# Default target: build the binary. Phony so go build (which has its
-	go build
+# own build cache) always decides what to recompile.
+sfdupes:
+	go build -ldflags "$(LDFLAGS)" -o $(BINARY)
 
-check: build
+build: sfdupes
-	test -z "$$(gofmt -l .)"
+
-	go vet ./...
+test:
+	@go test -timeout 30s -cover ./... || \
+		{ echo "--- Rerunning with -v for details ---"; \
+		  go test -timeout 30s -v ./...; exit 1; }
+
+lint:
+	golangci-lint run --config .golangci.yml ./...
+
+fmt:
+	gofmt -s -w .
+
+fmt-check:
+	@files="$$(gofmt -l -s .)"; if [ -n "$$files" ]; then \
+		echo "gofmt: files not formatted:"; echo "$$files"; exit 1; fi
+
+check: test lint fmt-check
+
+docker:
+	docker build -t $(BINARY) .
+
+# Hooks are shared between the main checkout and all worktrees, so
+# resolve the common git dir instead of assuming .git is a directory.
+HOOKS_DIR := $(shell git rev-parse --git-common-dir)/hooks
+
+hooks:
+	@printf '#!/bin/sh\nset -e\n' > $(HOOKS_DIR)/pre-commit
+	@printf 'go mod tidy\ngo fmt ./...\n' >> $(HOOKS_DIR)/pre-commit
+	@printf 'git diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> $(HOOKS_DIR)/pre-commit
+	@printf 'make check\n' >> $(HOOKS_DIR)/pre-commit
+	@chmod +x $(HOOKS_DIR)/pre-commit
 
 clean:
-	rm -f sfdupes files.dat
+	rm -f $(BINARY) files.dat

README.md

@@ -1,18 +1,69 @@
 # sfdupes
 
-`sfdupes` quickly identifies *candidate* duplicate files — and, ultimately,
+## Description
-entire duplicate directory trees — across very large filesystems without
+
-reading full file contents. Files are considered duplicates when they have
+`sfdupes` is an MIT-licensed Go CLI tool by
-identical size, identical SHA-256 of their first 1024 bytes, and identical
+[@sneak](https://sneak.berlin) that quickly identifies *candidate*
-SHA-256 of their last 1024 bytes. This is a strong candidate signal, not
+duplicate files — and, ultimately, entire duplicate directory trees —
-proof of identical content (the middle of the file is never read); the
+across very large filesystems without reading full file contents. Files
-intended use is finding duplicate downloads and duplicated directory trees
+are considered duplicates when they have identical size, identical
-on multi-terabyte ZFS servers where reading every byte is prohibitively
+SHA-256 of their first 1024 bytes, and identical SHA-256 of their last
-expensive.
+1024 bytes. This is a strong candidate signal, not proof of identical
+content (the middle of the file is never read); the intended use is
+finding duplicate downloads and duplicated directory trees on
+multi-terabyte ZFS servers where reading every byte is prohibitively
+expensive. `scan` maintains a persistent SQLite database of file
+signatures that survives between runs, so it can be run from cron and
+the reports can be generated at any time from the most recent scan.
+
+This tool was created by [@sneak](https://sneak.berlin) to scratch an itch,
+using Claude Code/Fable.
 
 This README is the complete and authoritative specification.
 
-## Design goals
+## Getting Started
+
+```sh
+make build
+export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
+./sfdupes scan /srv
+./sfdupes report > dupes.tsv
+./sfdupes trees > dupetrees.tsv
+```
+
+`scan` walks one or more filesystem trees and maintains one database
+record per regular file (path, size, mtime, head hash, tail hash). The
+database persists between runs; a rescan only hashes files that are new
+or changed, and removes records for files that no longer exist.
+`report` reads the database and prints the file-level duplicates
+report. `trees` reads the same database and prints the duplicate-tree
+report. A missing/invalid subcommand — or a `scan` invocation with no
+`PATH` operand — prints a usage message and exits 2.
+
+The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed
+anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a
+daily `sfdupes scan` cron job, with the reporting commands run
+interactively whenever needed; their results are as fresh as the last
+completed scan.
+
+## Rationale
+
+Duplicate finders that hash entire files do not scale to the target
+environment: ~10 million files and ~150 TB on possibly slow or busy
+disks (a ZFS pool under resilver). Reading at most 2 KiB per file — and
+only from files whose size at least one other file shares, since a
+size-unique file cannot be a duplicate — makes a full-filesystem sweep
+tractable, and the signatures are kept in a persistent database, so
+the expensive filesystem pass is incremental: a rescan re-hashes only
+files whose recorded mtime or size changed, and all analysis happens
+offline from the database alone. The end goal is
+not individual files but whole duplicated trees — duplicate
+extractions, duplicate downloads, copied project trees — which an
+operator can consider removing as a unit.
+
+## Design
+
+Goals, in order:
 
 1.  **Find whole duplicate trees, not just files.** The end goal is to
     identify places where the exact same set of files and directories
@@ -21,133 +72,245 @@ This README is the complete and authoritative specification.
     removing an entire subtree at once. File-level duplicate detection is
     the foundation; tree-level detection is built on top of it.
 2.  **Never read full file contents.** At most 2 KiB is read per file
-    (first and last 1024 bytes). Scale target: ~10 million files, ~150 TB
+    (first and last 1024 bytes), and only files whose size at least
+    one other file shares are read at all — a size-unique file cannot
+    be a duplicate. Scale target: tens of millions of files, ~150 TB
     filesystem, possibly slow or busy disks (ZFS pool under resilver).
-    Holding the full file list in memory is acceptable; reading file
+    Holding one small record (path, size, mtime) per file in memory
-    contents beyond 2 KiB per file is not.
+    during a scan is acceptable; holding every file's hashes is not
-3.  **Scan once, analyze offline.** The expensive filesystem scan
+    (they stay in the database).
-    produces a self-contained stream; all analysis (`report`, `trees`)
+3.  **Scan incrementally, analyze offline.** The expensive filesystem
-    works from that stream alone and must never touch the scanned
+    scan maintains a persistent database; an unchanged file is never
-    filesystem again.
+    read again on a rescan. All analysis (`report`, `trees`) works from
+    the database alone and must never touch the scanned filesystem
+    again. `scan` is designed to be cronned; the reports run at any
+    time against the last completed scan.
 4.  **Clean stream separation.** Everything on stdout is machine-readable
     data. All progress, warnings, and summaries go to stderr. Never mix
     them.
 
-## Constraints
+### Constraints
 
--   Language: Go (module `sneak.berlin/go/sfdupes`, already initialized
+-   Language: Go (module `sneak.berlin/go/sfdupes`). Binary name:
-    in `go.mod`). Binary name: `sfdupes`.
+    `sfdupes`.
--   Dependencies: standard library, `github.com/spf13/cobra` for the CLI,
+-   Dependencies: standard library, `github.com/spf13/cobra` for the
-    and **one progress-bar library**
+    CLI, **one progress-bar library**
-    (`github.com/schollz/progressbar/v3`). `github.com/spf13/viper` is
+    (`github.com/schollz/progressbar/v3`), and **one SQLite driver**
-    permitted if configuration-file support is ever needed, but is not
+    (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled).
-    currently used. No other third-party deps.
+    `github.com/spf13/viper` is permitted if configuration-file support
+    is ever needed, but is not currently used. No other third-party
+    deps.
 -   Cross-compilation is not a concern. Builds run with cgo disabled (the
     `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go.
 -   Analysis modes (`report`, `trees`) must be deterministic: identical
-    input stream, identical output, regardless of record order.
+    database contents, identical output, regardless of the order in
+    which records were inserted.
 
-## Plan
+### Subcommands
 
-Three subcommands, built in this order:
+Three subcommands, all implemented:
 
-1.  `scan` — walk the filesystem and emit one signature record per
+1.  `scan` — walk the filesystem and synchronize the database: one
-    regular file (implemented).
+    signature record per regular file.
-2.  `report` — file-level duplicate report from the scan stream
+2.  `report` — file-level duplicate report from the database.
-    (implemented).
 3.  `trees` — tree-level duplicate report: reconstruct the directory
-    hierarchy from the scan stream, compute a Merkle-style digest per
+    hierarchy from the database records, compute a Merkle-style digest
-    directory, and report maximal groups of identical trees
+    per directory, and report maximal groups of identical trees.
-    (implemented).
-
-Possible later work, explicitly out of scope for now: full-content
-verification of candidates, and helpers that emit removal scripts.
-
-## Usage
 
 ```
-sfdupes scan [-root /srv] [-workers N] > files.dat
+sfdupes scan [--workers N] [-x] PATH...
-sfdupes report [files.dat|-] > dupes.tsv
+sfdupes report > dupes.tsv
-sfdupes trees [files.dat|-] > dupetrees.tsv
+sfdupes trees > dupetrees.tsv
 ```
 
-`scan` walks a filesystem tree and emits one record per regular file
+### Database
-(path, size, mtime, head hash, tail hash). `report` ingests that stream
-and prints the file-level duplicates report. `trees` ingests the same
-stream and prints the duplicate-tree report. A missing/invalid subcommand
-prints a usage message and exits 2.
 
-## `scan` mode
+All three subcommands operate on a single SQLite database file:
 
-`scan` runs **three sequential passes**, in this order, so that every
+-   Location: the value of the `SFDUPES_DATABASE` environment variable
-expensive pass has an exact total for meaningful progress and ETA:
+    when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`.
+    There is no command-line flag.
+-   `scan` creates the database (and its parent directory) on first
+    use. `report` and `trees` require an existing database; a missing
+    database file is a fatal error (exit 1) telling the user to run
+    `scan` first.
+-   The database uses WAL journal mode and a busy timeout, so running a
+    report while a cron `scan` is in progress is safe. The filesystem
+    is authoritative; the database is an eventually-consistent
+    reflection of it. Hashed records are committed in batched
+    transactions while the scan is still running (keeping the WAL
+    small and letting concurrent reports observe progress), so a
+    report may see a scan's changes partially applied, and a scan
+    that dies partway leaves a valid database holding everything
+    hashed so far; the next scan skips those records and converges
+    toward the filesystem.
+-   Schema (`PRAGMA user_version` is the schema version, currently 1; a
+    database with any other version is a fatal error):
 
-1.  **walk** — recursively enumerate the tree under `-root` (default
+    ```sql
-    `/srv`), collecting the list of regular-file paths. Total unknown
+    CREATE TABLE files (
-    while running: show a live count, not a percentage.
+        path  BLOB PRIMARY KEY,  -- absolute path, raw bytes
-2.  **stat** — `lstat` every collected path, recording size and mtime.
+        size  INTEGER NOT NULL,  -- bytes, from lstat
-3.  **hash** — for each file, read the first `min(1024, size)` bytes and
+        mtime INTEGER NOT NULL,  -- Unix seconds, from lstat
-    the last `min(1024, size)` bytes (the two reads overlap when
+        head  TEXT NOT NULL,     -- lowercase-hex SHA-256, first 1 KiB
-    `size < 2048`; for `size == 0` hash the empty input) and compute the
+        tail  TEXT NOT NULL      -- lowercase-hex SHA-256, last 1 KiB
-    SHA-256 of each. Emit the output record.
+    ) WITHOUT ROWID;
+    ```
+
+    Paths are stored as BLOBs because Unix paths are raw bytes, not
+    guaranteed UTF-8. `mtime` is used only for change detection; it is
+    not part of the duplicate key. `head` and `tail` are empty strings
+    when the file has never been hashed because its size was unique as
+    of the last scan that covered it; such records still define the
+    file for tree reconstruction but never participate in duplicate
+    groups.
+
+### `scan` mode
+
+`scan` requires one or more `PATH` operands naming the trees to scan.
+There is no default path; invoking `scan` with no operand is a usage
+error (usage message on stderr, exit 2). An operand may be a directory
+or a regular file; an operand that does not exist is a fatal error
+(exit 1). Because database records persist between runs and are keyed
+by absolute path, each operand is resolved to an absolute, lexically
+cleaned path (symlinks are not resolved) before walking, so results do
+not depend on the working directory. All operands belong to a single
+scan and are enumerated concurrently: every operand seeds the shared
+walk worker pool. Overlapping operands are harmless — an operand that
+duplicates another or lies under another is dropped before walking,
+so every file is reached exactly once and produces one database
+record.
+
+`scan` synchronizes the database with the filesystem state under the
+scanned operands:
+
+-   Only a file whose size at least one other file shares is ever
+    read: a size-unique file cannot be a duplicate, so it is recorded
+    without hashes (`head` and `tail` empty). The size census covers
+    every file walked this scan plus every database record outside
+    the scanned operands, so a possible duplicate of a separately
+    scanned tree is still recognized.
+-   A file not yet in the database is inserted: hashed when its size
+    is shared, without hashes otherwise.
+-   A file already in the database is **skipped without reading its
+    contents** when its lstat size equals the recorded size and its
+    lstat mtime is not newer than the recorded mtime. This is what
+    makes a daily rescan cheap. Exception: an unchanged file whose
+    record lacks hashes is hashed — and its record updated — once its
+    size becomes shared, so hashing deferred by size-uniqueness
+    happens as soon as it could matter.
+-   A file whose mtime is newer than recorded, or whose size differs,
+    is processed as if new: re-hashed, or recorded without hashes,
+    per the shared-size rule.
+-   A database record whose path lies under one of the scanned operands
+    but was not successfully processed this run is deleted. This
+    removes records for deleted files. It also removes records for
+    paths that failed to stat or hash this run: the database only ever
+    contains signatures verified by the most recent scan that covered
+    them (a subsequent successful scan re-adds such files).
+-   Database records outside the scanned operands are untouched, so
+    disjoint trees can be scanned on different schedules into the same
+    database.
+
+`scan` runs **three sequential phases over the whole scan**.
+Parallelism lives inside each phase; batched database writes begin
+during the hash phase:
+
+1.  **walk + stat** — enumerate the trees under all `PATH` operands
+    concurrently with the walk worker pool: every operand seeds the
+    shared queue, and each worker reads one directory at a time,
+    handing discovered subdirectories back to the queue and running
+    `lstat` on each regular file as it is discovered (while the
+    directory's metadata is still hot). Sequential directory
+    enumeration is metadata-latency-bound and takes hours at tens of
+    millions of files; per-directory parallelism is what makes the
+    walk tractable on large or busy pools. The walk builds the size
+    census and resolves unchanged already-hashed files on the fly;
+    every other file is carried to the hash phase as a (path, size,
+    mtime) record.
+2.  **hash** — with the census complete, each carried file's size
+    decides its fate. Size-unique files are never read: new or
+    changed ones are recorded without hashes in the update phase,
+    unchanged unhashed ones simply keep their records. Every file
+    with a shared size is hashed by the worker pool: read the first
+    `min(1024, size)` bytes and the last `min(1024, size)` bytes
+    (one read when `size <= 1024`, since the two windows coincide;
+    for `size == 0` hash the empty input) and compute the SHA-256 of
+    each. The phase total is exact, so progress and ETA are
+    meaningful. Completed records are committed in batched
+    transactions **while hashing runs**, so a scan interrupted after
+    hours keeps everything hashed so far and the next scan resumes
+    cheaply, skipping records already written.
+3.  **update** — commit the final partial batch, the hash-less
+    records for size-unique new and changed files, and the deletions
+    for records the scan did not verify (vanished files, plus paths
+    that failed to stat or hash).
 
 Rules for the walk:
 
--   Only regular files. Skip directories, symlinks (do not follow),
+-   Only regular files. Skip directories, symlinks (do not follow,
-    sockets, FIFOs, and device nodes.
+    including symlink operands), sockets, FIFOs, and device nodes.
 -   Never descend into a directory named `.zfs` (ZFS snapshot pseudo-dirs;
     walking them would list every file once per snapshot).
+-   Filesystem boundaries are crossed by default. With `-x`
+    (long form `--one-file-system`, following the GNU `du`/`rsync`
+    convention), never descend into a directory on a different
+    filesystem than its `PATH` operand; each operand is bounded by its
+    own filesystem.
 -   On any per-path error (permission denied, file vanished between
     passes, unreadable): print a one-line warning to stderr, skip the
     path, and continue. Per-file errors never abort the run; the final
-    summary reports how many were skipped.
+    summary reports how many were skipped. As specified above, a
+    skipped path that has a database record from an earlier scan loses
+    that record; an unreadable directory subtree likewise loses its
+    records (accepted: the database mirrors what the latest scan could
+    actually verify).
 
-Concurrency: the stat and hash passes use a worker pool (`-workers`,
+Concurrency: the walk phase (which also stats files) and the hash
-default `runtime.NumCPU()`). The main goroutine owns stdout writing and
+phase each use a worker pool of `--workers` workers (default
-progress rendering; progress display must never block the workers.
+`runtime.NumCPU()`); the walk parallelizes across directories,
+hashing across files. Both phases are seek-bound on spinning disks,
+so raising `--workers` well past the core count can help on pools
+with many spindles. The main goroutine owns partitioning, database
+writes, and progress rendering; progress display must never block
+the workers.
 
-### Output record format
+`scan` writes nothing to stdout. The summary line on stderr reports the
-
+files seen this run broken down by disposition, plus skips:
-One record per file on stdout, NUL-terminated (`\x00`), with
-tab-separated fields, **path last** so tabs or newlines embedded in
-paths cannot corrupt the record structure:
 
 ```
-<size>\t<mtime_unix>\t<sha256_first1k_hex>\t<sha256_last1k_hex>\t<path>\x00
+scan: 123456 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped
 ```
 
--   `size`: decimal bytes, from the stat pass.
+(`removed` counts deleted database records, which are not part of the
--   `mtime_unix`: decimal Unix seconds. Informational only; not part of
+files-seen total.)
-    the duplicate key.
--   Hashes: lowercase hex, 64 chars each.
--   Record order is unspecified (workers complete out of order); the
-    analysis modes must not depend on ordering.
 
-## `report` mode
+### `report` mode
 
-`report` reads the scan stream from the file named in its first
+`report` reads every record from the database and takes no positional
-positional argument, or from stdin if the argument is absent or `-`.
+arguments.
 
 **`report` must never touch the filesystem being analyzed.** It does not
 stat, open, or otherwise access any path that appears in the records; its
-only I/O is reading the scan file/stdin and writing stdout/stderr. It must
+only I/O is reading the database and writing stdout/stderr. It must
 produce identical output whether or not the scanned filesystem is still
 mounted.
 
 Processing:
 
--   Parse records; a record that does not have exactly 5 fields or whose
+-   Records without hashes (size-unique when last scanned) are
-    size is non-numeric is counted as malformed and skipped (warn once
+    excluded: their content is unknown, so they are never reported as
-    with the total malformed count in the summary, not per record).
+    duplicates.
--   Group records by the key `(size, head_hash, tail_hash)`.
+-   Group the remaining records by the key
+    `(size, head_hash, tail_hash)`.
 -   Every group with two or more paths is a duplicate group.
 -   Within each group, sort paths lexicographically (byte order). The
     first path is the group's `first`; every other path is a `dupe`.
 -   Order groups by size descending (biggest reclaimable space first),
     tie-broken by `first` path ascending. Output must be fully
-    deterministic for a given input.
+    deterministic for a given database state.
 
-### Report output format
+#### Report output format
 
 TSV on stdout: a header line, then one row per duplicate file (N-1 rows
 for a group of N):
@@ -158,16 +321,16 @@ first	dupe	size
 /srv/a/big.iso	/srv/c/big-copy2.iso	4294967296
 ```
 
-Summary to stderr: records read, malformed count (if any), number of
+Summary to stderr: records read, number of duplicate groups, number of
-duplicate groups, number of dupe files, and total reclaimable bytes
+dupe files, and total reclaimable bytes (sum of `size` over all dupe
-(sum of `size` over all dupe rows) in human units.
+rows) in human units.
 
-## `trees` mode
+### `trees` mode
 
-`trees` reads the same scan stream as `report` (same argument handling,
+`trees` reads the same database as `report` (no positional arguments)
-same parsing and malformed-record rules) and reports **entire duplicate
+and reports **entire duplicate directory trees**: directories under
-directory trees**: directories under which the exact same set of relative
+which the exact same set of relative paths exists with the exact same
-paths exists with the exact same file signatures.
+file signatures.
 
 **`trees` must never touch the filesystem being analyzed** — the same
 rule as `report`. The directory hierarchy is reconstructed purely from
@@ -176,7 +339,10 @@ the paths in the records, split on `/`.
 Definitions:
 
 -   A file's **signature** is `(size, head_hash, tail_hash)` — mtime is
-    informational and excluded.
+    informational and excluded. An unhashed record (empty hashes) has
+    unknown content: its signature is treated as unique to that file,
+    so a tree containing an unhashed file never compares equal to any
+    other tree.
 -   A directory's **digest** is a SHA-256 Merkle digest computed
     bottom-up: serialize the directory's child entries — for a file
     child, its name and signature; for a subdirectory child, its name
@@ -188,10 +354,10 @@ Definitions:
     equal. Equal digests imply equal recursive file count and equal
     total byte size.
 
-Known limitation (accepted): only regular files that appear in the scan
+Known limitation (accepted): only regular files that appear in the
-stream define a tree. Empty directories are invisible, and a file skipped
+database define a tree. Empty directories are invisible, and a file
-during the scan (e.g. permission error) in one copy but not the other
+skipped during the scan (e.g. permission error) in one copy but not the
-will make otherwise-identical trees compare as different.
+other will make otherwise-identical trees compare as different.
 
 Processing:
 
@@ -209,7 +375,7 @@ Processing:
     path ascending. Output must be fully deterministic for a given
     input.
 
-### Trees output format
+#### Trees output format
 
 TSV on stdout: a header line, then one row per duplicate tree (N-1 rows
 for a group of N). `files` is the recursive regular-file count of one
@@ -220,32 +386,33 @@ first	dupe	files	size
 /srv/a/project	/srv/backup/project	3417	104857600
 ```
 
-Summary to stderr: records read, malformed count (if any), number of
+Summary to stderr: records read, number of duplicate-tree groups,
-duplicate-tree groups, number of dupe trees, and total reclaimable bytes
+number of dupe trees, and total reclaimable bytes (sum of `size` over
-(sum of `size` over all dupe rows) in human units.
+all dupe rows) in human units.
 
-## Progress
+### Progress
 
-Use the progress-bar library for all scan-pass progress; rendering in the
+Use the progress-bar library for all scan progress; rendering in the
 style of `pv` is the model. All progress goes to stderr.
 
-Each scan pass gets its own bar. Required elements for the stat and hash
+Each phase gets its own display. The walk has no known total while
-passes (known totals):
+running: show a live file count, rate, and elapsed time
+(spinner-style, no percentage or ETA). The hash and update phases
+have exact totals — only files that actually need hashing appear in
+the hash total, so its ETA is meaningful. Required elements for the
+bars with known totals:
 
 -   elapsed time
 -   estimated time remaining
--   a `[m/n] x%` display (files processed / total files, percent)
+-   a `[m/n] x%` display (items processed / total items, percent)
--   current rate (files/s)
+-   current rate (items/s)
 
 Example shape (exact layout is flexible, content is not):
 
 ```
-hash: [1234567/9876543] 12% |████      | 8123 files/s elapsed 2:32 eta 17:54
+hash: [12345/98765] 12% |████      | 92 files/s elapsed 2:32 eta 17:54
 ```
 
-The walk pass has no known total: show a live file count and elapsed time
-(spinner-style, no percentage or ETA).
-
 Additional requirements:
 
 -   When stderr is not a TTY, do not emit ANSI redraws: print a plain
@@ -255,31 +422,44 @@ Additional requirements:
 -   `report` and `trees` modes need no progress display, only their
     stderr summaries.
 
-## Error handling and exit codes
+### Error handling and exit codes
 
 -   `0`: success, even if individual files were skipped with warnings.
--   `1`: fatal error (e.g., `-root` does not exist, cannot read the scan
+-   `1`: fatal error (e.g., a `PATH` operand does not exist, the
-    input, stdout write failure).
+    database cannot be created/opened/read/written, a missing database
--   `2`: usage error.
+    for `report`/`trees`, stdout write failure).
+-   `2`: usage error (including `scan` with no `PATH` operand and
+    `report`/`trees` with any positional argument).
 
 ## Build
 
-`make` (or `make build`) builds the binary with cgo disabled. `make
+The `Makefile` is the single source of truth for all operations:
-check` builds and runs `gofmt` and `go vet`. `make clean` removes the
-binary and any local `files.dat`.
 
-## Definition of done
+-   `make` / `make build` — build the `sfdupes` binary (cgo
+    disabled); building is the default target.
+-   `make test` — run the test suite (30-second timeout; reruns with
+    `-v` on failure).
+-   `make lint` — run `golangci-lint` with the repo config.
+-   `make fmt` / `make fmt-check` — format Go sources / verify
+    formatting without writing.
+-   `make check` — `test`, `lint`, and `fmt-check`; modifies nothing.
+-   `make docker` — build the Docker image, which runs `make check` as
+    a build stage.
+-   `make hooks` — install the pre-commit hook.
+-   `make clean` — remove the binary and any legacy local `files.dat`.
+
+### Definition of done
 
 All of the following, run in this directory, must pass:
 
-1.  `gofmt -l .` prints nothing.
+1.  `make check` passes (tests, lint, `gofmt`).
-2.  `go vet ./...` is clean.
+2.  `make docker` succeeds.
-3.  `go build` succeeds (equivalently, `make check` passes).
+3.  Smoke test — create a throwaway tree in a temp dir (never test
-4.  Smoke test — create a throwaway tree in a temp dir (never test
     against real data):
 
     ```sh
     d=$(mktemp -d)
+    export SFDUPES_DATABASE="$d/db.sqlite"
     mkdir -p "$d/a" "$d/b"
     head -c 2000 /dev/urandom > "$d/a/one.bin"
     cp "$d/a/one.bin" "$d/b/copy.bin"
@@ -296,35 +476,59 @@ All of the following, run in this directory, must pass:
     cp "$d/t1/sub/f2" "$d/t2/sub/f2"
     cp "$d/t1/f1" "$d/t3/f1"
     cp "$d/t1/sub/f2" "$d/t3/sub/f2renamed"
-    ./sfdupes scan -root "$d" > files.dat
+    ./sfdupes scan "$d"
-    ./sfdupes report files.dat
+    ./sfdupes report
-    ./sfdupes trees files.dat
+    ./sfdupes trees
+    # incremental behavior (scan a subtree; records elsewhere persist):
+    ./sfdupes scan "$d/a"                  # everything unchanged, nothing hashed
+    printf 'z' >> "$d/a/one.bin"           # modify: next scan re-hashes it
+    rm "$d/a/unique.bin"                   # delete: next scan removes its record
+    ./sfdupes scan "$d/a"                  # 1 updated, 1 removed
+    ./sfdupes report
     ```
 
-    Expected from `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
+    (The scan database lives inside `$d` here purely for test hygiene;
-    group (two dupe rows, `first` is the lexicographically smallest
+    scanning `$d` therefore also records the SQLite file itself, which
-    path); `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
+    is harmless.)
-    `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair;
+
-    `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere;
+    Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin`
-    groups ordered by size descending; piping scan directly into report
+    form one group (two dupe rows, `first` is the lexicographically
-    (`./sfdupes scan -root "$d" | ./sfdupes report`) gives the same
+    smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group;
-    rows.
+    `t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group;
+    `tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and
+    `tiny3` appear nowhere; groups ordered by size descending.
 
     Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe`
     `$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is
     suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3`
     appears nowhere (its file set differs by name).
 
-5.  A negative check: run `report` and `trees` after deleting the temp
+    Expected from the second `report` (after the modify/delete rescan):
-    tree — output must be unchanged (proves the analysis modes never
+    `one.bin` has left its group (its content changed), so
-    touch the filesystem).
+    `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is
+    gone from the database.
+
+    The test suite automates this scenario (see `scan_test.go`), plus a
+    negative check: `report` and `trees` operate on the database alone
+    and never touch the scanned filesystem.
+
+## TODO
+
+Tracked in [TODO.md](TODO.md).
 
 ## Non-goals
 
 -   No full-content verification, no byte-for-byte compare, no deletion
     or linking of duplicates. The reports are advisory; acting on them is
     the user's job.
--   No persistence formats beyond the scan stream described above.
+-   No persistence beyond the SQLite database described above; no
--   No git repository setup and no CI — code, `go.mod`/`go.sum`, the
+    export/import formats.
-    `Makefile`, and this README only. Do not write anything outside this
+-   No daemon or filesystem watcher; scheduling rescans is cron's job.
-    directory (module cache aside).
+
+## License
+
+MIT. See [LICENSE](LICENSE).
+
+## Author
+
+[@sneak](https://sneak.berlin)

REPO_POLICIES.md

@@ -0,0 +1,368 @@
+---
+title: Repository Policies
+last_modified: 2026-07-06
+---
+
+This document covers repository structure, tooling, and workflow standards. Code
+style conventions are in separate documents:
+
+- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
+  (general, bash, Docker)
+- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
+- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
+- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
+- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
+
+---
+
+- Cross-project documentation (such as this file) must include
+  `last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
+  with the authoritative source as policies evolve.
+
+- **ALL external references must be pinned by cryptographic hash.** This
+  includes Docker base images, Go modules, npm packages, GitHub Actions, and
+  anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
+  `:3.21`, etc.) are server-mutable and therefore remote code execution
+  vulnerabilities. The ONLY acceptable way to reference an external dependency
+  is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
+  integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
+  This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
+  etc. Instead, download a specific release archive from GitHub, verify its hash
+  (hardcoded in the Dockerfile or script), and only then install. Unverified
+  install scripts are arbitrary remote code execution. This is the single most
+  important rule in this document. Double-check every external reference in
+  every file before committing. There are zero exceptions to this rule.
+
+- Every repo with software must have a root `Makefile` with these targets:
+  `make test`, `make lint`, `make fmt` (writes), `make fmt-check` (read-only),
+  `make check` (prereqs: `test`, `lint`, `fmt-check`), `make docker`, and
+  `make hooks` (installs pre-commit hook). A model Makefile is at
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
+
+- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
+  instead of invoking the underlying tools directly. The Makefile is the single
+  source of truth for how these operations are run.
+
+- The Makefile is authoritative documentation for how the repo is used. Beyond
+  the required targets above, it should have targets for every common operation:
+  running a local development server (`make run`, `make dev`), re-initializing
+  or migrating the database (`make db-reset`, `make migrate`), building
+  artifacts (`make build`), generating code, seeding data, or anything else a
+  developer would do regularly. If someone checks out the repo and types
+  `make<tab>`, they should see every meaningful operation available. A new
+  contributor should be able to understand the entire development workflow by
+  reading the Makefile.
+
+- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
+  as a build step so the build fails if the branch is not green. For non-server
+  repos, the Dockerfile should bring up a development environment and run
+  `make check`. For server repos, `make check` should run as an early build
+  stage before the final image is assembled.
+
+- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
+  repos use a multistage build where linting runs in an independent stage based
+  on the `golangci/golangci-lint` image (pinned by hash). This stage runs
+  `make fmt-check` and `make lint` before the full build begins. The build stage
+  then declares an explicit dependency on the lint stage via
+  `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
+  linting before proceeding to compilation and tests. This ensures lint failures
+  surface in seconds rather than minutes, without blocking on dependency
+  download or compilation in the build stage.
+
+    The standard pattern for a Go repo Dockerfile is:
+
+    ```dockerfile
+    # Lint stage — fast feedback on formatting and lint issues
+    # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
+    FROM golangci/golangci-lint@sha256:... AS lint
+    WORKDIR /src
+    COPY go.mod go.sum ./
+    RUN go mod download
+    COPY . .
+    RUN make fmt-check
+    RUN make lint
+
+    # Build stage
+    # golang:1.x-alpine, YYYY-MM-DD
+    FROM golang@sha256:... AS builder
+    WORKDIR /src
+
+    # Force BuildKit to run the lint stage before proceeding
+    COPY --from=lint /src/go.sum /dev/null
+
+    COPY go.mod go.sum ./
+    RUN go mod download
+    COPY . .
+    RUN make test
+
+    ARG VERSION=dev
+    RUN CGO_ENABLED=0 go build -trimpath \
+        -ldflags="-s -w -X main.Version=${VERSION}" \
+        -o /app ./cmd/app/
+
+    # Runtime stage
+    FROM alpine@sha256:...
+    COPY --from=builder /app /usr/local/bin/app
+    ENTRYPOINT ["app"]
+    ```
+
+    Key points:
+    - The lint stage uses the `golangci/golangci-lint` image directly (it
+      includes both Go and the linter), so there is no need to install the
+      linter separately.
+    - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
+      a stage dependency. BuildKit runs stages in parallel by default; without
+      this line, the build stage would not wait for lint to finish and a lint
+      failure might not fail the overall build.
+    - If the project uses `//go:embed` directives that reference build artifacts
+      (e.g. a web frontend compiled in a separate stage), the lint stage must
+      create placeholder files so the embed directives resolve. Example:
+      `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
+      The lint stage should not depend on the actual build output — it exists to
+      fail fast.
+    - If the project requires CGO or system libraries for linting (e.g.
+      `vips-dev`), install them in the lint stage with `apk add`.
+    - The build stage runs `make test` after compilation setup. Tests run in the
+      build stage, not the lint stage, because they may require compiled
+      artifacts or heavier dependencies.
+
+- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
+  runs `docker build .` on push. Since the Dockerfile already runs `make check`,
+  a successful build implies all checks pass.
+
+- Use platform-standard formatters: `black` for Python, `prettier` for
+  JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
+  two exceptions: four-space indents (except Go), and `proseWrap: always` for
+  Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
+  HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
+
+- Pre-commit hook: `make check` if local testing is possible, otherwise
+  `make lint && make fmt-check`. The Makefile should provide a `make hooks`
+  target to install the pre-commit hook.
+
+- All repos with software must have tests that run via the platform-standard
+  test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
+  tests exist yet, add the most minimal test possible — e.g. importing the
+  module under test to verify it compiles/parses. There is no excuse for
+  `make test` to be a no-op.
+
+- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
+  Makefile.
+
+- **`make test` should use the conditional verbose rerun pattern.** Run tests
+  without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
+  show full output. This keeps CI logs and `docker build` output clean on
+  success (just package/suite summaries) while providing full diagnostic detail
+  on failure (every test case, every assertion). The general shell pattern:
+
+    ```makefile
+    test:
+    	@<test-command> || \
+    		{ echo "--- Rerunning with -v for details ---"; \
+    		  <test-command-with-v>; exit 1; }
+    ```
+
+    Go example:
+
+    ```makefile
+    test:
+    	@go test -timeout 30s -race -cover ./... || \
+    		{ echo "--- Rerunning with -v for details ---"; \
+    		  go test -timeout 30s -race -v ./...; exit 1; }
+    ```
+
+    Python example:
+
+    ```makefile
+    test:
+    	@python -m pytest || \
+    		{ echo "--- Rerunning with -v for details ---"; \
+    		  python -m pytest -v; exit 1; }
+    ```
+
+    The `exit 1` ensures the target always fails after a rerun — the first run
+    already proved the tests are broken, so the build must not pass even if a
+    flaky test happens to succeed on the second attempt. The rerun exists solely
+    for diagnostic output.
+
+- Docker builds must complete in under 5 minutes.
+
+- `make check` must not modify any files in the repo. Tests may use temporary
+  directories.
+
+- `main` must always pass `make check`, no exceptions.
+
+- Never commit secrets. `.env` files, credentials, API keys, and private keys
+  must be in `.gitignore`. No exceptions.
+
+- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
+  editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
+  Fetch the standard `.gitignore` from
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
+  a new repo.
+
+- **No build artifacts in version control.** Code-derived data (compiled
+  bundles, minified output, generated assets) must never be committed to the
+  repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
+  should generate these at build time. Notable exception: Go protobuf generated
+  files (`.pb.go`) ARE committed because repos need to work with `go get`, which
+  downloads code but does not execute code generation.
+
+- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
+
+- Never force-push to `main`.
+
+- Make all changes on a feature branch. You can do whatever you want on a
+  feature branch.
+
+- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
+  manually by the user. Fetch from
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
+
+- When pinning images or packages by hash, add a comment above the reference
+  with the version and date (YYYY-MM-DD).
+
+- Use `yarn`, not `npm`.
+
+- Write all dates as YYYY-MM-DD (ISO 8601).
+
+- Simple projects should be configured with environment variables.
+
+- Dockerized web services listen on port 8080 by default, overridable with
+  `PORT`.
+
+- **HTTP/web services must be hardened for production internet exposure before
+  tagging 1.0.** This means full compliance with security best practices
+  including, without limitation, all of the following:
+    - **Security headers** on every response:
+        - `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
+          and `includeSubDomains`.
+        - `Content-Security-Policy` (CSP) with a restrictive default policy
+          (`default-src 'self'` as a baseline, tightened per-resource as
+          needed). Never use `unsafe-inline` or `unsafe-eval` unless
+          unavoidable, and document the reason.
+        - `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
+          Prefer the `frame-ancestors` CSP directive as the primary control.
+        - `X-Content-Type-Options: nosniff`.
+        - `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
+        - `Permissions-Policy` restricting access to browser features the
+          application does not use (camera, microphone, geolocation, etc.).
+    - **Request and response limits:**
+        - Maximum request body size enforced on all endpoints (e.g. Go
+          `http.MaxBytesReader`). Choose a sane default per-route; never accept
+          unbounded input.
+        - Maximum response body size where applicable (e.g. paginated APIs).
+        - `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
+          against slowloris attacks.
+        - `WriteTimeout` on the `http.Server`.
+        - `IdleTimeout` on the `http.Server`.
+        - Per-handler execution time limits via `context.WithTimeout` or
+          chi/stdlib `middleware.Timeout`.
+    - **Authentication and session security:**
+        - Rate limiting on password-based authentication endpoints. API keys are
+          high-entropy and not susceptible to brute force, so they are exempt.
+        - CSRF tokens on all state-mutating HTML forms. API endpoints
+          authenticated via `Authorization` header (Bearer token, API key) are
+          exempt because the browser does not attach these automatically.
+        - Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
+          MD5, or SHA.
+        - Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
+          `Strict`) attributes.
+    - **Reverse proxy awareness:**
+        - True client IP detection when behind a reverse proxy
+          (`X-Forwarded-For`, `X-Real-IP`). The application must accept
+          forwarded headers only from a configured set of trusted proxy
+          addresses — never trust `X-Forwarded-For` unconditionally.
+    - **CORS:**
+        - Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
+          an explicit allowlist of known origins. Wildcard (`*`) is acceptable
+          only for public, unauthenticated read-only APIs.
+    - **Error handling:**
+        - Internal errors must never leak stack traces, SQL queries, file paths,
+          or other implementation details to the client. Return generic error
+          messages in production; detailed errors only when `DEBUG` is enabled.
+    - **TLS:**
+        - Services never terminate TLS directly. They are always deployed behind
+          a TLS-terminating reverse proxy. The service itself listens on plain
+          HTTP. However, HSTS headers and `Secure` cookie flags must still be
+          set by the application so that the browser enforces HTTPS end-to-end.
+
+    This list is non-exhaustive. Apply defense-in-depth: if a standard security
+    hardening measure exists for HTTP services and is not listed here, it is
+    still expected. When in doubt, harden.
+
+- `README.md` is the primary documentation. Required sections:
+    - **Description**: First line must include the project name, purpose,
+      category (web server, SPA, CLI tool, etc.), license, and author. Example:
+      "µPaaS is an MIT-licensed Go web application by @sneak that receives
+      git-frontend webhooks and deploys applications via Docker in realtime."
+    - **Getting Started**: Copy-pasteable install/usage code block.
+    - **Rationale**: Why does this exist?
+    - **Design**: How is the program structured?
+    - **TODO**: Update meticulously, even between commits. When planning, put
+      the todo list in the README so a new agent can pick up where the last one
+      left off.
+    - **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
+      `LICENSE` file in the repo root and a License section in the README.
+    - **Author**: [@sneak](https://sneak.berlin).
+
+- First commit of a new repo should contain only `README.md`.
+
+- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
+  committing.
+
+- Use SemVer.
+
+- Database migrations live in `internal/db/migrations/` and must be embedded in
+  the binary.
+    - `000_migration.sql` — contains ONLY the creation of the migrations
+      tracking table itself. Nothing else.
+    - `001_schema.sql` — the full application schema.
+    - **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
+      There is no installed base to migrate. Edit `001_schema.sql` directly.
+    - **Post-1.0.0:** add new numbered migration files for each schema change.
+      Never edit existing migrations after release.
+
+- All repos should have an `.editorconfig` enforcing the project's indentation
+  settings.
+
+- **Claude Code repo memory is versioned in the repo**, not left only in
+  `~/.claude` on one machine. Each memory is one file at
+  `.claude/memory/<memory>.md`, and every memory file must be `@`-imported
+  from `.claude/CLAUDE.md` (one `- @memory/<memory>.md` list line per file;
+  relative import paths resolve against `.claude/`, and Claude Code expands
+  the imports into context at session launch). When adding a memory, add both
+  the file and its import line. A root `MEMORY.md` is a violation — Claude
+  Code never auto-loads it; split it into `.claude/memory/` files. Repos with
+  no memories yet need no `.claude/` scaffolding.
+
+- Avoid putting files in the repo root unless necessary. Root should contain
+  only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
+  `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
+  language-specific config). Everything else goes in a subdirectory. Canonical
+  subdirectory names:
+    - `bin/` — executable scripts and tools
+    - `cmd/` — Go command entrypoints
+    - `configs/` — configuration templates and examples
+    - `deploy/` — deployment manifests (k8s, compose, terraform)
+    - `docs/` — documentation and markdown (README.md stays in root)
+    - `internal/` — Go internal packages
+    - `internal/db/migrations/` — database migrations
+    - `pkg/` — Go library packages
+    - `share/` — systemd units, data files
+    - `static/` — static assets (images, fonts, etc.)
+    - `web/` — web frontend source
+
+- When setting up a new repo, files from the `prompts` repo may be used as
+  templates. Fetch them from
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
+
+- New repos must contain at minimum:
+    - `README.md`, `.git`, `.gitignore`, `.editorconfig`
+    - `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
+    - `Makefile`
+    - `Dockerfile`, `.dockerignore`
+    - `.gitea/workflows/check.yml`
+    - Go: `go.mod`, `go.sum`, `.golangci.yml`
+    - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
+    - Python: `pyproject.toml`

TODO.md

@@ -14,21 +14,67 @@
 
 # Next Step
 
-- bring the repo into full policy compliance (work the Repo Policy
+- convert Makefile targets to scripts-to-rule-them-all `script/`
-  Compliance checklist below)
+  entrypoints like the other managed repos
 
 # Completed Steps
 
+- make the binary the default Make target (2026-07-24, branch
+  `make-default-target`): plain `make` now builds `sfdupes`
+  (previously it ran `check` plus `build`); `make build` remains as
+  an alias
+- scan-wide phases, concurrent operands, batched updates (2026-07-24,
+  branch `scan-wide-phases`): all operands seed the shared walk pool
+  and every pass runs once over the whole scan, so totals and ETAs
+  are scan-global; the per-operand walk/hash/update cycles and their
+  stderr announcements are gone; the update pass commits in batched
+  transactions — the filesystem is authoritative and the database an
+  eventually-consistent reflection, so scan-level atomicity is not
+  required
+- split the stat pass back out of the walk (2026-07-24, branch
+  `parallel-phases`): phases are strictly sequential again — walk,
+  stat, hash, update per operand — with parallelism only inside each
+  phase; the walk enumerates paths with per-directory workers and the
+  stat pass lstats them with per-file workers, restoring the exact
+  total/ETA stat bar
+- announce each operand on stderr before its passes (2026-07-24,
+  branch `scan-operand-progress`): with per-operand walk/hash/update
+  cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals
+  that looked like the whole run's — an operator watching operand 3 of
+  14 hash 300k files concluded 20M files were being skipped
+- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass
+  was a single goroutine and took hours at ~20M files on a busy pool
+  (observed: 22M files in 4h on a ZFS server); it is now a
+  per-directory worker-pool traversal that records size/mtime during
+  the walk (folding away the separate stat pass, halving metadata
+  I/O), and each `PATH` operand commits in its own transaction so an
+  interrupted scan keeps completed operands
+
+- persistent scan database (2026-07-24, branch `persistent-database`):
+  `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure
+  Go, cgo stays disabled) keyed by absolute path that survives between
+  runs — a rescan hashes only new or changed files (by mtime/size),
+  deletes records for files vanished from under the scanned operands,
+  and leaves records outside them untouched, so `scan` can be cronned
+  daily; `report` and `trees` read the database (no positional
+  arguments) instead of a scan stream. Database at
+  `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`;
+  WAL journaling plus a single-transaction update keep a report run
+  during a scan safe
+- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag
+  `v0.0.1`, and push `main` plus tags (2026-07-23)
+- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
+  `PATH...` operands via cobra flags replacing the `/srv` `-root`
+  default; new `-x`/`--one-file-system` flag (GNU convention) to stop
+  at filesystem boundaries, which are crossed by default
+- bring the repo into full policy compliance (2026-07-23, branch
+  `repo-policy-compliance`; checklist below)
 - `git init` with README-only first commit; code baseline committed on
   `main` (2026-07-22)
 - implement `scan`, `report`, and `trees` subcommands (pre-git history)
 
 # Future Steps
 
-- add a remote on git.eeqj.de and push (`main` plus tags)
-- convert Makefile targets to scripts-to-rule-them-all `script/`
-  entrypoints like the other managed repos
-- tag `v0.0.1` once the compliance branch is merged
 - possible later features (explicitly out of scope per README):
   full-content verification of candidates, removal-script helpers
 
@@ -38,33 +84,34 @@ Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing
 repo checklist, and the Go styleguide. Code is already gofmt-clean, so no
 standalone formatting commit is needed.
 
-- [ ] `.gitignore` missing — the compiled `sfdupes` binary and
+- [x] `.gitignore` missing — the compiled `sfdupes` binary and
       `files.dat` sit untracked in the tree; needs OS/editor/Go
       artifacts plus secrets patterns
-- [ ] `.editorconfig` missing
+- [x] `.editorconfig` missing
-- [ ] `LICENSE` missing and README has no License section (MIT assumed
+- [x] `LICENSE` missing and README has no License section (MIT assumed
       from house convention — user to confirm)
-- [ ] `REPO_POLICIES.md` missing from repo root
+- [x] `REPO_POLICIES.md` missing from repo root
-- [ ] `.golangci.yml` missing (install canonical copy); code must then
+- [x] `.golangci.yml` missing (install canonical copy); code must then
-      pass `make lint`
+      pass `make lint` (150 findings fixed; `make lint` is clean)
-- [ ] `Makefile` lacks required targets `test`, `lint`, `fmt`,
+- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`,
       `fmt-check`, `docker`, `hooks`; `check` currently depends on
       `build`, which writes the binary (`make check` must not modify
       files)
-- [ ] no tests — `go test ./...` has nothing to run; policy requires
+- [x] no tests — `go test ./...` has nothing to run; policy requires
       real tests with a 30-second timeout and the conditional `-v`
-      rerun pattern
+      rerun pattern (suite covers parsing, grouping, digests,
-- [ ] `Dockerfile` missing — Go multistage with hash-pinned images:
+      suppression, hashing, and the scan pipeline; 64% coverage)
+- [x] `Dockerfile` missing — Go multistage with hash-pinned images:
       fail-fast lint stage, build stage running `make check`
-- [ ] `.dockerignore` missing
+- [x] `.dockerignore` missing
-- [ ] `.gitea/workflows/check.yml` missing (`docker build .` on push,
+- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push,
       checkout action pinned by commit SHA)
-- [ ] README lacks required sections: Description first line
+- [x] README lacks required sections: Description first line
       (name/purpose/category/license/author), Getting Started,
       Rationale, TODO, License, Author
-- [ ] README non-goal "no git repository setup and no CI" is stale now
+- [x] README non-goal "no git repository setup and no CI" is stale now
       that the repo is under git with CI
-- [ ] pre-commit hook not installed (`make hooks` once the target
+- [x] pre-commit hook not installed (`make hooks` once the target
       exists)
 
 Accepted divergences (no action):

db.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+
+	// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
+	// disabled.
+	_ "modernc.org/sqlite"
+)
+
+// defaultDatabasePath is where the persistent scan database lives when
+// SFDUPES_DATABASE is not set.
+const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite"
+
+// databaseEnv is the environment variable that overrides the database
+// path.
+const databaseEnv = "SFDUPES_DATABASE"
+
+// schemaVersion is the database schema version this build reads and
+// writes, stored in PRAGMA user_version.
+const schemaVersion = 1
+
+// dbDirPerm is the mode for a database parent directory created by
+// scan.
+const dbDirPerm = 0o755
+
+// createTableSQL is the schema applied to a fresh database. Paths are
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+const createTableSQL = `
+CREATE TABLE files (
+    path  BLOB PRIMARY KEY,
+    size  INTEGER NOT NULL,
+    mtime INTEGER NOT NULL,
+    head  TEXT NOT NULL,
+    tail  TEXT NOT NULL
+) WITHOUT ROWID
+`
+
+// upsertSQL inserts one file record, replacing any existing record for
+// the same path.
+const upsertSQL = `
+INSERT INTO files (path, size, mtime, head, tail)
+VALUES (?, ?, ?, ?, ?)
+ON CONFLICT (path) DO UPDATE SET
+    size = excluded.size, mtime = excluded.mtime,
+    head = excluded.head, tail = excluded.tail
+`
+
+// errNoDatabase reports a missing database file for report/trees.
+var errNoDatabase = errors.New(
+	"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
+
+// errSchemaVersion reports a database whose schema version this build
+// does not understand.
+var errSchemaVersion = errors.New("unsupported database schema version")
+
+// databasePath resolves the database location: SFDUPES_DATABASE when
+// set and non-empty, the compiled-in default otherwise.
+func databasePath() string {
+	if p := os.Getenv(databaseEnv); p != "" {
+		return p
+	}
+
+	return defaultDatabasePath
+}
+
+// openDB opens the SQLite database at path with WAL journaling and a
+// busy timeout, so a report can run while a cron scan is in progress.
+// It does not create or verify the schema.
+func openDB(path string) (*sql.DB, error) {
+	dsn := "file:" + path +
+		"?_pragma=busy_timeout(10000)" +
+		"&_pragma=journal_mode(WAL)" +
+		"&_pragma=synchronous(NORMAL)"
+
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, fmt.Errorf("open database %s: %w", path, err)
+	}
+
+	// A single connection avoids SQLITE_BUSY between this process's
+	// own connections; concurrency lives in the worker pools, not in
+	// parallel database access.
+	db.SetMaxOpenConns(1)
+
+	return db, nil
+}
+
+// openScanDatabase opens the database for the scan subcommand, creating
+// the file, its parent directory, and the schema as needed.
+func openScanDatabase(path string) (*sql.DB, error) {
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
+	if err != nil {
+		return nil, fmt.Errorf("create database directory: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = initSchema(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	return db, nil
+}
+
+// openReportDatabase opens an existing database for the report and
+// trees subcommands. A missing database file is an error directing the
+// user to run scan first; the schema version must match exactly.
+func openReportDatabase(path string) (*sql.DB, error) {
+	_, err := os.Stat(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, fmt.Errorf("%s: %w", path, errNoDatabase)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("database: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	v, err := userVersion(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	if v != schemaVersion {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: version %d, want %d: %w",
+			path, v, schemaVersion, errSchemaVersion)
+	}
+
+	return db, nil
+}
+
+// initSchema creates the schema on a fresh database and verifies the
+// schema version on an existing one.
+func initSchema(db *sql.DB) error {
+	v, err := userVersion(db)
+	if err != nil {
+		return err
+	}
+
+	switch v {
+	case 0:
+		return createSchema(db)
+	case schemaVersion:
+		return nil
+	default:
+		return fmt.Errorf("version %d, want %d: %w",
+			v, schemaVersion, errSchemaVersion)
+	}
+}
+
+// createSchema applies the schema to a fresh database and stamps the
+// schema version.
+func createSchema(db *sql.DB) error {
+	ctx := context.Background()
+
+	_, err := db.ExecContext(ctx, createTableSQL)
+	if err != nil {
+		return fmt.Errorf("create schema: %w", err)
+	}
+
+	_, err = db.ExecContext(ctx,
+		"PRAGMA user_version = "+strconv.Itoa(schemaVersion))
+	if err != nil {
+		return fmt.Errorf("set schema version: %w", err)
+	}
+
+	return nil
+}
+
+// userVersion reads the database's PRAGMA user_version.
+func userVersion(db *sql.DB) (int, error) {
+	var v int
+
+	err := db.QueryRowContext(context.Background(),
+		"PRAGMA user_version").Scan(&v)
+	if err != nil {
+		return 0, fmt.Errorf("read schema version: %w", err)
+	}
+
+	return v, nil
+}
+
+// loadFileRows reads every record from the files table.
+func loadFileRows(db *sql.DB) ([]scanRec, error) {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head, tail FROM files")
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var recs []scanRec
+
+	for rows.Next() {
+		var (
+			path []byte
+			r    scanRec
+		)
+
+		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail)
+		if err != nil {
+			return nil, fmt.Errorf("read record: %w", err)
+		}
+
+		r.path = string(path)
+		recs = append(recs, r)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	return recs, nil
+}
+
+// loadFileMeta streams every record's path, size, mtime, and whether
+// it carries hashes to fn. Scan change detection needs no hash
+// values, and skipping the hash columns keeps the scan's in-memory
+// index small on multi-million-file databases.
+func loadFileMeta(db *sql.DB,
+	fn func(path string, size, mtime int64, hashed bool),
+) error {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head <> '' FROM files")
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var (
+			path        []byte
+			size, mtime int64
+			hashed      int64
+		)
+
+		err = rows.Scan(&path, &size, &mtime, &hashed)
+		if err != nil {
+			return fmt.Errorf("read record: %w", err)
+		}
+
+		fn(string(path), size, mtime, hashed != 0)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	return nil
+}
+
+// updateBatchSize is the number of record changes committed per
+// transaction during the update pass. The filesystem is authoritative
+// and the database an eventually-consistent reflection of it, so
+// scan-level atomicity is not required; smaller transactions keep the
+// WAL small and let concurrent reports observe progress.
+const updateBatchSize = 10000
+
+// applyChanges writes one scan's database changes — upserts for new and
+// changed files, deletes for vanished ones — in batched transactions.
+// Progress is rendered on prog (one increment per change).
+func applyChanges(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	for batch := range slices.Chunk(upserts, updateBatchSize) {
+		err := applyBatch(db, batch, nil, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	for batch := range slices.Chunk(deletes, updateBatchSize) {
+		err := applyBatch(db, nil, batch, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// applyBatch commits one batch of upserts and deletes in a single
+// transaction.
+func applyBatch(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	ctx := context.Background()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+
+	defer func() { _ = tx.Rollback() }()
+
+	err = execUpserts(ctx, tx, upserts, prog)
+	if err != nil {
+		return err
+	}
+
+	err = execDeletes(ctx, tx, deletes, prog)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("commit: %w", err)
+	}
+
+	return nil
+}
+
+// execUpserts inserts or updates one record per new or changed file.
+func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, upsertSQL)
+	if err != nil {
+		return fmt.Errorf("prepare upsert: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, r := range upserts {
+		_, err = st.ExecContext(ctx,
+			[]byte(r.path), r.size, r.mtime, r.head, r.tail)
+		if err != nil {
+			return fmt.Errorf("upsert %s: %w", r.path, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}
+
+// execDeletes removes the records for paths no longer present.
+func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, "DELETE FROM files WHERE path = ?")
+	if err != nil {
+		return fmt.Errorf("prepare delete: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, p := range deletes {
+		_, err = st.ExecContext(ctx, []byte(p))
+		if err != nil {
+			return fmt.Errorf("delete %s: %w", p, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}

db_test.go

@@ -0,0 +1,224 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// testDBPath returns a database path inside a fresh temp dir.
+func testDBPath(t *testing.T) string {
+	t.Helper()
+
+	return filepath.Join(t.TempDir(), "db.sqlite")
+}
+
+// openTestDB creates a fresh scan database in a temp dir.
+func openTestDB(t *testing.T) *sql.DB {
+	t.Helper()
+
+	db, err := openScanDatabase(testDBPath(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() { _ = db.Close() })
+
+	return db
+}
+
+func TestDatabasePath(t *testing.T) {
+	t.Setenv(databaseEnv, "")
+
+	if got := databasePath(); got != defaultDatabasePath {
+		t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
+	}
+
+	t.Setenv(databaseEnv, "/custom/place.sqlite")
+
+	if got := databasePath(); got != "/custom/place.sqlite" {
+		t.Errorf("databasePath() = %q, want the env override", got)
+	}
+}
+
+func TestOpenScanDatabaseCreates(t *testing.T) {
+	t.Parallel()
+
+	// The parent directory does not exist yet; scan must create it.
+	path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("openScanDatabase: %v", err)
+	}
+
+	v, err := userVersion(db)
+	if err != nil || v != schemaVersion {
+		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
+	}
+
+	_ = db.Close()
+
+	// Reopening an existing database must succeed and find the schema.
+	db, err = openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+
+	defer func() { _ = db.Close() }()
+
+	recs, err := loadFileRows(db)
+	if err != nil || len(recs) != 0 {
+		t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err)
+	}
+}
+
+func TestOpenReportDatabaseMissing(t *testing.T) {
+	t.Parallel()
+
+	_, err := openReportDatabase(testDBPath(t))
+	if !errors.Is(err, errNoDatabase) {
+		t.Fatalf("err = %v, want errNoDatabase", err)
+	}
+}
+
+func TestOpenReportDatabaseVersionMismatch(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	_, err = openReportDatabase(path)
+	if !errors.Is(err, errSchemaVersion) {
+		t.Fatalf("err = %v, want errSchemaVersion", err)
+	}
+}
+
+func TestOpenReportDatabaseOK(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	db, err = openReportDatabase(path)
+	if err != nil {
+		t.Fatalf("openReportDatabase: %v", err)
+	}
+
+	_ = db.Close()
+}
+
+func TestApplyChangesRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// Paths may contain tabs and newlines; the database must store
+	// them byte-exactly.
+	recs := []scanRec{
+		{size: 2, mtime: 20, head: "h2", tail: "t2", path: "/a/tab\tnew\nline"},
+		{size: 1, mtime: 10, head: "h1", tail: "t1", path: "/a/x"},
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	slices.SortFunc(got, func(a, b scanRec) int {
+		return strings.Compare(a.path, b.path)
+	})
+
+	if !slices.Equal(got, recs) {
+		t.Fatalf("rows = %+v, want %+v", got, recs)
+	}
+
+	// An upsert for an existing path updates in place; a delete
+	// removes exactly its path.
+	upd := scanRec{size: 3, mtime: 30, head: "h3", tail: "t3", path: "/a/x"}
+
+	err = applyChanges(db, []scanRec{upd},
+		[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(got) != 1 || got[0] != upd {
+		t.Fatalf("rows = %+v, want just %+v", got, upd)
+	}
+}
+
+func TestApplyChangesBatching(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// One more change than the batch size, so the update spans two
+	// transactions.
+	n := updateBatchSize + 1
+
+	recs := make([]scanRec, 0, n)
+	for i := range n {
+		recs = append(recs, scanRec{
+			size: int64(i), mtime: 1, head: "h", tail: "t",
+			path: fmt.Sprintf("/batch/%07d", i),
+		})
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", int64(n)))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil || len(got) != n {
+		t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n)
+	}
+
+	deletes := make([]string, 0, n)
+	for _, r := range recs {
+		deletes = append(deletes, r.path)
+	}
+
+	err = applyChanges(db, nil, deletes, newProgress("update", int64(n)))
+	if err != nil {
+		t.Fatalf("applyChanges deletes: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil || len(got) != 0 {
+		t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err)
+	}
+}

go.mod

@@ -5,13 +5,22 @@ go 1.25.7
 require (
 	github.com/schollz/progressbar/v3 v3.19.1
 	github.com/spf13/cobra v1.10.2
+	modernc.org/sqlite v1.54.0
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.22 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/term v0.44.0 // indirect
+	modernc.org/libc v1.74.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -3,14 +3,28 @@ github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
+github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -23,10 +37,44 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
 golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
+modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog=
+modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

main.go

@@ -2,13 +2,15 @@
 // very large filesystems without reading full file contents. Files are
 // considered duplicates when they have identical size, identical SHA-256
 // of their first 1024 bytes, and identical SHA-256 of their last 1024
-// bytes.
+// bytes. scan maintains a persistent SQLite database of file signatures
+// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
+// reporting subcommands read.
 //
 // Usage:
 //
-//	sfdupes scan [-root /srv] [-workers N] > files.dat
+//	sfdupes scan [--workers N] [-x] PATH...
-//	sfdupes report [files.dat|-] > dupes.tsv
+//	sfdupes report > dupes.tsv
-//	sfdupes trees [files.dat|-] > dupetrees.tsv
+//	sfdupes trees > dupetrees.tsv
 //
 // See README.md for the complete specification.
 package main
@@ -16,19 +18,35 @@ package main
 import (
 	"fmt"
 	"os"
+	"runtime"
 
 	"github.com/spf13/cobra"
 )
 
+// Exit codes: 0 is success (even with per-file warnings), exitFatal is
+// a fatal error, exitUsage is a usage error.
+const (
+	exitFatal = 1
+	exitUsage = 2
+)
+
+// Version is the build version, injected at link time via -ldflags
+// (see the Makefile); "dev" for a plain go build.
+//
+//nolint:gochecknoglobals // written only by the linker
+var Version = "dev"
+
 func main() {
 	root := &cobra.Command{
 		Use:     "sfdupes",
 		Short:   "Find candidate duplicate files by size and head/tail SHA-256",
+		Version: Version,
 		Args:    cobra.NoArgs,
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(cmd *cobra.Command, _ []string) {
 			// A missing subcommand prints usage and exits 2.
 			_ = cmd.Usage()
-			os.Exit(2)
+
+			os.Exit(exitUsage)
 		},
 	}
 	// Everything on stdout is machine-readable data; all human-facing
@@ -37,45 +55,54 @@ func main() {
 	root.SetErr(os.Stderr)
 	root.CompletionOptions.DisableDefaultCmd = true
 
+	var (
+		scanWorkers int
+		scanOneFS   bool
+	)
+
 	scanCmd := &cobra.Command{
-		Use:   "scan [-root /srv] [-workers N]",
+		Use:   "scan [--workers N] [-x] PATH...",
-		Short: "Walk a tree and emit one record per regular file on stdout",
+		Short: "Walk trees and synchronize the scan database",
-		// The README specifies single-dash flags (-root, -workers);
+		Args:  cobra.MinimumNArgs(1),
-		// parse them with the stdlib flag package inside runScan.
+		Run: func(_ *cobra.Command, args []string) {
-		DisableFlagParsing: true,
+			runScan(args, scanWorkers, scanOneFS)
-		Run: func(cmd *cobra.Command, args []string) {
-			runScan(args)
 		},
 	}
+	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
+		"concurrent workers for the walk and hash phases")
+	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
+		"do not cross filesystem boundaries")
 
 	reportCmd := &cobra.Command{
-		Use:   "report [files.dat|-]",
+		Use:   "report",
-		Short: "Read a scan stream and print the file-level duplicates report",
+		Short: "Read the scan database and print the file-level duplicates report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runReport(args)
+			runReport()
 		},
 	}
 
 	treesCmd := &cobra.Command{
-		Use:   "trees [files.dat|-]",
+		Use:   "trees",
-		Short: "Read a scan stream and print the duplicate-tree report",
+		Short: "Read the scan database and print the duplicate-tree report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runTrees(args)
+			runTrees()
 		},
 	}
 
 	root.AddCommand(scanCmd, reportCmd, treesCmd)
-	if err := root.Execute(); err != nil {
+
+	err := root.Execute()
+	if err != nil {
 		// Cobra has already printed the error and usage to stderr;
 		// an invalid subcommand or bad arguments is a usage error.
-		os.Exit(2)
+		os.Exit(exitUsage)
 	}
 }
 
 // fatalf reports a fatal error and exits 1.
 func fatalf(format string, args ...any) {
 	fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
-	os.Exit(1)
+	os.Exit(exitFatal)
 }

progress.go

@@ -12,12 +12,23 @@ import (
 // is not a TTY.
 const plainInterval = 5 * time.Second
 
+// barThrottle caps how often the TTY progress bar redraws.
+const barThrottle = 100 * time.Millisecond
+
+// walkSpinnerType selects the progressbar library's spinner style used
+// for the walk pass, which has no known total.
+const walkSpinnerType = 14
+
+// percentScale converts a fraction to a percentage.
+const percentScale = 100
+
 // stderrIsTTY reports whether stderr is attached to a terminal.
 func stderrIsTTY() bool {
 	fi, err := os.Stderr.Stat()
 	if err != nil {
 		return false
 	}
+
 	return fi.Mode()&os.ModeCharDevice != 0
 }
 
@@ -27,7 +38,10 @@ func stderrIsTTY() bool {
 // stderr is not a TTY it emits no ANSI redraws: it prints a plain
 // one-line update no more often than every plainInterval.
 //
-// All methods must be called from the main goroutine only.
+// All methods must be called from the main goroutine only. A nil
+// *progress is a valid no-display receiver: every method is a no-op,
+// so batched database flushes during the streaming pass can reuse the
+// update-pass helpers without rendering anything.
 type progress struct {
 	label string
 	total int64 // -1 when unknown (walk pass)
@@ -42,6 +56,7 @@ func newProgress(label string, total int64) *progress {
 	if !stderrIsTTY() {
 		return p
 	}
+
 	opts := []progressbar.Option{
 		progressbar.OptionSetWriter(os.Stderr),
 		progressbar.OptionSetDescription(label),
@@ -49,7 +64,7 @@ func newProgress(label string, total int64) *progress {
 		progressbar.OptionShowIts(),
 		progressbar.OptionSetItsString("files"),
 		progressbar.OptionSetElapsedTime(true),
-		progressbar.OptionThrottle(100 * time.Millisecond),
+		progressbar.OptionThrottle(barThrottle),
 	}
 	if total >= 0 {
 		opts = append(opts,
@@ -59,20 +74,28 @@ func newProgress(label string, total int64) *progress {
 	} else {
 		opts = append(opts,
 			progressbar.OptionSetPredictTime(false),
-			progressbar.OptionSpinnerType(14),
+			progressbar.OptionSpinnerType(walkSpinnerType),
 		)
 	}
+
 	p.bar = progressbar.NewOptions64(total, opts...)
+
 	return p
 }
 
 // increment records one completed item and refreshes the display.
 func (p *progress) increment() {
+	if p == nil {
+		return
+	}
+
 	p.count++
 	if p.bar != nil {
 		_ = p.bar.Add(1)
+
 		return
 	}
+
 	if time.Since(p.last) >= plainInterval {
 		p.last = time.Now()
 		fmt.Fprintln(os.Stderr, p.plainLine())
@@ -81,19 +104,31 @@ func (p *progress) increment() {
 
 // warnf prints a one-line warning to stderr without corrupting the bar.
 func (p *progress) warnf(format string, args ...any) {
+	if p == nil {
+		return
+	}
+
 	if p.bar != nil {
 		_ = p.bar.Clear()
 	}
+
 	fmt.Fprintf(os.Stderr, format+"\n", args...)
 }
 
 // finish terminates the pass's display.
 func (p *progress) finish() {
-	if p.bar != nil {
+	if p == nil {
-		_ = p.bar.Finish()
-		fmt.Fprintln(os.Stderr)
 		return
 	}
+
+	if p.bar != nil {
+		_ = p.bar.Finish()
+
+		fmt.Fprintln(os.Stderr)
+
+		return
+	}
+
 	fmt.Fprintln(os.Stderr, p.plainLine())
 }
 
@@ -104,19 +139,24 @@ func (p *progress) plainLine() string {
 		return fmt.Sprintf("%s: %d files, elapsed %s",
 			p.label, p.count, elapsed.Round(time.Second))
 	}
-	pct := 100.0
+
+	pct := float64(percentScale)
 	if p.total > 0 {
-		pct = 100 * float64(p.count) / float64(p.total)
+		pct = percentScale * float64(p.count) / float64(p.total)
 	}
+
 	rate := 0.0
 	if s := elapsed.Seconds(); s > 0 {
 		rate = float64(p.count) / s
 	}
+
 	eta := "?"
+
 	if rate > 0 && p.count <= p.total {
 		remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
 		eta = remaining.Round(time.Second).String()
 	}
+
 	return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
 		p.label, p.count, p.total, pct, rate,
 		elapsed.Round(time.Second), eta)

report.go

@@ -2,71 +2,49 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"slices"
-	"strconv"
 	"strings"
 )
 
-// scanRec is one well-formed record parsed from a scan stream. The
+// ioBufSize is the buffer size for the buffered stdout writers.
-// signature (size, head, tail) is the duplicate key; mtime is
+const ioBufSize = 1 << 20
-// informational and not retained.
+
+// minGroupSize is the smallest number of members that makes a
+// duplicate group.
+const minGroupSize = 2
+
+// scanRec is one file record from the database. The signature (size,
+// head, tail) is the duplicate key; mtime is informational only and
+// used by scan for change detection.
 type scanRec struct {
 	size  int64
+	mtime int64
 	head  string
 	tail  string
 	path  string
 }
 
-// openScanInput resolves the analysis-mode input: the file named by the
+// loadRecords opens the database and reads every file record for the
-// single optional positional argument, or stdin when it is absent or
+// report and trees subcommands. Any database problem — including a
-// "-". The returned closer must be called when reading is done.
+// missing database — is fatal.
-func openScanInput(args []string) (in io.Reader, name string, closer func()) {
+func loadRecords() []scanRec {
-	if len(args) == 1 && args[0] != "-" {
+	dbPath := databasePath()
-		f, err := os.Open(args[0])
-		if err != nil {
-			fatalf("open %s: %v", args[0], err)
-		}
-		return f, args[0], func() { f.Close() }
-	}
-	return os.Stdin, "stdin", func() {}
-}
 
-// parseScanStream reads every NUL-terminated record from in. A record
+	db, err := openReportDatabase(dbPath)
-// that does not have exactly 5 fields or whose size is non-numeric is
-// counted as malformed and skipped. Total records read is
-// len(recs) + malformed.
-func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
-	sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
-	sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
-	sc.Split(splitNUL)
-	for sc.Scan() {
-		// The path is the last field and may itself contain tabs,
-		// so split into at most 5 fields.
-		fields := strings.SplitN(sc.Text(), "\t", 5)
-		if len(fields) != 5 {
-			malformed++
-			continue
-		}
-		size, err := strconv.ParseInt(fields[0], 10, 64)
 	if err != nil {
-			malformed++
+		fatalf("%v", err)
-			continue
 	}
-		recs = append(recs, scanRec{
+
-			size: size,
+	defer func() { _ = db.Close() }()
-			head: fields[2],
+
-			tail: fields[3],
+	recs, err := loadFileRows(db)
-			path: fields[4],
+	if err != nil {
-		})
+		fatalf("database %s: %v", dbPath, err)
 	}
-	if err := sc.Err(); err != nil {
+
-		fatalf("read %s: %v", name, err)
+	return recs
-	}
-	return recs, malformed
 }
 
 // dupeGroup is one set of candidate-duplicate files: identical size,
@@ -77,93 +55,92 @@ type dupeGroup struct {
 	paths []string
 }
 
-// runReport implements the report subcommand: it reads a scan stream
+// runReport implements the report subcommand: it reads every record
-// from the named file (or stdin when absent or "-") and prints the
+// from the database and prints the file-level duplicates report as TSV
-// file-level duplicates report as TSV on stdout. It never touches the
+// on stdout. It never touches the scanned filesystem; its only I/O is
-// scanned filesystem; its only I/O is the scan input, stdout, and
+// the database, stdout, and stderr.
-// stderr. args holds the positional arguments already validated by
+func runReport() {
-// cobra (at most one).
+	recs := loadRecords()
-func runReport(args []string) {
+	dupes := collectDupeGroups(recs)
-	in, name, closer := openScanInput(args)
-	defer closer()
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 
-	type dupeKey struct {
+	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
-		size int64
+
-		head string
+	_, err := fmt.Fprintln(out, "first\tdupe\tsize")
-		tail string
+	if err != nil {
+		fatalf("write stdout: %v", err)
 	}
-	groups := make(map[dupeKey][]string)
+
+	dupeFiles := 0
+
+	var reclaimable int64
+
+	for _, g := range dupes {
+		for _, p := range g.paths[1:] {
+			_, err = fmt.Fprintf(out, "%s\t%s\t%d\n",
+				g.paths[0], p, g.size)
+			if err != nil {
+				fatalf("write stdout: %v", err)
+			}
+
+			dupeFiles++
+			reclaimable += g.size
+		}
+	}
+
+	err = out.Flush()
+	if err != nil {
+		fatalf("write stdout: %v", err)
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"report: %d records read, %d duplicate groups, %d dupe files, "+
+			"%s reclaimable\n",
+		len(recs), len(dupes), dupeFiles, humanBytes(reclaimable))
+}
+
+// collectDupeGroups groups records by signature and returns every group
+// with two or more paths, each group's paths sorted lexicographically,
+// groups ordered by size descending then by first path ascending.
+func collectDupeGroups(recs []scanRec) []dupeGroup {
+	groups := make(map[fileSig][]string)
+
 	for _, r := range recs {
-		k := dupeKey{size: r.size, head: r.head, tail: r.tail}
+		// A record without hashes (its size was unique when last
+		// scanned) has unknown content and is never reported as a
+		// duplicate.
+		if r.head == "" {
+			continue
+		}
+
+		k := fileSig{size: r.size, head: r.head, tail: r.tail}
 		groups[k] = append(groups[k], r.path)
 	}
 
 	var dupes []dupeGroup
+
 	for k, paths := range groups {
-		if len(paths) < 2 {
+		if len(paths) < minGroupSize {
 			continue
 		}
+
 		slices.Sort(paths)
 		dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
 	}
+
 	// Biggest reclaimable space first; ties broken by first path.
 	slices.SortFunc(dupes, func(a, b dupeGroup) int {
 		if a.size != b.size {
 			if a.size > b.size {
 				return -1
 			}
+
 			return 1
 		}
+
 		return strings.Compare(a.paths[0], b.paths[0])
 	})
 
-	out := bufio.NewWriterSize(os.Stdout, 1<<20)
+	return dupes
-	if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
-		fatalf("write stdout: %v", err)
-	}
-	dupeFiles := 0
-	var reclaimable int64
-	for _, g := range dupes {
-		for _, p := range g.paths[1:] {
-			if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
-				g.paths[0], p, g.size); err != nil {
-				fatalf("write stdout: %v", err)
-			}
-			dupeFiles++
-			reclaimable += g.size
-		}
-	}
-	if err := out.Flush(); err != nil {
-		fatalf("write stdout: %v", err)
-	}
-
-	fmt.Fprintf(os.Stderr,
-		"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
-		records, malformedNote(malformed), len(dupes), dupeFiles,
-		humanBytes(reclaimable))
-}
-
-// malformedNote formats the optional malformed-record note for the
-// stderr summaries.
-func malformedNote(malformed int) string {
-	if malformed == 0 {
-		return ""
-	}
-	return fmt.Sprintf(" (%d malformed, skipped)", malformed)
-}
-
-// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
-// data without a terminator at EOF is returned as a final record.
-func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
-	if i := bytes.IndexByte(data, 0); i >= 0 {
-		return i + 1, data[:i], nil
-	}
-	if atEOF && len(data) > 0 {
-		return len(data), data, nil
-	}
-	return 0, nil, nil
 }
 
 // humanBytes formats a byte count in human units (binary prefixes).
@@ -172,10 +149,12 @@ func humanBytes(n int64) string {
 	if n < unit {
 		return fmt.Sprintf("%d B", n)
 	}
+
 	div, exp := int64(unit), 0
 	for m := n / unit; m >= unit; m /= unit {
 		div *= unit
 		exp++
 	}
+
 	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
 }

report_test.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"slices"
+	"testing"
+)
+
+func TestCollectDupeGroups(t *testing.T) {
+	t.Parallel()
+
+	recs := []scanRec{
+		{size: 100, head: "h", tail: "t", path: "/z/b"},
+		{size: 100, head: "h", tail: "t", path: "/z/a"},
+		{size: 100, head: "h", tail: "t", path: "/z/c"},
+		{size: 4000, head: "H", tail: "T", path: "/big/2"},
+		{size: 4000, head: "H", tail: "T", path: "/big/1"},
+		// Same size as the /z group but a different head hash.
+		{size: 100, head: "other", tail: "t", path: "/z/d"},
+		// A singleton signature must not form a group.
+		{size: 7, head: "u", tail: "u", path: "/lonely"},
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 2 {
+		t.Fatalf("len(groups) = %d, want 2", len(groups))
+	}
+
+	if groups[0].size != 4000 ||
+		!slices.Equal(groups[0].paths, []string{"/big/1", "/big/2"}) {
+		t.Errorf("groups[0] = %+v, want size 4000, paths /big/1 /big/2",
+			groups[0])
+	}
+
+	if groups[1].size != 100 ||
+		!slices.Equal(groups[1].paths, []string{"/z/a", "/z/b", "/z/c"}) {
+		t.Errorf("groups[1] = %+v, want size 100, paths /z/a /z/b /z/c",
+			groups[1])
+	}
+}
+
+func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
+	t.Parallel()
+
+	// mtime is informational only; records differing only in mtime
+	// still group together.
+	recs := []scanRec{
+		{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
+		{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 {
+		t.Fatalf("len(groups) = %d, want 1", len(groups))
+	}
+}
+
+func TestCollectDupeGroupsTieBreak(t *testing.T) {
+	t.Parallel()
+
+	recs := []scanRec{
+		{size: 50, head: "b", tail: "b", path: "/beta/2"},
+		{size: 50, head: "b", tail: "b", path: "/beta/1"},
+		{size: 50, head: "a", tail: "a", path: "/alpha/2"},
+		{size: 50, head: "a", tail: "a", path: "/alpha/1"},
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 2 {
+		t.Fatalf("len(groups) = %d, want 2", len(groups))
+	}
+
+	// Equal sizes: ordered by first path ascending.
+	if groups[0].paths[0] != "/alpha/1" || groups[1].paths[0] != "/beta/1" {
+		t.Fatalf("tie-break order wrong: %q then %q",
+			groups[0].paths[0], groups[1].paths[0])
+	}
+}
+
+func TestCollectDupeGroupsDeterministic(t *testing.T) {
+	t.Parallel()
+
+	recs := []scanRec{
+		{size: 1, head: "a", tail: "a", path: "/p/1"},
+		{size: 1, head: "a", tail: "a", path: "/p/2"},
+		{size: 2, head: "b", tail: "b", path: "/q/1"},
+		{size: 2, head: "b", tail: "b", path: "/q/2"},
+	}
+
+	forward := collectDupeGroups(recs)
+
+	reversed := slices.Clone(recs)
+	slices.Reverse(reversed)
+
+	backward := collectDupeGroups(reversed)
+	if !slices.EqualFunc(forward, backward, func(a, b dupeGroup) bool {
+		return a.size == b.size && slices.Equal(a.paths, b.paths)
+	}) {
+		t.Fatalf("output depends on record order: %+v vs %+v",
+			forward, backward)
+	}
+}
+
+func TestHumanBytes(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		n    int64
+		want string
+	}{
+		{0, "0 B"},
+		{1, "1 B"},
+		{1023, "1023 B"},
+		{1024, "1.0 KiB"},
+		{1536, "1.5 KiB"},
+		{1 << 20, "1.0 MiB"},
+		{5 << 30, "5.0 GiB"},
+		{1 << 40, "1.0 TiB"},
+		{1 << 50, "1.0 PiB"},
+		{1 << 60, "1.0 EiB"},
+	}
+	for _, c := range cases {
+		if got := humanBytes(c.n); got != c.want {
+			t.Errorf("humanBytes(%d) = %q, want %q", c.n, got, c.want)
+		}
+	}
+}

scan.go

@@ -1,231 +1,765 @@
 package main
 
 import (
-	"bufio"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
-	"flag"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
-	"runtime"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
 )
 
 // chunk is the number of bytes hashed from each end of a file.
 const chunk = 1024
 
-// fileRec carries one file between the stat and hash passes.
+// workQueueDepth bounds the job and result channels feeding the walk
+// and hash worker pools.
+const workQueueDepth = 1024
+
+// fileRec carries one statted file between the scan phases.
 type fileRec struct {
 	path  string
 	size  int64
 	mtime int64
 }
 
-// runScan implements the scan subcommand: three sequential passes (walk,
+// fileMeta is the in-memory index entry for one existing database
-// stat, hash) over the tree under -root, emitting one NUL-terminated
+// record: just enough for change detection, plus whether the record
-// record per regular file on stdout.
+// carries hashes. Hashes stay on disk; at tens of millions of records
-func runScan(args []string) {
+// they would dominate the scan's memory.
-	fl := flag.NewFlagSet("scan", flag.ExitOnError)
+type fileMeta struct {
-	root := fl.String("root", "/srv", "directory tree to scan")
+	size   int64
-	workers := fl.Int("workers", runtime.NumCPU(),
+	mtime  int64
-		"concurrent workers for the stat and hash passes")
+	hashed bool
-	fl.Parse(args)
-	if fl.NArg() > 0 {
-		fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
-		os.Exit(2)
-	}
-	if *workers < 1 {
-		*workers = 1
-	}
-
-	st, err := os.Stat(*root)
-	if err != nil {
-		fatalf("root %s: %v", *root, err)
-	}
-	if !st.IsDir() {
-		fatalf("root %s: not a directory", *root)
-	}
-
-	paths, walkErrs := walkPass(*root)
-	recs, statErrs := statPass(paths, *workers)
-	emitted, hashErrs := hashPass(recs, *workers)
-
-	skipped := walkErrs + statErrs + hashErrs
-	fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n",
-		emitted, skipped)
 }
 
-// walkPass enumerates every regular file under root. It never follows
+// runScan implements the scan subcommand: three sequential phases —
-// symlinks, never descends into directories named .zfs, and warns and
+// walk (which stats each file as it is discovered), hash, update —
-// continues on any per-path error.
+// that synchronize the persistent database with the filesystem state
-func walkPass(root string) (paths []string, errs int) {
+// under the PATH operands. Only files whose size at least one other
-	prog := newProgress("walk", -1)
+// file shares are ever hashed: a size-unique file cannot be a
-	walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+// duplicate. Flag parsing and the at-least-one-operand check are done
+// by cobra.
+func runScan(roots []string, workers int, oneFS bool) {
+	if workers < 1 {
+		workers = 1
+	}
+
+	roots = resolveRoots(roots)
+	dbPath := databasePath()
+
+	db, err := openScanDatabase(dbPath)
 	if err != nil {
-			errs++
+		fatalf("%v", err)
-			prog.warnf("walk %s: %v", p, err)
-			if d != nil && d.IsDir() {
-				return filepath.SkipDir
 	}
-			return nil
+
+	defer func() { _ = db.Close() }()
+
+	st, err := syncScan(db, roots, workers, oneFS)
+	if err != nil {
+		fatalf("update database %s: %v", dbPath, err)
 	}
-		if d.IsDir() {
+
-			// ZFS snapshot pseudo-dirs would list every file
+	fmt.Fprintf(os.Stderr,
-			// once per snapshot; never descend.
+		"scan: %d files seen (%d added, %d updated, %d removed, "+
-			if d.Name() == ".zfs" {
+			"%d unchanged), %d skipped\n",
-				return filepath.SkipDir
+		st.added+st.updated+st.unchanged, st.added, st.updated,
+		st.removed, st.unchanged, st.skipped)
+}
+
+// resolveRoots converts each PATH operand to an absolute, lexically
+// cleaned path (symlinks are not resolved) and verifies that it
+// exists. Database records are keyed by absolute path, so scan results
+// must not depend on the working directory.
+func resolveRoots(roots []string) []string {
+	abs := make([]string, 0, len(roots))
+
+	for _, root := range roots {
+		a, err := filepath.Abs(root)
+		if err != nil {
+			fatalf("resolve %s: %v", root, err)
 		}
-			return nil
+
+		// A nonexistent operand is a fatal error before any scanning.
+		_, err = os.Lstat(a)
+		if err != nil {
+			fatalf("%v", err)
 		}
-		// Regular files only: skip symlinks, sockets, FIFOs, and
+
-		// device nodes.
+		abs = append(abs, a)
-		if !d.Type().IsRegular() {
-			return nil
 	}
-		paths = append(paths, p)
+
-		prog.increment()
+	return abs
-		return nil
+}
+
+// pruneRoots drops operands already covered by another operand:
+// duplicates and any operand lying under another one. Every remaining
+// file is then reachable through exactly one root, so walked paths are
+// unique without keeping a scan-wide set of every path seen.
+func pruneRoots(roots []string) []string {
+	// Shorter-first ordering guarantees an ancestor is kept before any
+	// operand under it is considered.
+	sorted := slices.Clone(roots)
+	slices.SortFunc(sorted, func(a, b string) int {
+		if len(a) != len(b) {
+			return len(a) - len(b)
+		}
+
+		return strings.Compare(a, b)
 	})
-	prog.finish()
+
-	if walkErr != nil {
+	kept := make([]string, 0, len(sorted))
-		fatalf("walk %s: %v", root, walkErr)
+
+	for _, r := range sorted {
+		if !underAnyRoot(r, kept) {
+			kept = append(kept, r)
 		}
-	return paths, errs
+	}
+
+	return kept
 }
 
-// statPass lstats every collected path in a worker pool, recording size
+// scanStats summarizes one scan's database synchronization for the
-// and mtime. Paths that fail to stat (or are no longer regular files)
+// final stderr summary.
-// are warned about and dropped.
+type scanStats struct {
-func statPass(paths []string, workers int) (recs []fileRec, errs int) {
+	added     int
-	type result struct {
+	updated   int
-		rec fileRec
+	removed   int
-		err error
+	unchanged int
+	skipped   int
+}
+
+// scanState carries one scan's evolving state across its phases.
+// Entries consumed from existing mark files verified this run;
+// whatever remains after the walk and hash phases is deleted by the
+// update phase.
+type scanState struct {
+	db        *sql.DB
+	existing  map[string]fileMeta
+	sizes     []int64   // size census: every walked file, plus records outside the roots
+	toHash    []fileRec // files whose size is shared: must be read
+	sentinels []fileRec // new/changed size-unique files: recorded without hashes
+	batch     []scanRec
+	st        scanStats
+}
+
+// syncScan synchronizes the database with the filesystem under roots
+// in three sequential phases: walk (enumerate and stat every file,
+// building a complete size census), hash (read only the new or
+// changed — or previously unhashed — files whose size at least one
+// other file shares, committing results in batches as they arrive),
+// and update (record the size-unique files without reading them, and
+// delete the records the scan no longer verifies). Records outside
+// the roots are never touched.
+func syncScan(db *sql.DB, roots []string, workers int,
+	oneFS bool,
+) (scanStats, error) {
+	roots = pruneRoots(roots)
+
+	s := &scanState{db: db}
+
+	err := s.loadIndex(roots)
+	if err != nil {
+		return s.st, err
 	}
-	jobs := make(chan string, 1024)
+
-	results := make(chan result, 1024)
+	changed, unhashed := s.walkPhase(startWalk(roots, oneFS, workers))
-	for range workers {
+
-		go func() {
+	s.partition(changed, unhashed)
-			for p := range jobs {
+
-				fi, err := os.Lstat(p)
+	err = s.hashPhase(workers)
+	if err != nil {
+		return s.st, err
+	}
+
+	return s.st, s.updatePhase()
+}
+
+// loadIndex indexes the database records under the scan roots for
+// change detection and collects the sizes of every record outside
+// them: out-of-scope records join the size census so a scanned file
+// can be recognized as a possible duplicate of a tree scanned
+// separately into the same database.
+func (s *scanState) loadIndex(roots []string) error {
+	s.existing = make(map[string]fileMeta)
+
+	return loadFileMeta(s.db,
+		func(path string, size, mtime int64, hashed bool) {
+			if underAnyRoot(path, roots) {
+				s.existing[path] = fileMeta{
+					size: size, mtime: mtime, hashed: hashed,
+				}
+
+				return
+			}
+
+			s.sizes = append(s.sizes, size)
+		})
+}
+
+// walkPhase drains the walk, appending every walked file's size to
+// the census and resolving what it can immediately: an unchanged file
+// whose record already has hashes needs nothing further. It returns
+// the new-or-changed files and the unchanged files whose records lack
+// hashes; both remain candidates until the census decides whether
+// their sizes are shared.
+func (s *scanState) walkPhase(
+	events <-chan walkEvent,
+) ([]fileRec, []fileRec) {
+	prog := newProgress("walk", -1)
+
+	var changed, unhashed []fileRec
+
+	for ev := range events {
+		if ev.fail {
+			s.st.skipped++
+
+			prog.warnf("%s", ev.warn)
+
+			continue
+		}
+
+		s.sizes = append(s.sizes, ev.rec.size)
+
+		prog.increment()
+
+		old, ok := s.existing[ev.rec.path]
+
 		switch {
-				case err != nil:
+		case !ok || old.size != ev.rec.size || old.mtime < ev.rec.mtime:
-					results <- result{rec: fileRec{path: p}, err: err}
+			changed = append(changed, ev.rec)
-				case !fi.Mode().IsRegular():
+		case old.hashed:
-					results <- result{
+			delete(s.existing, ev.rec.path)
-						rec: fileRec{path: p},
+
-						err: fmt.Errorf("no longer a regular file"),
+			s.st.unchanged++
-					}
 		default:
-					results <- result{rec: fileRec{
+			unhashed = append(unhashed, ev.rec)
-						path:  p,
-						size:  fi.Size(),
-						mtime: fi.ModTime().Unix(),
-					}}
 		}
 	}
-		}()
+
+	prog.finish()
+
+	return changed, unhashed
+}
+
+// partition decides each candidate's disposition now that the size
+// census is complete. A file whose size no other file shares cannot
+// be a duplicate and is never read: a new or changed one is recorded
+// without hashes, an unchanged unhashed one keeps its record. Every
+// file with a shared size queues for the hash phase.
+func (s *scanState) partition(changed, unhashed []fileRec) {
+	slices.Sort(s.sizes)
+
+	for _, rec := range changed {
+		if s.sizeShared(rec.size) {
+			s.toHash = append(s.toHash, rec)
+
+			continue
 		}
+
+		s.sentinels = append(s.sentinels, rec)
+		s.resolve(rec.path)
+	}
+
+	for _, rec := range unhashed {
+		if s.sizeShared(rec.size) {
+			s.toHash = append(s.toHash, rec)
+
+			continue
+		}
+
+		delete(s.existing, rec.path)
+
+		s.st.unchanged++
+	}
+
+	s.sizes = nil
+}
+
+// sizeShared reports whether at least two census entries have this
+// size. Every candidate's own size is in the census exactly once, so
+// a second entry means another file (or a record outside the scan
+// roots) could share its content.
+func (s *scanState) sizeShared(size int64) bool {
+	i, found := slices.BinarySearch(s.sizes, size)
+
+	return found && i+1 < len(s.sizes) && s.sizes[i+1] == size
+}
+
+// resolve counts one written record as added or updated and marks its
+// path verified.
+func (s *scanState) resolve(path string) {
+	if _, ok := s.existing[path]; ok {
+		s.st.updated++
+
+		delete(s.existing, path)
+
+		return
+	}
+
+	s.st.added++
+}
+
+// hashPhase hashes every queued file with the worker pool, committing
+// completed records to the database in batches as results arrive, so
+// a long scan persists its progress as it goes (an interrupted scan
+// resumes cheaply: the next run skips everything already recorded).
+// The total is exact, so the bar shows a real ETA. Files that fail to
+// hash are warned about and skipped; their stale records, if any, are
+// deleted by the update phase.
+func (s *scanState) hashPhase(workers int) error {
+	jobs := make(chan fileRec, workQueueDepth)
+	results := make(chan hashResult, workQueueDepth)
+
+	startHashWorkers(jobs, results, workers)
+
+	// The feeder ranges over its own reference: s.toHash is released
+	// below while the feeder may still be running.
+	toHash := s.toHash
+	s.toHash = nil
+
 	go func() {
-		for _, p := range paths {
+		for _, rec := range toHash {
-			jobs <- p
+			jobs <- rec
 		}
+
 		close(jobs)
 	}()
 
-	prog := newProgress("stat", int64(len(paths)))
+	prog := newProgress("hash", int64(len(toHash)))
-	recs = make([]fileRec, 0, len(paths))
+	defer prog.finish()
-	for range paths {
+
+	for range toHash {
 		r := <-results
-		if r.err != nil {
+
-			errs++
-			prog.warnf("stat %s: %v", r.rec.path, r.err)
-		} else {
-			recs = append(recs, r.rec)
-		}
 		prog.increment()
+
+		if r.err != nil {
+			s.st.skipped++
+
+			prog.warnf("hash %s: %v", r.rec.path, r.err)
+
+			continue
 		}
-	prog.finish()
+
-	return recs, errs
+		s.resolve(r.rec.path)
+
+		s.batch = append(s.batch, scanRec{
+			size:  r.rec.size,
+			mtime: r.rec.mtime,
+			head:  r.head,
+			tail:  r.tail,
+			path:  r.rec.path,
+		})
+
+		if len(s.batch) < updateBatchSize {
+			continue
+		}
+
+		err := applyBatch(s.db, s.batch, nil, nil)
+		if err != nil {
+			return err
+		}
+
+		s.batch = s.batch[:0]
+	}
+
+	return nil
 }
 
-// hashPass hashes the first and last chunk bytes of every file in a
+// updatePhase writes the scan's tail under one progress display: the
-// worker pool and emits the output records on stdout from the main
+// final partial batch of hashed records, a hash-less record for every
-// goroutine. Files that fail to open or read are warned about and
+// size-unique new or changed file, and deletions for every record the
-// dropped.
+// scan did not verify (vanished files, plus paths that failed to stat
-func hashPass(recs []fileRec, workers int) (emitted, errs int) {
+// or hash).
-	type result struct {
+func (s *scanState) updatePhase() error {
+	deletes := make([]string, 0, len(s.existing))
+	for path := range s.existing {
+		deletes = append(deletes, path)
+	}
+
+	// Sorted deletes keep the update phase deterministic.
+	slices.Sort(deletes)
+
+	s.st.removed = len(deletes)
+
+	total := len(s.batch) + len(s.sentinels) + len(deletes)
+	prog := newProgress("update", int64(total))
+
+	defer prog.finish()
+
+	err := applyChanges(s.db, s.batch, nil, prog)
+	if err != nil {
+		return err
+	}
+
+	s.batch = nil
+
+	// Sentinel records are converted in batch-sized chunks rather than
+	// materialized all at once; a first scan can have millions.
+	for chunk := range slices.Chunk(s.sentinels, updateBatchSize) {
+		recs := make([]scanRec, 0, len(chunk))
+		for _, rec := range chunk {
+			recs = append(recs, scanRec{
+				size: rec.size, mtime: rec.mtime, path: rec.path,
+			})
+		}
+
+		err = applyBatch(s.db, recs, nil, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	return applyChanges(s.db, nil, deletes, prog)
+}
+
+// underAnyRoot reports whether path is any of the roots or lies under
+// one of them.
+func underAnyRoot(path string, roots []string) bool {
+	for _, root := range roots {
+		if underRoot(path, root) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// underRoot reports whether path is root itself or lies under it. Both
+// must be absolute and lexically clean.
+func underRoot(path, root string) bool {
+	if path == root {
+		return true
+	}
+
+	prefix := root
+	if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+
+	return strings.HasPrefix(path, prefix)
+}
+
+// dirJob is one directory awaiting traversal by the walk workers. It
+// carries its operand's filesystem device so -x can stop at
+// filesystem boundaries.
+type dirJob struct {
+	path      string
+	rootDev   uint64
+	rootDevOK bool
+}
+
+// walkEvent is one walk result delivered to the walk phase: a regular
+// file's statted record, or a warning when fail is set.
+type walkEvent struct {
+	rec  fileRec
+	warn string
+	fail bool
+}
+
+// startWalk seeds every root into the shared walk worker pool and
+// returns the event stream: one record per regular file, one warning
+// event per per-path error. The channel is closed when the walk
+// completes.
+func startWalk(roots []string, oneFS bool, workers int) <-chan walkEvent {
+	jobs, subdirs, events := startWalkWorkers(workers, oneFS)
+
+	go func() {
+		initial := make([]dirJob, 0, len(roots))
+		for _, root := range roots {
+			initial = append(initial, seedRoot(root, events)...)
+		}
+
+		dispatchDirs(initial, jobs, subdirs)
+	}()
+
+	return events
+}
+
+// seedRoot turns one PATH operand into the walk's starting state: a
+// regular-file operand is statted and emitted directly, a directory
+// operand becomes an initial job, and a symlink or other non-regular
+// operand yields nothing (symlinks are never followed, including as
+// operands).
+func seedRoot(root string, events chan<- walkEvent) []dirJob {
+	fi, err := os.Lstat(root)
+	if err != nil {
+		events <- walkEvent{
+			warn: fmt.Sprintf("walk %s: %v", root, err),
+			fail: true,
+		}
+
+		return nil
+	}
+
+	switch {
+	case fi.IsDir():
+		if filepath.Base(root) == ".zfs" {
+			return nil
+		}
+
+		dev, ok := deviceOfInfo(fi)
+
+		return []dirJob{{path: root, rootDev: dev, rootDevOK: ok}}
+	case fi.Mode().IsRegular():
+		events <- walkEvent{rec: fileRec{
+			path:  root,
+			size:  fi.Size(),
+			mtime: fi.ModTime().Unix(),
+		}}
+
+		return nil
+	default:
+		return nil
+	}
+}
+
+// startWalkWorkers starts the walk worker pool. Each worker processes
+// one directory at a time, emitting an event per regular file and
+// handing discovered subdirectories back to the dispatcher; events is
+// closed once every worker has finished.
+func startWalkWorkers(workers int,
+	oneFS bool,
+) (chan dirJob, chan []dirJob, chan walkEvent) {
+	jobs := make(chan dirJob, workQueueDepth)
+	subdirs := make(chan []dirJob, workers)
+	events := make(chan walkEvent, workQueueDepth)
+
+	var wg sync.WaitGroup
+
+	for range workers {
+		wg.Go(func() {
+			for job := range jobs {
+				subdirs <- walkOneDir(job, oneFS, events)
+			}
+		})
+	}
+
+	go func() {
+		wg.Wait()
+		close(events)
+	}()
+
+	return jobs, subdirs, events
+}
+
+// dispatchDirs feeds directory jobs to the walk workers, queueing
+// newly discovered subdirectories (newest first, which keeps the
+// frontier small) until every directory has been processed, then
+// closes jobs.
+func dispatchDirs(initial []dirJob, jobs chan<- dirJob,
+	subdirs <-chan []dirJob,
+) {
+	go func() {
+		queue := slices.Clone(initial)
+		pending := len(queue)
+
+		for pending > 0 {
+			var (
+				out  chan<- dirJob
+				next dirJob
+			)
+
+			if len(queue) > 0 {
+				out = jobs
+				next = queue[len(queue)-1]
+			}
+
+			select {
+			case out <- next:
+				queue = queue[:len(queue)-1]
+			case subs := <-subdirs:
+				pending += len(subs) - 1
+				queue = append(queue, subs...)
+			}
+		}
+
+		close(jobs)
+	}()
+}
+
+// walkOneDir reads one directory, emitting an event per regular-file
+// entry and a warning event per unreadable one, and returns the
+// subdirectories to descend into.
+func walkOneDir(job dirJob, oneFS bool, events chan<- walkEvent) []dirJob {
+	entries, err := os.ReadDir(job.path)
+	if err != nil {
+		events <- walkEvent{
+			warn: fmt.Sprintf("walk %s: %v", job.path, err),
+			fail: true,
+		}
+
+		return nil
+	}
+
+	var subs []dirJob
+
+	for _, e := range entries {
+		p := filepath.Join(job.path, e.Name())
+
+		if e.IsDir() {
+			if sub, ok := subdirJob(p, e, job, oneFS, events); ok {
+				subs = append(subs, sub)
+			}
+
+			continue
+		}
+		// Regular files only: skip symlinks, sockets, FIFOs, and
+		// device nodes.
+		if !e.Type().IsRegular() {
+			continue
+		}
+
+		emitFile(p, e, events)
+	}
+
+	return subs
+}
+
+// emitFile stats one regular directory entry and emits its record.
+// The lstat happens here in the walk worker, while the directory's
+// metadata is still hot; a path that fails to stat (or stops being a
+// regular file) between the directory read and the lstat is warned
+// about and skipped.
+func emitFile(p string, e fs.DirEntry, events chan<- walkEvent) {
+	info, err := e.Info()
+	if err != nil {
+		events <- walkEvent{
+			warn: fmt.Sprintf("stat %s: %v", p, err),
+			fail: true,
+		}
+
+		return
+	}
+
+	if !info.Mode().IsRegular() {
+		return
+	}
+
+	events <- walkEvent{rec: fileRec{
+		path:  p,
+		size:  info.Size(),
+		mtime: info.ModTime().Unix(),
+	}}
+}
+
+// subdirJob applies the descent rules to directory p: never enter
+// .zfs (ZFS snapshot pseudo-dirs would list every file once per
+// snapshot), and with -x never enter a directory on a different
+// filesystem than its operand.
+func subdirJob(p string, e fs.DirEntry, parent dirJob, oneFS bool,
+	events chan<- walkEvent,
+) (dirJob, bool) {
+	if e.Name() == ".zfs" {
+		return dirJob{}, false
+	}
+
+	job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
+	if !oneFS || !parent.rootDevOK {
+		return job, true
+	}
+
+	info, err := e.Info()
+	if err != nil {
+		events <- walkEvent{
+			warn: fmt.Sprintf("walk %s: %v", p, err),
+			fail: true,
+		}
+
+		return dirJob{}, false
+	}
+
+	if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
+		return dirJob{}, false
+	}
+
+	return job, true
+}
+
+// deviceOfInfo extracts the filesystem device ID from a FileInfo, when
+// the platform exposes one.
+func deviceOfInfo(fi fs.FileInfo) (uint64, bool) {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, false
+	}
+
+	return statDev(st), true
+}
+
+// hashResult carries one file's head/tail hashes (or the error that
+// prevented hashing it) from the hash workers to the hash phase.
+type hashResult struct {
 	rec  fileRec
 	head string
 	tail string
 	err  error
-	}
+}
-	jobs := make(chan fileRec, 1024)
+
-	results := make(chan result, 1024)
+// startHashWorkers starts the hash worker pool: workers read jobs,
+// write one result per record, and exit when jobs is closed.
+func startHashWorkers(jobs <-chan fileRec, results chan<- hashResult,
+	workers int,
+) {
 	for range workers {
 		go func() {
 			for rec := range jobs {
 				head, tail, err := hashHeadTail(rec.path, rec.size)
-				results <- result{rec: rec, head: head, tail: tail, err: err}
+				results <- hashResult{
+					rec: rec, head: head, tail: tail, err: err,
+				}
 			}
 		}()
 	}
-	go func() {
-		for _, rec := range recs {
-			jobs <- rec
-		}
-		close(jobs)
-	}()
-
-	prog := newProgress("hash", int64(len(recs)))
-	out := bufio.NewWriterSize(os.Stdout, 1<<20)
-	for range recs {
-		r := <-results
-		if r.err != nil {
-			errs++
-			prog.warnf("hash %s: %v", r.rec.path, r.err)
-			prog.increment()
-			continue
-		}
-		if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
-			r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil {
-			fatalf("write stdout: %v", err)
-		}
-		emitted++
-		prog.increment()
-	}
-	prog.finish()
-	if err := out.Flush(); err != nil {
-		fatalf("write stdout: %v", err)
-	}
-	return emitted, errs
 }
 
 // hashHeadTail returns the lowercase-hex SHA-256 of the first
 // min(chunk, size) bytes and of the last min(chunk, size) bytes of the
 // file at path. The two reads overlap when size < 2*chunk; for
 // size == 0 both hashes are of the empty input. size is the value
-// recorded by the stat pass.
+// recorded when the file was statted.
-func hashHeadTail(path string, size int64) (head, tail string, err error) {
+func hashHeadTail(path string, size int64) (string, string, error) {
+	//nolint:gosec // hashing operator-supplied paths is the tool's purpose
 	f, err := os.Open(path)
 	if err != nil {
 		return "", "", err
 	}
-	defer f.Close()
+
+	defer func() { _ = f.Close() }()
 
 	n := min(int64(chunk), size)
+
 	buf := make([]byte, n)
 	if n > 0 {
-		if _, err := f.ReadAt(buf, 0); err != nil {
+		_, err = f.ReadAt(buf, 0)
+		if err != nil {
 			return "", "", err
 		}
 	}
+
 	h := sha256.Sum256(buf)
-	if n > 0 {
+
-		if _, err := f.ReadAt(buf, size-n); err != nil {
+	// When the whole file fits in one chunk the tail window is exactly
+	// the bytes just read: reuse the head hash instead of issuing a
+	// second read for every small file.
+	if size <= int64(chunk) {
+		hh := hex.EncodeToString(h[:])
+
+		return hh, hh, nil
+	}
+
+	_, err = f.ReadAt(buf, size-n)
+	if err != nil {
 		return "", "", err
 	}
-	}
+
 	t := sha256.Sum256(buf)
+
 	return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
 }

scan_dev_darwin.go

@@ -0,0 +1,14 @@
+//go:build darwin
+
+package main
+
+import "syscall"
+
+// statDev returns the filesystem device ID of st as uint64. Device IDs
+// are opaque; the conversion only needs to be consistent so equality
+// comparisons work, not numerically meaningful.
+//
+//nolint:gosec // int32 device IDs convert consistently; only equality matters
+func statDev(st *syscall.Stat_t) uint64 {
+	return uint64(st.Dev)
+}

scan_dev_other.go

@@ -0,0 +1,10 @@
+//go:build !darwin
+
+package main
+
+import "syscall"
+
+// statDev returns the filesystem device ID of st.
+func statDev(st *syscall.Stat_t) uint64 {
+	return st.Dev
+}

scan_test.go

@@ -0,0 +1,809 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+// writeFile creates a file with the given content and returns its path.
+func writeFile(t *testing.T, dir, name string, data []byte) string {
+	t.Helper()
+
+	p := filepath.Join(dir, name)
+
+	err := os.MkdirAll(filepath.Dir(p), 0o750)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.WriteFile(p, data, 0o600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return p
+}
+
+// hexSum returns the lowercase-hex SHA-256 of data.
+func hexSum(data []byte) string {
+	s := sha256.Sum256(data)
+
+	return hex.EncodeToString(s[:])
+}
+
+// pattern returns n bytes of deterministic content seeded by tag.
+func pattern(tag byte, n int) []byte {
+	data := make([]byte, n)
+	for i := range data {
+		data[i] = tag ^ byte(i)
+	}
+
+	return data
+}
+
+func TestHashHeadTail(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	cases := []struct {
+		name string
+		data []byte
+	}{
+		{"empty", nil},
+		{"one-byte", []byte("x")},
+		{"under-one-chunk", pattern(1, chunk-1)},
+		{"exactly-one-chunk", pattern(2, chunk)},
+		{"overlapping-reads", pattern(3, chunk+chunk/2)},
+		{"exactly-two-chunks", pattern(4, 2*chunk)},
+		{"beyond-two-chunks", pattern(5, 3*chunk)},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			t.Parallel()
+
+			p := writeFile(t, dir, c.name, c.data)
+
+			head, tail, err := hashHeadTail(p, int64(len(c.data)))
+			if err != nil {
+				t.Fatalf("hashHeadTail: %v", err)
+			}
+
+			n := min(chunk, len(c.data))
+			if want := hexSum(c.data[:n]); head != want {
+				t.Errorf("head = %s, want %s", head, want)
+			}
+
+			if want := hexSum(c.data[len(c.data)-n:]); tail != want {
+				t.Errorf("tail = %s, want %s", tail, want)
+			}
+		})
+	}
+}
+
+func TestHashHeadTailErrors(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	_, _, err := hashHeadTail(filepath.Join(dir, "missing"), 1)
+	if err == nil {
+		t.Error("no error for a missing file")
+	}
+
+	// A file that shrank between the stat and hash passes: reading at
+	// the stat-reported size must fail rather than emit wrong hashes.
+	p := writeFile(t, dir, "shrunk", []byte("tiny"))
+
+	_, _, err = hashHeadTail(p, int64(2*chunk))
+	if err == nil {
+		t.Error("no error when the stat size exceeds the file size")
+	}
+}
+
+// collectWalk runs a walk over roots and returns the emitted records
+// and the number of warning events.
+func collectWalk(t *testing.T, roots []string, oneFS bool,
+	workers int,
+) ([]fileRec, int) {
+	t.Helper()
+
+	var (
+		recs []fileRec
+		errs int
+	)
+
+	for ev := range startWalk(roots, oneFS, workers) {
+		if ev.fail {
+			errs++
+
+			continue
+		}
+
+		recs = append(recs, ev.rec)
+	}
+
+	return recs, errs
+}
+
+// walkedPaths returns the sorted paths of the walked records.
+func walkedPaths(recs []fileRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+func TestWalk(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	want := []string{
+		writeFile(t, dir, "a.txt", []byte("a")),
+		writeFile(t, dir, "sub/b.txt", []byte("bb")),
+		writeFile(t, dir, "sub/deeper/c.txt", []byte("ccc")),
+	}
+
+	slices.Sort(want)
+
+	// Files under a .zfs directory must never be walked.
+	writeFile(t, dir, ".zfs/snapshot/hourly/a.txt", []byte("a"))
+
+	// Symlinks are skipped, not followed.
+	err := os.Symlink(want[0], filepath.Join(dir, "link"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	recs, errs := collectWalk(t, []string{dir}, false, 4)
+	if errs != 0 {
+		t.Fatalf("errs = %d, want 0", errs)
+	}
+
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+
+	// The walk stats each file as it is discovered: every record must
+	// carry the real size and a plausible mtime.
+	for _, r := range recs {
+		if r.size < 1 || r.size > 3 {
+			t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
+		}
+
+		if r.mtime <= 0 {
+			t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
+		}
+	}
+}
+
+func TestWalkDeepAndWide(t *testing.T) {
+	t.Parallel()
+
+	// Exercise the dispatcher with more directories than workers and
+	// with nesting deeper than the worker count.
+	dir := t.TempDir()
+	deep := "deep" + strings.Repeat("/d", 30)
+
+	want := make([]string, 0, 41)
+	want = append(want, writeFile(t, dir, deep+"/f", []byte("x")))
+
+	for i := range 40 {
+		want = append(want, writeFile(t, dir,
+			fmt.Sprintf("wide/%02d/f", i), []byte("y")))
+	}
+
+	slices.Sort(want)
+
+	recs, errs := collectWalk(t, []string{dir}, false, 8)
+	if errs != 0 {
+		t.Fatalf("errs = %d, want 0", errs)
+	}
+
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("walked %d paths, want %d", len(got), len(want))
+	}
+}
+
+func TestWalkMultipleRoots(t *testing.T) {
+	t.Parallel()
+
+	rootA := t.TempDir()
+	rootB := t.TempDir()
+	want := []string{
+		writeFile(t, rootA, "a1", []byte("1")),
+		writeFile(t, rootA, "sub/a2", []byte("2")),
+		writeFile(t, rootB, "b1", []byte("3")),
+	}
+
+	slices.Sort(want)
+
+	// Operands are enumerated concurrently by the shared pool; order
+	// is unspecified.
+	recs, errs := collectWalk(t, []string{rootA, rootB}, false, 4)
+	if errs != 0 {
+		t.Fatalf("errs = %d, want 0", errs)
+	}
+
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+}
+
+func TestWalkFileAndSymlinkOperands(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	f := writeFile(t, dir, "plain", []byte("data"))
+
+	link := filepath.Join(dir, "link")
+
+	err := os.Symlink(f, link)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A regular-file operand is emitted as itself, statted.
+	recs, errs := collectWalk(t, []string{f}, false, 2)
+	if errs != 0 || len(recs) != 1 || recs[0].path != f || recs[0].size != 4 {
+		t.Fatalf("file operand: recs = %+v, errs = %d", recs, errs)
+	}
+
+	// A symlink operand is not followed and yields nothing.
+	recs, errs = collectWalk(t, []string{link}, false, 2)
+	if errs != 0 || len(recs) != 0 {
+		t.Fatalf("symlink operand: recs = %+v, errs = %d", recs, errs)
+	}
+}
+
+func TestWalkOneFilesystemSameFS(t *testing.T) {
+	t.Parallel()
+
+	// Everything in one filesystem: -x must not skip anything.
+	dir := t.TempDir()
+	want := []string{
+		writeFile(t, dir, "a", []byte("a")),
+		writeFile(t, dir, "sub/deep/b", []byte("b")),
+	}
+
+	recs, errs := collectWalk(t, []string{dir}, true, 4)
+	if errs != 0 {
+		t.Fatalf("errs = %d, want 0", errs)
+	}
+
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+}
+
+func TestDeviceOfInfo(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	fi1, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fi2, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev1, ok1 := deviceOfInfo(fi1)
+
+	dev2, ok2 := deviceOfInfo(fi2)
+	if !ok1 || !ok2 || dev1 != dev2 {
+		t.Fatalf("deviceOfInfo unstable: %d/%v vs %d/%v",
+			dev1, ok1, dev2, ok2)
+	}
+}
+
+// buildSmokeTree recreates the README smoke-test filesystem layout
+// with deterministic content and returns the tree root.
+func buildSmokeTree(t *testing.T) string {
+	t.Helper()
+
+	dir := t.TempDir()
+	one := pattern(10, 2000)
+	f1 := pattern(30, 3000)
+	f2 := pattern(40, 100)
+
+	writeFile(t, dir, "a/one.bin", one)
+	writeFile(t, dir, "b/copy.bin", one)
+	writeFile(t, dir, "b/copy2.bin", one)
+	// Same size as one.bin, different content.
+	writeFile(t, dir, "a/unique.bin", pattern(20, 2000))
+	writeFile(t, dir, "tiny1", []byte("x"))
+	writeFile(t, dir, "tiny2", []byte("x"))
+	writeFile(t, dir, "tiny3", []byte("y"))
+	writeFile(t, dir, "empty1", nil)
+	writeFile(t, dir, "empty2", nil)
+	writeFile(t, dir, "t1/f1", f1)
+	writeFile(t, dir, "t1/sub/f2", f2)
+	writeFile(t, dir, "t2/f1", f1)
+	writeFile(t, dir, "t2/sub/f2", f2)
+	writeFile(t, dir, "t3/f1", f1)
+	writeFile(t, dir, "t3/sub/f2renamed", f2)
+
+	return dir
+}
+
+// smokeTreeFiles is the number of regular files buildSmokeTree creates.
+const smokeTreeFiles = 15
+
+// syncTree synchronizes the database with the given roots and returns
+// the scan stats.
+func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats {
+	t.Helper()
+
+	st, err := syncScan(db, roots, 4, false)
+	if err != nil {
+		t.Fatalf("syncScan: %v", err)
+	}
+
+	return st
+}
+
+// dbRecords returns every record currently in the database.
+func dbRecords(t *testing.T, db *sql.DB) []scanRec {
+	t.Helper()
+
+	recs, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return recs
+}
+
+// recordByPath finds the record with the given path.
+func recordByPath(t *testing.T, recs []scanRec, path string) scanRec {
+	t.Helper()
+
+	for _, r := range recs {
+		if r.path == path {
+			return r
+		}
+	}
+
+	t.Fatalf("no record for %q", path)
+
+	return scanRec{}
+}
+
+// recordPaths returns the sorted paths of recs.
+func recordPaths(recs []scanRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+// assertSmokeDupeGroups checks the file-level duplicate groups for the
+// smoke tree rooted at dir.
+func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
+
+	groups := collectDupeGroups(parsed)
+	if len(groups) != 5 {
+		t.Fatalf("len(groups) = %d, want 5", len(groups))
+	}
+
+	wantSizes := []int64{3000, 2000, 100, 1, 0}
+	for i, g := range groups {
+		if g.size != wantSizes[i] {
+			t.Errorf("groups[%d].size = %d, want %d",
+				i, g.size, wantSizes[i])
+		}
+	}
+
+	wantF1 := []string{
+		filepath.Join(dir, "t1/f1"),
+		filepath.Join(dir, "t2/f1"),
+		filepath.Join(dir, "t3/f1"),
+	}
+	if !slices.Equal(groups[0].paths, wantF1) {
+		t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
+	}
+}
+
+// assertSmokeTreeGroups checks the duplicate-tree groups for the smoke
+// tree rooted at dir.
+func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
+
+	super, dirs := buildHierarchy(parsed)
+	super.compute()
+
+	tg := collectTreeGroups(dirs, super)
+	if len(tg) != 1 {
+		t.Fatalf("len(tree groups) = %d, want 1", len(tg))
+	}
+
+	wantTrees := []string{filepath.Join(dir, "t1"), filepath.Join(dir, "t2")}
+	if got := groupPaths(tg)[0]; !slices.Equal(got, wantTrees) {
+		t.Fatalf("tree group = %q, want %q", got, wantTrees)
+	}
+
+	if tg[0][0].fileCount != 2 || tg[0][0].totalSize != 3100 {
+		t.Fatalf("tree totals: %d files %d bytes, want 2 3100",
+			tg[0][0].fileCount, tg[0][0].totalSize)
+	}
+}
+
+func TestScanPipeline(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: smokeTreeFiles}) {
+		t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles)
+	}
+
+	parsed := dbRecords(t, db)
+	if len(parsed) != smokeTreeFiles {
+		t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles)
+	}
+
+	assertSmokeDupeGroups(t, dir, parsed)
+	assertSmokeTreeGroups(t, dir, parsed)
+}
+
+func TestSyncScanUnchangedReuse(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("first scan stats = %+v, want 2 added", st)
+	}
+
+	// An immediate rescan reuses every record without reading file
+	// contents. Prove the files are not re-read by corrupting a stored
+	// hash and observing that it survives the rescan.
+	_, err := db.ExecContext(context.Background(),
+		"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{unchanged: 2}) {
+		t.Fatalf("rescan stats = %+v, want 2 unchanged", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
+		t.Fatalf("head = %q, want sentinel (file must not be re-read)",
+			r.head)
+	}
+}
+
+func TestSyncScanMtimeBump(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	syncTree(t, db, dir)
+
+	// Bump the mtime forward: the file must be re-hashed even though
+	// its size is unchanged.
+	future := time.Now().Add(time.Hour)
+
+	err := os.Chtimes(a, future, future)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{updated: 1}) {
+		t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
+		t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
+	}
+}
+
+func TestSyncScanAddRemove(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+	syncTree(t, db, dir)
+
+	// Add one file, remove another.
+	c := writeFile(t, dir, "c.bin", pattern(3, 700))
+
+	err := os.Remove(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 1, removed: 1, unchanged: 1}) {
+		t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged",
+			st)
+	}
+
+	want := []string{filepath.Join(dir, "b.bin"), c}
+	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+}
+
+func TestSyncScanSizeChange(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 100))
+
+	syncTree(t, db, dir)
+
+	// Rewrite with a different size but force the mtime back to the
+	// recorded value: the size mismatch alone must trigger a re-hash.
+	old := recordByPath(t, dbRecords(t, db), p)
+
+	writeFile(t, dir, "f", pattern(1, 200))
+
+	mt := time.Unix(old.mtime, 0)
+
+	err := os.Chtimes(p, mt, mt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.updated != 1 {
+		t.Fatalf("stats = %+v, want 1 updated", st)
+	}
+
+	if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 {
+		t.Fatalf("size = %d, want 200", got.size)
+	}
+}
+
+func TestSyncScanScope(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "a/keep", pattern(1, 10))
+
+	gone := writeFile(t, dir, "b/gone", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Deleting a file outside the rescanned root must not remove its
+	// record: records outside the scanned operands are untouched.
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, filepath.Join(dir, "a"))
+	if st.removed != 0 || st.unchanged != 1 {
+		t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 2 {
+		t.Fatalf("records = %q, want both retained", got)
+	}
+
+	// Rescanning the parent now removes the vanished file's record.
+	st = syncTree(t, db, dir)
+	if st.removed != 1 {
+		t.Fatalf("parent stats = %+v, want 1 removed", st)
+	}
+}
+
+func TestSyncScanRemovesNonRegular(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 10))
+	keep := writeFile(t, dir, "g", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Replace the file with a symlink: it is no longer walked, so its
+	// record must be deleted.
+	err := os.Remove(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Symlink(keep, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.removed != 1 || st.unchanged != 1 {
+		t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st)
+	}
+}
+
+func TestSyncScanOverlappingRoots(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	// A file reachable via two overlapping operands is deduplicated
+	// by path in the shared walk and processed once.
+	st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
+	if st != (scanStats{added: 1}) {
+		t.Fatalf("stats = %+v, want 1 added", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 1 {
+		t.Fatalf("records = %q, want exactly one", got)
+	}
+}
+
+func TestScanSkipsUniqueSizes(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	// Neither size is shared, so neither file is read: both records
+	// are written without hashes and no duplicates are reported.
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("stats = %+v, want 2 added", st)
+	}
+
+	recs := dbRecords(t, db)
+	for _, r := range recs {
+		if r.head != "" || r.tail != "" {
+			t.Errorf("%s: head = %q tail = %q, want unhashed",
+				r.path, r.head, r.tail)
+		}
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 0 {
+		t.Fatalf("groups = %+v, want none from unhashed records", groups)
+	}
+
+	// A new same-size file makes 500 a shared size: the next scan
+	// hashes both the new file and the previously unhashed unchanged
+	// one, and they group as duplicates.
+	c := writeFile(t, dir, "c.bin", pattern(1, 500))
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{added: 1, updated: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 added 1 updated 1 unchanged",
+			st)
+	}
+
+	groups := collectDupeGroups(dbRecords(t, db))
+	if len(groups) != 1 {
+		t.Fatalf("groups = %+v, want the a/c pair", groups)
+	}
+
+	if want := []string{a, c}; !slices.Equal(groups[0].paths, want) {
+		t.Fatalf("group paths = %q, want %q", groups[0].paths, want)
+	}
+}
+
+func TestTreesUnhashedNeverEqual(t *testing.T) {
+	t.Parallel()
+
+	// Two trees identical except for unhashed same-name, same-size
+	// files (possible when the trees were scanned separately) must not
+	// compare equal: unhashed content is unknown.
+	shared := pattern(1, 100)
+	recs := []scanRec{
+		{path: "/x/t1/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
+		{path: "/x/t2/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
+		{path: "/x/t1/u", size: 50},
+		{path: "/x/t2/u", size: 50},
+	}
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
+		t.Fatalf("tree groups = %d, want 0 (unhashed files differ)",
+			len(tg))
+	}
+}
+
+func TestPruneRoots(t *testing.T) {
+	t.Parallel()
+
+	// Duplicates and operands under other operands are dropped; /cc is
+	// not under /c (sibling with a shared prefix).
+	got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
+
+	want := []string{"/a", "/c", "/cc"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("pruneRoots = %q, want %q", got, want)
+	}
+}
+
+func TestReportsNeverTouchFilesystem(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	syncTree(t, db, dir)
+
+	// Remove the scanned tree entirely; the analysis must be
+	// unaffected because it reads the database alone.
+	err := os.RemoveAll(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	recs := dbRecords(t, db)
+
+	assertSmokeDupeGroups(t, dir, recs)
+	assertSmokeTreeGroups(t, dir, recs)
+}
+
+func TestUnderRoot(t *testing.T) {
+	t.Parallel()
+
+	const abRoot = "/a/b"
+
+	cases := []struct {
+		path string
+		root string
+		want bool
+	}{
+		{"/a/b/c", abRoot, true},
+		{abRoot, abRoot, true},
+		{"/a/bc", abRoot, false},
+		{"/a", abRoot, false},
+		{"/x/y", "/", true},
+		{"/", "/", true},
+	}
+	for _, c := range cases {
+		if got := underRoot(c.path, c.root); got != c.want {
+			t.Errorf("underRoot(%q, %q) = %v, want %v",
+				c.path, c.root, got, c.want)
+		}
+	}
+}

trees.go

@@ -28,26 +28,69 @@ type treeNode struct {
 	totalSize int64
 }
 
-// runTrees implements the trees subcommand: it reads a scan stream from
+// runTrees implements the trees subcommand: it reads every record from
-// the named file (or stdin when absent or "-"), reconstructs the
+// the database, reconstructs the directory hierarchy from the record
-// directory hierarchy from the record paths, computes a Merkle-style
+// paths, computes a Merkle-style digest per directory, and prints
-// digest per directory, and prints maximal duplicate-tree groups as TSV
+// maximal duplicate-tree groups as TSV on stdout. It never touches the
-// on stdout. It never touches the scanned filesystem; its only I/O is
+// scanned filesystem; its only I/O is the database, stdout, and
-// the scan input, stdout, and stderr. args holds the positional
+// stderr.
-// arguments already validated by cobra (at most one).
+func runTrees() {
-func runTrees(args []string) {
+	recs := loadRecords()
-	in, name, closer := openScanInput(args)
-	defer closer()
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 
-	// Build the hierarchy under a synthetic super-root. Paths are
+	super, allDirs := buildHierarchy(recs)
-	// split on "/"; for absolute paths the first component is empty,
+	super.compute()
-	// which simply becomes a top-level node representing "/".
+
+	dupes := collectTreeGroups(allDirs, super)
+
+	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
+
+	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
+	if err != nil {
+		fatalf("write stdout: %v", err)
+	}
+
+	dupeTrees := 0
+
+	var reclaimable int64
+
+	for _, g := range dupes {
+		first := g[0]
+		for _, n := range g[1:] {
+			_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
+				first.path, n.path, first.fileCount, first.totalSize)
+			if err != nil {
+				fatalf("write stdout: %v", err)
+			}
+
+			dupeTrees++
+			reclaimable += first.totalSize
+		}
+	}
+
+	err = out.Flush()
+	if err != nil {
+		fatalf("write stdout: %v", err)
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
+			"%s reclaimable\n",
+		len(recs), len(dupes), dupeTrees, humanBytes(reclaimable))
+}
+
+// buildHierarchy reconstructs the directory hierarchy from the record
+// paths under a synthetic super-root. Paths are split on "/"; for
+// absolute paths the first component is empty, which simply becomes a
+// top-level node representing "/". It returns the super-root and every
+// directory node created.
+func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
 	super := &treeNode{}
+
 	var allDirs []*treeNode
+
 	for _, r := range recs {
 		comps := strings.Split(r.path, "/")
+
 		node := super
 		for _, c := range comps[:len(comps)-1] {
 			child := node.dirs[c]
@@ -56,76 +99,76 @@ func runTrees(args []string) {
 				if node != super {
 					childPath = node.path + "/" + c
 				}
+
 				child = &treeNode{path: childPath, parent: node}
 				if node.dirs == nil {
 					node.dirs = make(map[string]*treeNode)
 				}
+
 				node.dirs[c] = child
 				allDirs = append(allDirs, child)
 			}
+
 			node = child
 		}
+
 		if node.files == nil {
 			node.files = make(map[string]fileSig)
 		}
-		node.files[comps[len(comps)-1]] = fileSig{
-			size: r.size, head: r.head, tail: r.tail,
-		}
-	}
-	super.compute()
 
-	// Group directories by digest; two or more dirs sharing a digest
+		sig := fileSig{size: r.size, head: r.head, tail: r.tail}
-	// are candidate duplicate trees.
+
+		// An unhashed record (its size was unique when last scanned)
+		// has unknown content: give it a signature no other file can
+		// share, so trees containing it never compare equal. Real
+		// heads are hex, so the NUL-prefixed form cannot collide.
+		if sig.head == "" {
+			sig.head = "unhashed\x00" + r.path
+		}
+
+		node.files[comps[len(comps)-1]] = sig
+	}
+
+	return super, allDirs
+}
+
+// collectTreeGroups groups directories by digest and returns every
+// maximal group with two or more members, each group's members sorted
+// by path, groups ordered by tree size descending then by first path
+// ascending.
+func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
 	groups := make(map[[sha256.Size]byte][]*treeNode)
 	for _, d := range allDirs {
 		groups[d.digest] = append(groups[d.digest], d)
 	}
+
 	var dupes [][]*treeNode
+
 	for _, g := range groups {
-		if len(g) < 2 || suppressed(g, super) {
+		if len(g) < minGroupSize || suppressed(g, super) {
 			continue
 		}
+
 		slices.SortFunc(g, func(a, b *treeNode) int {
 			return strings.Compare(a.path, b.path)
 		})
 		dupes = append(dupes, g)
 	}
+
 	// Biggest reclaimable space first; ties broken by first path.
 	slices.SortFunc(dupes, func(a, b []*treeNode) int {
 		if a[0].totalSize != b[0].totalSize {
 			if a[0].totalSize > b[0].totalSize {
 				return -1
 			}
+
 			return 1
 		}
+
 		return strings.Compare(a[0].path, b[0].path)
 	})
 
-	out := bufio.NewWriterSize(os.Stdout, 1<<20)
+	return dupes
-	if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
-		fatalf("write stdout: %v", err)
-	}
-	dupeTrees := 0
-	var reclaimable int64
-	for _, g := range dupes {
-		first := g[0]
-		for _, n := range g[1:] {
-			if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
-				first.path, n.path, first.fileCount, first.totalSize); err != nil {
-				fatalf("write stdout: %v", err)
-			}
-			dupeTrees++
-			reclaimable += first.totalSize
-		}
-	}
-	if err := out.Flush(); err != nil {
-		fatalf("write stdout: %v", err)
-	}
-
-	fmt.Fprintf(os.Stderr,
-		"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
-		records, malformedNote(malformed), len(dupes), dupeTrees,
-		humanBytes(reclaimable))
 }
 
 // compute fills in digest, fileCount, and totalSize for n and all of
@@ -143,18 +186,22 @@ func (n *treeNode) compute() {
 		n.fileCount++
 		n.totalSize += sig.size
 	}
+
 	for name, child := range n.dirs {
 		child.compute()
 		entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
 		n.fileCount += child.fileCount
 		n.totalSize += child.totalSize
 	}
+
 	slices.Sort(entries)
+
 	h := sha256.New()
 	for _, e := range entries {
 		h.Write([]byte(e))
 		h.Write([]byte{0})
 	}
+
 	copy(n.digest[:], h.Sum(nil))
 }
 
@@ -165,15 +212,19 @@ func (n *treeNode) compute() {
 // parent) or members whose parents differ are always reported.
 func suppressed(g []*treeNode, super *treeNode) bool {
 	seen := make(map[*treeNode]bool, len(g))
+
 	var parentDigest [sha256.Size]byte
+
 	for i, n := range g {
 		p := n.parent
 		if p == nil || p == super {
 			return false
 		}
+
 		if seen[p] {
 			return false // siblings: not implied by any parent group
 		}
+
 		seen[p] = true
 		if i == 0 {
 			parentDigest = p.digest
@@ -181,5 +232,6 @@ func suppressed(g []*treeNode, super *treeNode) bool {
 			return false
 		}
 	}
+
 	return true
 }

trees_test.go

@@ -0,0 +1,220 @@
+package main
+
+import (
+	"slices"
+	"testing"
+)
+
+// Signature hashes shared by the smoke-test records.
+const (
+	f1Head = "f1h"
+	f1Tail = "f1t"
+	f2Head = "f2h"
+	f2Tail = "f2t"
+)
+
+// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
+// /d/t2 are identical, /d/t3 differs from them only by one filename.
+func smokeTreeRecs() []scanRec {
+	return []scanRec{
+		{size: 3000, head: f1Head, tail: f1Tail, path: "/d/t1/f1"},
+		{size: 100, head: f2Head, tail: f2Tail, path: "/d/t1/sub/f2"},
+		{size: 3000, head: f1Head, tail: f1Tail, path: "/d/t2/f1"},
+		{size: 100, head: f2Head, tail: f2Tail, path: "/d/t2/sub/f2"},
+		{size: 3000, head: f1Head, tail: f1Tail, path: "/d/t3/f1"},
+		{size: 100, head: f2Head, tail: f2Tail, path: "/d/t3/sub/f2renamed"},
+	}
+}
+
+// nodeByPath finds the directory node with the given path.
+func nodeByPath(t *testing.T, dirs []*treeNode, path string) *treeNode {
+	t.Helper()
+
+	for _, d := range dirs {
+		if d.path == path {
+			return d
+		}
+	}
+
+	t.Fatalf("no directory node with path %q", path)
+
+	return nil
+}
+
+// groupPaths flattens tree groups into their member path lists.
+func groupPaths(groups [][]*treeNode) [][]string {
+	out := make([][]string, 0, len(groups))
+	for _, g := range groups {
+		paths := make([]string, 0, len(g))
+		for _, n := range g {
+			paths = append(paths, n.path)
+		}
+
+		out = append(out, paths)
+	}
+
+	return out
+}
+
+func TestBuildHierarchyCounts(t *testing.T) {
+	t.Parallel()
+
+	super, dirs := buildHierarchy(smokeTreeRecs())
+	super.compute()
+
+	d := nodeByPath(t, dirs, "/d")
+	if d.fileCount != 6 || d.totalSize != 9300 {
+		t.Errorf("/d: fileCount %d size %d, want 6 9300",
+			d.fileCount, d.totalSize)
+	}
+
+	t1 := nodeByPath(t, dirs, "/d/t1")
+	if t1.fileCount != 2 || t1.totalSize != 3100 {
+		t.Errorf("/d/t1: fileCount %d size %d, want 2 3100",
+			t1.fileCount, t1.totalSize)
+	}
+
+	sub := nodeByPath(t, dirs, "/d/t1/sub")
+	if sub.fileCount != 1 || sub.totalSize != 100 {
+		t.Errorf("/d/t1/sub: fileCount %d size %d, want 1 100",
+			sub.fileCount, sub.totalSize)
+	}
+}
+
+func TestTreeDigests(t *testing.T) {
+	t.Parallel()
+
+	super, dirs := buildHierarchy(smokeTreeRecs())
+	super.compute()
+
+	t1 := nodeByPath(t, dirs, "/d/t1")
+	t2 := nodeByPath(t, dirs, "/d/t2")
+	t3 := nodeByPath(t, dirs, "/d/t3")
+
+	if t1.digest != t2.digest {
+		t.Error("identical trees /d/t1 and /d/t2 have different digests")
+	}
+
+	// t3 differs only in a filename; names are part of the digest.
+	if t1.digest == t3.digest {
+		t.Error("/d/t3 digest equals /d/t1 despite a renamed file")
+	}
+
+	sub1 := nodeByPath(t, dirs, "/d/t1/sub")
+	sub3 := nodeByPath(t, dirs, "/d/t3/sub")
+
+	if sub1.digest == sub3.digest {
+		t.Error("subdirs with differently-named files share a digest")
+	}
+}
+
+func TestTreeDigestContentSensitivity(t *testing.T) {
+	t.Parallel()
+
+	const sharedTail = "same"
+
+	recs := []scanRec{
+		{size: 10, head: sharedTail, tail: sharedTail, path: "/r/a/f"},
+		{size: 10, head: "DIFF", tail: sharedTail, path: "/r/b/f"},
+	}
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	a := nodeByPath(t, dirs, "/r/a")
+	b := nodeByPath(t, dirs, "/r/b")
+
+	if a.digest == b.digest {
+		t.Error("trees with different file content share a digest")
+	}
+}
+
+func TestCollectTreeGroupsMaximal(t *testing.T) {
+	t.Parallel()
+
+	super, dirs := buildHierarchy(smokeTreeRecs())
+	super.compute()
+
+	groups := collectTreeGroups(dirs, super)
+
+	want := [][]string{{"/d/t1", "/d/t2"}}
+	if got := groupPaths(groups); !slices.EqualFunc(got, want,
+		slices.Equal) {
+		t.Fatalf("groups = %v, want %v", got, want)
+	}
+
+	// The /d/t1/sub vs /d/t2/sub group must be suppressed as implied
+	// by its parents' group; the winning group reports one copy's
+	// recursive totals.
+	if groups[0][0].fileCount != 2 || groups[0][0].totalSize != 3100 {
+		t.Errorf("group totals: %d files %d bytes, want 2 3100",
+			groups[0][0].fileCount, groups[0][0].totalSize)
+	}
+}
+
+func TestCollectTreeGroupsDeterministic(t *testing.T) {
+	t.Parallel()
+
+	recs := smokeTreeRecs()
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	forward := groupPaths(collectTreeGroups(dirs, super))
+
+	reversed := slices.Clone(recs)
+	slices.Reverse(reversed)
+
+	superR, dirsR := buildHierarchy(reversed)
+	superR.compute()
+
+	backward := groupPaths(collectTreeGroups(dirsR, superR))
+	if !slices.EqualFunc(forward, backward, slices.Equal) {
+		t.Fatalf("output depends on record order: %v vs %v",
+			forward, backward)
+	}
+}
+
+func TestCollectTreeGroupsSiblings(t *testing.T) {
+	t.Parallel()
+
+	// Identical sibling dirs share a parent, so their group cannot be
+	// implied by a parent group and must be reported.
+	recs := []scanRec{
+		{size: 10, head: "h", tail: "t", path: "/p/x1/f"},
+		{size: 10, head: "h", tail: "t", path: "/p/x2/f"},
+	}
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	got := groupPaths(collectTreeGroups(dirs, super))
+
+	want := [][]string{{"/p/x1", "/p/x2"}}
+	if !slices.EqualFunc(got, want, slices.Equal) {
+		t.Fatalf("groups = %v, want %v", got, want)
+	}
+}
+
+func TestCollectTreeGroupsDifferingParents(t *testing.T) {
+	t.Parallel()
+
+	// /p/a and /q/b contain identical x subtrees, but /p/a has an
+	// extra file, so the parents' digests differ and the x group must
+	// be reported.
+	recs := []scanRec{
+		{size: 10, head: "h", tail: "t", path: "/p/a/x/f"},
+		{size: 99, head: "e", tail: "e", path: "/p/a/extra"},
+		{size: 10, head: "h", tail: "t", path: "/q/b/x/f"},
+	}
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	got := groupPaths(collectTreeGroups(dirs, super))
+
+	want := [][]string{{"/p/a/x", "/q/b/x"}}
+	if !slices.EqualFunc(got, want, slices.Equal) {
+		t.Fatalf("groups = %v, want %v", got, want)
+	}
+}