Merge branch 'todo-next-step-stale': point TODO.md at the tracker (closes #27)

TODO.md's Next Step still named the scripts-to-rule-them-all conversion,
which landed in 3abeacf on 2026-07-26 without recording itself in
Completed Steps, so the file pointed the next contributor at finished
work. Next Step now names the 1.0.0 milestone on the tracker, the
scaffold gains its missing Completed Steps entry, and Workflow matches
the issue-branch-review-merge process actually in use.

Docs-only change (TODO.md only), so the adversarial review was skipped
per the standing exception. make check green.

.claude/settings.json

@@ -0,0 +1,5 @@
+{
+    "worktree": {
+        "bgIsolation": "none"
+    }
+}

.gitea/workflows/check.yml

@@ -6,4 +6,4 @@ jobs:
         steps:
             # actions/checkout v4.2.2, 2026-02-22
             - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-            - run: docker build .
+            - run: script/cibuild

.gitignore

@@ -28,6 +28,9 @@ node_modules/
 
 # Local scan data
 files.dat
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
 
 # Agent worktrees
 .claude/worktrees/

.golangci.yml

@@ -1,5 +1,9 @@
 version: "2"
 
+# Config schema uses the golangci-lint v2 layout (settings live under
+# linters.settings, not top-level linters-settings) so that the
+# thresholds below are actually applied by golangci-lint >= v2.
+
 run:
   timeout: 5m
   modules-download-mode: readonly
@@ -14,19 +18,17 @@ linters:
     - wsl              # Deprecated, replaced by wsl_v5
     - wrapcheck        # Too verbose for internal packages
     - varnamelen       # Short names like db, id are idiomatic Go
-
+  settings:
-linters-settings:
+    lll:
-  lll:
+      line-length: 88
-    line-length: 88
+    funlen:
-  funlen:
+      lines: 80
-    lines: 80
+      statements: 50
-    statements: 50
+    cyclop:
-  cyclop:
+      max-complexity: 15
-    max-complexity: 15
+    dupl:
-  dupl:
+      threshold: 100
-    threshold: 100
 
 issues:
-  exclude-use-default: false
   max-issues-per-linter: 0
   max-same-issues: 0

Dockerfile

@@ -1,6 +1,6 @@
 # Lint stage — fast feedback on formatting and lint issues
-# golangci/golangci-lint:v2.12.1, 2026-07-23
+# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
-FROM golangci/golangci-lint@sha256:c9843d374ca80ecbac86081ec4dd7fe2bb6187b03224f59a0cc2f80759e1845b AS lint
+FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
@@ -14,6 +14,14 @@ FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c
 
 RUN apk add --no-cache make
 
+# We never build or run as root. Create an unprivileged user and point
+# HOME and the Go caches at its home so go build/test and golangci-lint
+# can write their caches when we drop to it below.
+RUN adduser -D -u 1000 builder
+ENV HOME=/home/builder
+ENV GOPATH=/home/builder/go
+ENV GOCACHE=/home/builder/.cache/go-build
+
 WORKDIR /src
 
 # Reuse the linter binary from the lint stage; the copy also forces
@@ -24,7 +32,14 @@ COPY go.mod go.sum ./
 RUN go mod download
 COPY . .
 
-# Fail the build unless the branch is green.
+# Hand the sources and caches to the unprivileged user, then drop root
+# before running any checks or builds.
+RUN chown -R builder:builder /src /home/builder
+USER builder
+
+# Fail the build unless the branch is green. Runs as non-root so the
+# permission-denied test paths are exercised legitimately (root would
+# bypass the chmod(0) the tests rely on).
 RUN make check
 
 RUN make build

Makefile

@@ -6,43 +6,44 @@ BINARY := sfdupes
 VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 LDFLAGS := -X main.Version=$(VERSION)
 
-.PHONY: all build test lint fmt fmt-check check docker hooks clean
+.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
 
-all: check build
+# Standard targets are thin shims; the implementations live in script/
+# per the scripts-to-rule-them-all pattern.
 
-build:
+# Default target: build the binary. Phony so go build (which has its
+# own build cache) always decides what to recompile.
+sfdupes:
 	go build -ldflags "$(LDFLAGS)" -o $(BINARY)
 
+build: sfdupes
+
+bootstrap:
+	@script/bootstrap
+
+setup:
+	@script/setup
+
 test:
-	@go test -timeout 30s -cover ./... || \
+	@script/test
-		{ echo "--- Rerunning with -v for details ---"; \
-		  go test -timeout 30s -v ./...; exit 1; }
 
 lint:
-	golangci-lint run --config .golangci.yml ./...
+	@script/lint
 
 fmt:
-	gofmt -s -w .
+	@script/fmt
 
 fmt-check:
-	@files="$$(gofmt -l -s .)"; if [ -n "$$files" ]; then \
+	@script/fmt-check
-		echo "gofmt: files not formatted:"; echo "$$files"; exit 1; fi
 
-check: test lint fmt-check
+check:
+	@script/check
 
 docker:
-	docker build -t $(BINARY) .
+	@script/docker
-
-# Hooks are shared between the main checkout and all worktrees, so
-# resolve the common git dir instead of assuming .git is a directory.
-HOOKS_DIR := $(shell git rev-parse --git-common-dir)/hooks
 
 hooks:
-	@printf '#!/bin/sh\nset -e\n' > $(HOOKS_DIR)/pre-commit
+	@script/install-precommit
-	@printf 'go mod tidy\ngo fmt ./...\n' >> $(HOOKS_DIR)/pre-commit
-	@printf 'git diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> $(HOOKS_DIR)/pre-commit
-	@printf 'make check\n' >> $(HOOKS_DIR)/pre-commit
-	@chmod +x $(HOOKS_DIR)/pre-commit
 
 clean:
 	rm -f $(BINARY) files.dat

README.md

@@ -12,7 +12,12 @@ SHA-256 of their first 1024 bytes, and identical SHA-256 of their last
 content (the middle of the file is never read); the intended use is
 finding duplicate downloads and duplicated directory trees on
 multi-terabyte ZFS servers where reading every byte is prohibitively
-expensive.
+expensive. `scan` maintains a persistent SQLite database of file
+signatures that survives between runs, so it can be run from cron and
+the reports can be generated at any time from the most recent scan.
+
+This tool was created by [@sneak](https://sneak.berlin) to scratch an itch,
+using Claude Code/Fable.
 
 This README is the complete and authoritative specification.
 
@@ -20,29 +25,41 @@ This README is the complete and authoritative specification.
 
 ```sh
 make build
-./sfdupes scan /srv > files.dat
+export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
-./sfdupes report files.dat > dupes.tsv
+./sfdupes scan /srv
-./sfdupes trees files.dat > dupetrees.tsv
+./sfdupes report > dupes.tsv
+./sfdupes trees > dupetrees.tsv
 ```
 
-`scan` walks one or more filesystem trees and emits one record per
+`scan` walks one or more filesystem trees and maintains one database
-regular file (path, size, mtime, head hash, tail hash). `report`
+record per regular file (path, size, mtime, head hash, tail hash). The
-ingests that stream and prints the file-level duplicates report.
+database persists between runs; a rescan only hashes files that are new
-`trees` ingests the same stream and prints the duplicate-tree report. A
+or changed, and removes records for files that no longer exist.
-missing/invalid subcommand — or a `scan` invocation with no `PATH`
+`report` reads the database and prints the file-level duplicates
-operand — prints a usage message and exits 2.
+report. `trees` reads the same database and prints the duplicate-tree
+report. A missing/invalid subcommand — or a `scan` invocation with no
+`PATH` operand — prints a usage message and exits 2.
+
+The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed
+anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a
+daily `sfdupes scan` cron job, with the reporting commands run
+interactively whenever needed; their results are as fresh as the last
+completed scan.
 
 ## Rationale
 
 Duplicate finders that hash entire files do not scale to the target
 environment: ~10 million files and ~150 TB on possibly slow or busy
-disks (a ZFS pool under resilver). Reading at most 2 KiB per file makes
+disks (a ZFS pool under resilver). Reading at most 2 KiB per file — and
-a full-filesystem sweep tractable, and the resulting scan stream is
+only from files whose size at least one other file shares, since a
-self-contained, so the expensive filesystem pass runs exactly once and
+size-unique file cannot be a duplicate — makes a full-filesystem sweep
-all analysis happens offline. The end goal is not individual files but
+tractable, and the signatures are kept in a persistent database, so
-whole duplicated trees — duplicate extractions, duplicate downloads,
+the expensive filesystem pass is incremental: a rescan re-hashes only
-copied project trees — which an operator can consider removing as a
+files whose recorded mtime or size changed, and all analysis happens
-unit.
+offline from the database alone. The end goal is
+not individual files but whole duplicated trees — duplicate
+extractions, duplicate downloads, copied project trees — which an
+operator can consider removing as a unit.
 
 ## Design
 
@@ -55,14 +72,19 @@ Goals, in order:
     removing an entire subtree at once. File-level duplicate detection is
     the foundation; tree-level detection is built on top of it.
 2.  **Never read full file contents.** At most 2 KiB is read per file
-    (first and last 1024 bytes). Scale target: ~10 million files, ~150 TB
+    (first and last 1024 bytes), and only files whose size at least
+    one other file shares are read at all — a size-unique file cannot
+    be a duplicate. Scale target: tens of millions of files, ~150 TB
     filesystem, possibly slow or busy disks (ZFS pool under resilver).
-    Holding the full file list in memory is acceptable; reading file
+    Holding one small record (path, size, mtime) per file in memory
-    contents beyond 2 KiB per file is not.
+    during a scan is acceptable; holding every file's hashes is not
-3.  **Scan once, analyze offline.** The expensive filesystem scan
+    (they stay in the database).
-    produces a self-contained stream; all analysis (`report`, `trees`)
+3.  **Scan incrementally, analyze offline.** The expensive filesystem
-    works from that stream alone and must never touch the scanned
+    scan maintains a persistent database; an unchanged file is never
-    filesystem again.
+    read again on a rescan. All analysis (`report`, `trees`) works from
+    the database alone and must never touch the scanned filesystem
+    again. `scan` is designed to be cronned; the reports run at any
+    time against the last completed scan.
 4.  **Clean stream separation.** Everything on stdout is machine-readable
     data. All progress, warnings, and summaries go to stderr. Never mix
     them.
@@ -71,54 +93,162 @@ Goals, in order:
 
 -   Language: Go (module `sneak.berlin/go/sfdupes`). Binary name:
     `sfdupes`.
--   Dependencies: standard library, `github.com/spf13/cobra` for the CLI,
+-   Dependencies: standard library, `github.com/spf13/cobra` for the
-    and **one progress-bar library**
+    CLI, **one progress-bar library**
-    (`github.com/schollz/progressbar/v3`). `github.com/spf13/viper` is
+    (`github.com/schollz/progressbar/v3`), and **one SQLite driver**
-    permitted if configuration-file support is ever needed, but is not
+    (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled).
-    currently used. No other third-party deps.
+    `github.com/spf13/viper` is permitted if configuration-file support
+    is ever needed, but is not currently used. No other third-party
+    deps.
 -   Cross-compilation is not a concern. Builds run with cgo disabled (the
     `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go.
 -   Analysis modes (`report`, `trees`) must be deterministic: identical
-    input stream, identical output, regardless of record order.
+    database contents, identical output, regardless of the order in
+    which records were inserted.
 
 ### Subcommands
 
 Three subcommands, all implemented:
 
-1.  `scan` — walk the filesystem and emit one signature record per
+1.  `scan` — walk the filesystem and synchronize the database: one
-    regular file.
+    signature record per regular file.
-2.  `report` — file-level duplicate report from the scan stream.
+2.  `report` — file-level duplicate report from the database.
 3.  `trees` — tree-level duplicate report: reconstruct the directory
-    hierarchy from the scan stream, compute a Merkle-style digest per
+    hierarchy from the database records, compute a Merkle-style digest
-    directory, and report maximal groups of identical trees.
+    per directory, and report maximal groups of identical trees.
 
 ```
-sfdupes scan [--workers N] [-x] PATH... > files.dat
+sfdupes scan [--workers N] [-x] PATH...
-sfdupes report [files.dat|-] > dupes.tsv
+sfdupes report > dupes.tsv
-sfdupes trees [files.dat|-] > dupetrees.tsv
+sfdupes trees > dupetrees.tsv
 ```
 
+### Database
+
+All three subcommands operate on a single SQLite database file:
+
+-   Location: the value of the `SFDUPES_DATABASE` environment variable
+    when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`.
+    There is no command-line flag.
+-   `scan` creates the database (and its parent directory) on first
+    use. `report` and `trees` require an existing database; a missing
+    database file is a fatal error (exit 1) telling the user to run
+    `scan` first.
+-   The database uses WAL journal mode and a busy timeout, so running a
+    report while a cron `scan` is in progress is safe. The filesystem
+    is authoritative; the database is an eventually-consistent
+    reflection of it. Hashed records are committed in batched
+    transactions while the scan is still running (keeping the WAL
+    small and letting concurrent reports observe progress), so a
+    report may see a scan's changes partially applied, and a scan
+    that dies partway leaves a valid database holding everything
+    hashed so far; the next scan skips those records and converges
+    toward the filesystem.
+-   Schema (`PRAGMA user_version` is the schema version, currently 1; a
+    database with any other version is a fatal error):
+
+    ```sql
+    CREATE TABLE files (
+        path  BLOB PRIMARY KEY,  -- absolute path, raw bytes
+        size  INTEGER NOT NULL,  -- bytes, from lstat
+        mtime INTEGER NOT NULL,  -- Unix seconds, from lstat
+        head  TEXT NOT NULL,     -- lowercase-hex SHA-256, first 1 KiB
+        tail  TEXT NOT NULL      -- lowercase-hex SHA-256, last 1 KiB
+    ) WITHOUT ROWID;
+    ```
+
+    Paths are stored as BLOBs because Unix paths are raw bytes, not
+    guaranteed UTF-8. `mtime` is used only for change detection; it is
+    not part of the duplicate key. `head` and `tail` are empty strings
+    when the file has never been hashed because its size was unique as
+    of the last scan that covered it; such records still define the
+    file for tree reconstruction but never participate in duplicate
+    groups.
+
 ### `scan` mode
 
 `scan` requires one or more `PATH` operands naming the trees to scan.
 There is no default path; invoking `scan` with no operand is a usage
 error (usage message on stderr, exit 2). An operand may be a directory
 or a regular file; an operand that does not exist is a fatal error
-(exit 1). Operands are walked in the order given; overlapping operands
+(exit 1). Because database records persist between runs and are keyed
-(one containing another) emit their common files once per operand, so
+by absolute path, each operand is resolved to an absolute, lexically
-callers should pass disjoint paths.
+cleaned path (symlinks are not resolved) before walking, so results do
+not depend on the working directory. All operands belong to a single
+scan and are enumerated concurrently: every operand seeds the shared
+walk worker pool. Overlapping operands are harmless — an operand that
+duplicates another or lies under another is dropped before walking,
+so every file is reached exactly once and produces one database
+record.
 
-`scan` runs **three sequential passes**, in this order, so that every
+`scan` synchronizes the database with the filesystem state under the
-expensive pass has an exact total for meaningful progress and ETA:
+scanned operands:
 
-1.  **walk** — recursively enumerate the tree under each `PATH` in
+-   Only a file whose size at least one other file shares is ever
-    turn, collecting the list of regular-file paths. Total unknown
+    read: a size-unique file cannot be a duplicate, so it is recorded
-    while running: show a live count, not a percentage.
+    without hashes (`head` and `tail` empty). The size census covers
-2.  **stat** — `lstat` every collected path, recording size and mtime.
+    every file walked this scan plus every database record outside
-3.  **hash** — for each file, read the first `min(1024, size)` bytes and
+    the scanned operands, so a possible duplicate of a separately
-    the last `min(1024, size)` bytes (the two reads overlap when
+    scanned tree is still recognized.
-    `size < 2048`; for `size == 0` hash the empty input) and compute the
+-   A file not yet in the database is inserted: hashed when its size
-    SHA-256 of each. Emit the output record.
+    is shared, without hashes otherwise.
+-   A file already in the database is **skipped without reading its
+    contents** when its lstat size equals the recorded size and its
+    lstat mtime is not newer than the recorded mtime. This is what
+    makes a daily rescan cheap. Exception: an unchanged file whose
+    record lacks hashes is hashed — and its record updated — once its
+    size becomes shared, so hashing deferred by size-uniqueness
+    happens as soon as it could matter.
+-   A file whose mtime is newer than recorded, or whose size differs,
+    is processed as if new: re-hashed, or recorded without hashes,
+    per the shared-size rule.
+-   A database record whose path lies under one of the scanned operands
+    but was not successfully processed this run is deleted. This
+    removes records for deleted files. It also removes records for
+    paths that failed to stat or hash this run: the database only ever
+    contains signatures verified by the most recent scan that covered
+    them (a subsequent successful scan re-adds such files).
+-   Database records outside the scanned operands are untouched, so
+    disjoint trees can be scanned on different schedules into the same
+    database.
+
+`scan` runs **three sequential phases over the whole scan**.
+Parallelism lives inside each phase; batched database writes begin
+during the hash phase:
+
+1.  **walk + stat** — enumerate the trees under all `PATH` operands
+    concurrently with the walk worker pool: every operand seeds the
+    shared queue, and each worker reads one directory at a time,
+    handing discovered subdirectories back to the queue and running
+    `lstat` on each regular file as it is discovered (while the
+    directory's metadata is still hot). Sequential directory
+    enumeration is metadata-latency-bound and takes hours at tens of
+    millions of files; per-directory parallelism is what makes the
+    walk tractable on large or busy pools. The walk builds the size
+    census and resolves unchanged already-hashed files on the fly;
+    every other file is carried to the hash phase as a (path, size,
+    mtime) record.
+2.  **hash** — with the census complete, each carried file's size
+    decides its fate. Size-unique files are never read: new or
+    changed ones are recorded without hashes in the update phase,
+    unchanged unhashed ones simply keep their records. Every file
+    with a shared size is hashed by the worker pool: read the first
+    `min(1024, size)` bytes and the last `min(1024, size)` bytes
+    (one read when `size <= 1024`, since the two windows coincide)
+    and compute the SHA-256 of each. Zero-length files have constant
+    hashes and are never opened. Files are hashed in **inode order**
+    (minimizing seeks on spinning disks), and paths that are hard
+    links to the same inode are **read once**, all sharing the one
+    result — a hard-link backup farm costs one read per inode, not
+    per path. The phase total counts actual reads, so progress and
+    ETA are meaningful. Completed records are committed in batched
+    transactions **while hashing runs**, so a scan interrupted after
+    hours keeps everything hashed so far and the next scan resumes
+    cheaply, skipping records already written.
+3.  **update** — commit the final partial batch, the hash-less
+    records for size-unique new and changed files, and the deletions
+    for records the scan did not verify (vanished files, plus paths
+    that failed to stat or hash).
 
 Rules for the walk:
 
@@ -134,52 +264,55 @@ Rules for the walk:
 -   On any per-path error (permission denied, file vanished between
     passes, unreadable): print a one-line warning to stderr, skip the
     path, and continue. Per-file errors never abort the run; the final
-    summary reports how many were skipped.
+    summary reports how many were skipped. As specified above, a
+    skipped path that has a database record from an earlier scan loses
+    that record; an unreadable directory subtree likewise loses its
+    records (accepted: the database mirrors what the latest scan could
+    actually verify).
 
-Concurrency: the stat and hash passes use a worker pool (`--workers`,
+Concurrency: the walk phase (which also stats files) and the hash
-default `runtime.NumCPU()`). The main goroutine owns stdout writing and
+phase each use a worker pool of `--workers` workers (default
-progress rendering; progress display must never block the workers.
+`runtime.NumCPU()`); the walk parallelizes across directories,
+hashing across files. Both phases are seek-bound on spinning disks,
+so raising `--workers` well past the core count can help on pools
+with many spindles. The main goroutine owns partitioning, database
+writes, and progress rendering; progress display must never block
+the workers.
 
-#### Output record format
+`scan` writes nothing to stdout. The summary line on stderr reports the
-
+files seen this run broken down by disposition, plus skips:
-One record per file on stdout, NUL-terminated (`\x00`), with
-tab-separated fields, **path last** so tabs or newlines embedded in
-paths cannot corrupt the record structure:
 
 ```
-<size>\t<mtime_unix>\t<sha256_first1k_hex>\t<sha256_last1k_hex>\t<path>\x00
+scan: 123456 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped
 ```
 
--   `size`: decimal bytes, from the stat pass.
+(`removed` counts deleted database records, which are not part of the
--   `mtime_unix`: decimal Unix seconds. Informational only; not part of
+files-seen total.)
-    the duplicate key.
--   Hashes: lowercase hex, 64 chars each.
--   Record order is unspecified (workers complete out of order); the
-    analysis modes must not depend on ordering.
 
 ### `report` mode
 
-`report` reads the scan stream from the file named in its first
+`report` reads every record from the database and takes no positional
-positional argument, or from stdin if the argument is absent or `-`.
+arguments.
 
 **`report` must never touch the filesystem being analyzed.** It does not
 stat, open, or otherwise access any path that appears in the records; its
-only I/O is reading the scan file/stdin and writing stdout/stderr. It must
+only I/O is reading the database and writing stdout/stderr. It must
 produce identical output whether or not the scanned filesystem is still
 mounted.
 
 Processing:
 
--   Parse records; a record that does not have exactly 5 fields or whose
+-   Records without hashes (size-unique when last scanned) are
-    size is non-numeric is counted as malformed and skipped (warn once
+    excluded: their content is unknown, so they are never reported as
-    with the total malformed count in the summary, not per record).
+    duplicates.
--   Group records by the key `(size, head_hash, tail_hash)`.
+-   Group the remaining records by the key
+    `(size, head_hash, tail_hash)`.
 -   Every group with two or more paths is a duplicate group.
 -   Within each group, sort paths lexicographically (byte order). The
     first path is the group's `first`; every other path is a `dupe`.
 -   Order groups by size descending (biggest reclaimable space first),
     tie-broken by `first` path ascending. Output must be fully
-    deterministic for a given input.
+    deterministic for a given database state.
 
 #### Report output format
 
@@ -192,16 +325,16 @@ first	dupe	size
 /srv/a/big.iso	/srv/c/big-copy2.iso	4294967296
 ```
 
-Summary to stderr: records read, malformed count (if any), number of
+Summary to stderr: records read, number of duplicate groups, number of
-duplicate groups, number of dupe files, and total reclaimable bytes
+dupe files, and total reclaimable bytes (sum of `size` over all dupe
-(sum of `size` over all dupe rows) in human units.
+rows) in human units.
 
 ### `trees` mode
 
-`trees` reads the same scan stream as `report` (same argument handling,
+`trees` reads the same database as `report` (no positional arguments)
-same parsing and malformed-record rules) and reports **entire duplicate
+and reports **entire duplicate directory trees**: directories under
-directory trees**: directories under which the exact same set of relative
+which the exact same set of relative paths exists with the exact same
-paths exists with the exact same file signatures.
+file signatures.
 
 **`trees` must never touch the filesystem being analyzed** — the same
 rule as `report`. The directory hierarchy is reconstructed purely from
@@ -210,7 +343,10 @@ the paths in the records, split on `/`.
 Definitions:
 
 -   A file's **signature** is `(size, head_hash, tail_hash)` — mtime is
-    informational and excluded.
+    informational and excluded. An unhashed record (empty hashes) has
+    unknown content: its signature is treated as unique to that file,
+    so a tree containing an unhashed file never compares equal to any
+    other tree.
 -   A directory's **digest** is a SHA-256 Merkle digest computed
     bottom-up: serialize the directory's child entries — for a file
     child, its name and signature; for a subdirectory child, its name
@@ -222,10 +358,16 @@ Definitions:
     equal. Equal digests imply equal recursive file count and equal
     total byte size.
 
-Known limitation (accepted): only regular files that appear in the scan
+Known limitation (accepted): hard-linked paths are reported as
-stream define a tree. Empty directories are invisible, and a file skipped
+duplicates by `report` and count toward duplicate trees — their
-during the scan (e.g. permission error) in one copy but not the other
+content is genuinely identical — even though they share storage, so
-will make otherwise-identical trees compare as different.
+removing one reclaims no space. Inode identity is used during the
+scan to avoid redundant reads but is not persisted in the database.
+
+Known limitation (accepted): only regular files that appear in the
+database define a tree. Empty directories are invisible, and a file
+skipped during the scan (e.g. permission error) in one copy but not the
+other will make otherwise-identical trees compare as different.
 
 Processing:
 
@@ -254,32 +396,35 @@ first	dupe	files	size
 /srv/a/project	/srv/backup/project	3417	104857600
 ```
 
-Summary to stderr: records read, malformed count (if any), number of
+Summary to stderr: records read, number of duplicate-tree groups,
-duplicate-tree groups, number of dupe trees, and total reclaimable bytes
+number of dupe trees, and total reclaimable bytes (sum of `size` over
-(sum of `size` over all dupe rows) in human units.
+all dupe rows) in human units.
 
 ### Progress
 
-Use the progress-bar library for all scan-pass progress; rendering in the
+Use the progress-bar library for all scan progress; rendering in the
 style of `pv` is the model. All progress goes to stderr.
 
-Each scan pass gets its own bar. Required elements for the stat and hash
+Each phase gets its own display, rendered the moment the phase
-passes (known totals):
+starts — a scan must never look hung. Loading the existing-record
+index (`load`) and the walk have no known totals while running: show
+a live count, rate, and elapsed time (spinner-style, no percentage or
+ETA). The hash and update phases
+have exact totals — only files that actually need hashing appear in
+the hash total, so its ETA is meaningful. Required elements for the
+bars with known totals:
 
 -   elapsed time
 -   estimated time remaining
--   a `[m/n] x%` display (files processed / total files, percent)
+-   a `[m/n] x%` display (items processed / total items, percent)
--   current rate (files/s)
+-   current rate (items/s)
 
 Example shape (exact layout is flexible, content is not):
 
 ```
-hash: [1234567/9876543] 12% |████      | 8123 files/s elapsed 2:32 eta 17:54
+hash: [12345/98765] 12% |████      | 92 files/s elapsed 2:32 eta 17:54
 ```
 
-The walk pass has no known total: show a live file count and elapsed time
-(spinner-style, no percentage or ETA).
-
 Additional requirements:
 
 -   When stderr is not a TTY, do not emit ANSI redraws: print a plain
@@ -292,15 +437,18 @@ Additional requirements:
 ### Error handling and exit codes
 
 -   `0`: success, even if individual files were skipped with warnings.
--   `1`: fatal error (e.g., a `PATH` operand does not exist, cannot
+-   `1`: fatal error (e.g., a `PATH` operand does not exist, the
-    read the scan input, stdout write failure).
+    database cannot be created/opened/read/written, a missing database
--   `2`: usage error (including `scan` with no `PATH` operand).
+    for `report`/`trees`, stdout write failure).
+-   `2`: usage error (including `scan` with no `PATH` operand and
+    `report`/`trees` with any positional argument).
 
 ## Build
 
 The `Makefile` is the single source of truth for all operations:
 
--   `make build` — build the `sfdupes` binary (cgo disabled).
+-   `make` / `make build` — build the `sfdupes` binary (cgo
+    disabled); building is the default target.
 -   `make test` — run the test suite (30-second timeout; reruns with
     `-v` on failure).
 -   `make lint` — run `golangci-lint` with the repo config.
@@ -310,7 +458,7 @@ The `Makefile` is the single source of truth for all operations:
 -   `make docker` — build the Docker image, which runs `make check` as
     a build stage.
 -   `make hooks` — install the pre-commit hook.
--   `make clean` — remove the binary and any local `files.dat`.
+-   `make clean` — remove the binary and any legacy local `files.dat`.
 
 ### Definition of done
 
@@ -323,6 +471,7 @@ All of the following, run in this directory, must pass:
 
     ```sh
     d=$(mktemp -d)
+    export SFDUPES_DATABASE="$d/db.sqlite"
     mkdir -p "$d/a" "$d/b"
     head -c 2000 /dev/urandom > "$d/a/one.bin"
     cp "$d/a/one.bin" "$d/b/copy.bin"
@@ -339,28 +488,41 @@ All of the following, run in this directory, must pass:
     cp "$d/t1/sub/f2" "$d/t2/sub/f2"
     cp "$d/t1/f1" "$d/t3/f1"
     cp "$d/t1/sub/f2" "$d/t3/sub/f2renamed"
-    ./sfdupes scan "$d" > files.dat
+    ./sfdupes scan "$d"
-    ./sfdupes report files.dat
+    ./sfdupes report
-    ./sfdupes trees files.dat
+    ./sfdupes trees
+    # incremental behavior (scan a subtree; records elsewhere persist):
+    ./sfdupes scan "$d/a"                  # everything unchanged, nothing hashed
+    printf 'z' >> "$d/a/one.bin"           # modify: next scan re-hashes it
+    rm "$d/a/unique.bin"                   # delete: next scan removes its record
+    ./sfdupes scan "$d/a"                  # 1 updated, 1 removed
+    ./sfdupes report
     ```
 
-    Expected from `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
+    (The scan database lives inside `$d` here purely for test hygiene;
-    group (two dupe rows, `first` is the lexicographically smallest
+    scanning `$d` therefore also records the SQLite file itself, which
-    path); `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
+    is harmless.)
-    `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair;
+
-    `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere;
+    Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin`
-    groups ordered by size descending; piping scan directly into report
+    form one group (two dupe rows, `first` is the lexicographically
-    (`./sfdupes scan "$d" | ./sfdupes report`) gives the same
+    smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group;
-    rows.
+    `t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group;
+    `tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and
+    `tiny3` appear nowhere; groups ordered by size descending.
 
     Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe`
     `$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is
     suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3`
     appears nowhere (its file set differs by name).
 
+    Expected from the second `report` (after the modify/delete rescan):
+    `one.bin` has left its group (its content changed), so
+    `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is
+    gone from the database.
+
     The test suite automates this scenario (see `scan_test.go`), plus a
-    negative check: `report` and `trees` operate on the captured stream
+    negative check: `report` and `trees` operate on the database alone
-    alone and never touch the scanned filesystem.
+    and never touch the scanned filesystem.
 
 ## TODO
 
@@ -371,7 +533,9 @@ Tracked in [TODO.md](TODO.md).
 -   No full-content verification, no byte-for-byte compare, no deletion
     or linking of duplicates. The reports are advisory; acting on them is
     the user's job.
--   No persistence formats beyond the scan stream described above.
+-   No persistence beyond the SQLite database described above; no
+    export/import formats.
+-   No daemon or filesystem watcher; scheduling rescans is cron's job.
 
 ## License
 

TODO.md

@@ -1,23 +1,93 @@
 # Workflow
 
+- take an issue from the `1.0.0` milestone on the tracker; work not
+  yet on the tracker gets filed as an issue first
 - branch (from `main`)
-- do the work in Next Step
+- do the work, with tests, in small focused commits
-- move Next Step to the top of Completed Steps
+- record it at the top of Completed Steps (`TODO.md` changes in the
-- move the top item of Future Steps into Next Step
+  same commit as the work)
-- commit (`TODO.md` changes in the same commit as the work)
+- push the branch and open a PR whose title ends with
-- merge to `main` if the branch is not protected, otherwise open a PR
+  ` (closes #N)`
-- push
+- an independent review gates the merge; every finding is addressed
+  or explicitly rebutted on the PR
+- merge to `main` once the review passes
 
 # Status
 
 - pre-1.0
+- the Gitea tracker is authoritative for the pre-1.0 backlog: the
+  open issues under the `1.0.0` milestone are what remains before
+  the tag, and this file records history and process, not the queue
 
 # Next Step
 
-- add a remote on git.eeqj.de and push (`main` plus tags)
+- take the next issue from the `1.0.0` milestone on the tracker:
+  https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is
+  the source of truth for what is left before 1.0.0. Individual
+  issues are deliberately not restated here; a copy in this file
+  drifts out of date the moment the tracker moves
 
 # Completed Steps
 
+- update golangci-lint to v2.12.2 with the canonical config
+  (2026-08-09, branch `golangci-v2.12.2`, merged as `38a01bd`,
+  closes #3): bumped the pinned linter in the `Dockerfile` lint
+  stage and `script/bootstrap` from v2.12.1 to v2.12.2, and replaced
+  `.golangci.yml` with the canonical file — the linter settings
+  (`lll`, `funlen`, `cyclop`, `dupl` thresholds) now live under
+  `linters.settings` per the v2 schema, so they are actually
+  applied; no new lint findings surfaced
+- convert Makefile targets to scripts-to-rule-them-all `script/`
+  entrypoints like the other managed repos (2026-07-26, commit
+  `3abeacf`, closes #1): all 12 `script/` entrypoints exist
+  (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`,
+  `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
+  `install-precommit`) and every Makefile target is now a thin shim
+  over them, matching the other managed repos
+- make the binary the default Make target (2026-07-24, branch
+  `make-default-target`): plain `make` now builds `sfdupes`
+  (previously it ran `check` plus `build`); `make build` remains as
+  an alias
+- scan-wide phases, concurrent operands, batched updates (2026-07-24,
+  branch `scan-wide-phases`): all operands seed the shared walk pool
+  and every pass runs once over the whole scan, so totals and ETAs
+  are scan-global; the per-operand walk/hash/update cycles and their
+  stderr announcements are gone; the update pass commits in batched
+  transactions — the filesystem is authoritative and the database an
+  eventually-consistent reflection, so scan-level atomicity is not
+  required
+- split the stat pass back out of the walk (2026-07-24, branch
+  `parallel-phases`): phases are strictly sequential again — walk,
+  stat, hash, update per operand — with parallelism only inside each
+  phase; the walk enumerates paths with per-directory workers and the
+  stat pass lstats them with per-file workers, restoring the exact
+  total/ETA stat bar
+- announce each operand on stderr before its passes (2026-07-24,
+  branch `scan-operand-progress`): with per-operand walk/hash/update
+  cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals
+  that looked like the whole run's — an operator watching operand 3 of
+  14 hash 300k files concluded 20M files were being skipped
+- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass
+  was a single goroutine and took hours at ~20M files on a busy pool
+  (observed: 22M files in 4h on a ZFS server); it is now a
+  per-directory worker-pool traversal that records size/mtime during
+  the walk (folding away the separate stat pass, halving metadata
+  I/O), and each `PATH` operand commits in its own transaction so an
+  interrupted scan keeps completed operands
+
+- persistent scan database (2026-07-24, branch `persistent-database`):
+  `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure
+  Go, cgo stays disabled) keyed by absolute path that survives between
+  runs — a rescan hashes only new or changed files (by mtime/size),
+  deletes records for files vanished from under the scanned operands,
+  and leaves records outside them untouched, so `scan` can be cronned
+  daily; `report` and `trees` read the database (no positional
+  arguments) instead of a scan stream. Database at
+  `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`;
+  WAL journaling plus a single-transaction update keep a report run
+  during a scan safe
+- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag
+  `v0.0.1`, and push `main` plus tags (2026-07-23)
 - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
   `PATH...` operands via cobra flags replacing the `/srv` `-root`
   default; new `-x`/`--one-file-system` flag (GNU convention) to stop
@@ -30,9 +100,6 @@
 
 # Future Steps
 
-- convert Makefile targets to scripts-to-rule-them-all `script/`
-  entrypoints like the other managed repos
-- tag `v0.0.1` once the compliance branch is merged
 - possible later features (explicitly out of scope per README):
   full-content verification of candidates, removal-script helpers
 

db.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+
+	// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
+	// disabled.
+	_ "modernc.org/sqlite"
+)
+
+// defaultDatabasePath is where the persistent scan database lives when
+// SFDUPES_DATABASE is not set.
+const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite"
+
+// databaseEnv is the environment variable that overrides the database
+// path.
+const databaseEnv = "SFDUPES_DATABASE"
+
+// schemaVersion is the database schema version this build reads and
+// writes, stored in PRAGMA user_version.
+const schemaVersion = 1
+
+// dbDirPerm is the mode for a database parent directory created by
+// scan.
+const dbDirPerm = 0o755
+
+// createTableSQL is the schema applied to a fresh database. Paths are
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+const createTableSQL = `
+CREATE TABLE files (
+    path  BLOB PRIMARY KEY,
+    size  INTEGER NOT NULL,
+    mtime INTEGER NOT NULL,
+    head  TEXT NOT NULL,
+    tail  TEXT NOT NULL
+) WITHOUT ROWID
+`
+
+// upsertSQL inserts one file record, replacing any existing record for
+// the same path.
+const upsertSQL = `
+INSERT INTO files (path, size, mtime, head, tail)
+VALUES (?, ?, ?, ?, ?)
+ON CONFLICT (path) DO UPDATE SET
+    size = excluded.size, mtime = excluded.mtime,
+    head = excluded.head, tail = excluded.tail
+`
+
+// errNoDatabase reports a missing database file for report/trees.
+var errNoDatabase = errors.New(
+	"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
+
+// errSchemaVersion reports a database whose schema version this build
+// does not understand.
+var errSchemaVersion = errors.New("unsupported database schema version")
+
+// databasePath resolves the database location: SFDUPES_DATABASE when
+// set and non-empty, the compiled-in default otherwise.
+func databasePath() string {
+	if p := os.Getenv(databaseEnv); p != "" {
+		return p
+	}
+
+	return defaultDatabasePath
+}
+
+// openDB opens the SQLite database at path with WAL journaling and a
+// busy timeout, so a report can run while a cron scan is in progress.
+// It does not create or verify the schema.
+func openDB(path string) (*sql.DB, error) {
+	dsn := "file:" + path +
+		"?_pragma=busy_timeout(10000)" +
+		"&_pragma=journal_mode(WAL)" +
+		"&_pragma=synchronous(NORMAL)"
+
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, fmt.Errorf("open database %s: %w", path, err)
+	}
+
+	// A single connection avoids SQLITE_BUSY between this process's
+	// own connections; concurrency lives in the worker pools, not in
+	// parallel database access.
+	db.SetMaxOpenConns(1)
+
+	return db, nil
+}
+
+// openScanDatabase opens the database for the scan subcommand, creating
+// the file, its parent directory, and the schema as needed.
+func openScanDatabase(path string) (*sql.DB, error) {
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
+	if err != nil {
+		return nil, fmt.Errorf("create database directory: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = initSchema(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	return db, nil
+}
+
+// openReportDatabase opens an existing database for the report and
+// trees subcommands. A missing database file is an error directing the
+// user to run scan first; the schema version must match exactly.
+func openReportDatabase(path string) (*sql.DB, error) {
+	_, err := os.Stat(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, fmt.Errorf("%s: %w", path, errNoDatabase)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("database: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	v, err := userVersion(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	if v != schemaVersion {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: version %d, want %d: %w",
+			path, v, schemaVersion, errSchemaVersion)
+	}
+
+	return db, nil
+}
+
+// initSchema creates the schema on a fresh database and verifies the
+// schema version on an existing one.
+func initSchema(db *sql.DB) error {
+	v, err := userVersion(db)
+	if err != nil {
+		return err
+	}
+
+	switch v {
+	case 0:
+		return createSchema(db)
+	case schemaVersion:
+		return nil
+	default:
+		return fmt.Errorf("version %d, want %d: %w",
+			v, schemaVersion, errSchemaVersion)
+	}
+}
+
+// createSchema applies the schema to a fresh database and stamps the
+// schema version.
+func createSchema(db *sql.DB) error {
+	ctx := context.Background()
+
+	_, err := db.ExecContext(ctx, createTableSQL)
+	if err != nil {
+		return fmt.Errorf("create schema: %w", err)
+	}
+
+	_, err = db.ExecContext(ctx,
+		"PRAGMA user_version = "+strconv.Itoa(schemaVersion))
+	if err != nil {
+		return fmt.Errorf("set schema version: %w", err)
+	}
+
+	return nil
+}
+
+// userVersion reads the database's PRAGMA user_version.
+func userVersion(db *sql.DB) (int, error) {
+	var v int
+
+	err := db.QueryRowContext(context.Background(),
+		"PRAGMA user_version").Scan(&v)
+	if err != nil {
+		return 0, fmt.Errorf("read schema version: %w", err)
+	}
+
+	return v, nil
+}
+
+// loadFileRows reads every record from the files table.
+func loadFileRows(db *sql.DB) ([]scanRec, error) {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head, tail FROM files")
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var recs []scanRec
+
+	for rows.Next() {
+		var (
+			path []byte
+			r    scanRec
+		)
+
+		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail)
+		if err != nil {
+			return nil, fmt.Errorf("read record: %w", err)
+		}
+
+		r.path = string(path)
+		recs = append(recs, r)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	return recs, nil
+}
+
+// loadFileMeta streams every record's path, size, mtime, and whether
+// it carries hashes to fn. Scan change detection needs no hash
+// values, and skipping the hash columns keeps the scan's in-memory
+// index small on multi-million-file databases.
+func loadFileMeta(db *sql.DB,
+	fn func(path string, size, mtime int64, hashed bool),
+) error {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head <> '' FROM files")
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var (
+			path        []byte
+			size, mtime int64
+			hashed      int64
+		)
+
+		err = rows.Scan(&path, &size, &mtime, &hashed)
+		if err != nil {
+			return fmt.Errorf("read record: %w", err)
+		}
+
+		fn(string(path), size, mtime, hashed != 0)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	return nil
+}
+
+// updateBatchSize is the number of record changes committed per
+// transaction during the update pass. The filesystem is authoritative
+// and the database an eventually-consistent reflection of it, so
+// scan-level atomicity is not required; smaller transactions keep the
+// WAL small and let concurrent reports observe progress.
+const updateBatchSize = 10000
+
+// applyChanges writes one scan's database changes — upserts for new and
+// changed files, deletes for vanished ones — in batched transactions.
+// Progress is rendered on prog (one increment per change).
+func applyChanges(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	for batch := range slices.Chunk(upserts, updateBatchSize) {
+		err := applyBatch(db, batch, nil, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	for batch := range slices.Chunk(deletes, updateBatchSize) {
+		err := applyBatch(db, nil, batch, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// applyBatch commits one batch of upserts and deletes in a single
+// transaction.
+func applyBatch(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	ctx := context.Background()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+
+	defer func() { _ = tx.Rollback() }()
+
+	err = execUpserts(ctx, tx, upserts, prog)
+	if err != nil {
+		return err
+	}
+
+	err = execDeletes(ctx, tx, deletes, prog)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("commit: %w", err)
+	}
+
+	return nil
+}
+
+// execUpserts inserts or updates one record per new or changed file.
+func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, upsertSQL)
+	if err != nil {
+		return fmt.Errorf("prepare upsert: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, r := range upserts {
+		_, err = st.ExecContext(ctx,
+			[]byte(r.path), r.size, r.mtime, r.head, r.tail)
+		if err != nil {
+			return fmt.Errorf("upsert %s: %w", r.path, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}
+
+// execDeletes removes the records for paths no longer present.
+func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, "DELETE FROM files WHERE path = ?")
+	if err != nil {
+		return fmt.Errorf("prepare delete: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, p := range deletes {
+		_, err = st.ExecContext(ctx, []byte(p))
+		if err != nil {
+			return fmt.Errorf("delete %s: %w", p, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}

db_test.go

@@ -0,0 +1,224 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// testDBPath returns a database path inside a fresh temp dir.
+func testDBPath(t *testing.T) string {
+	t.Helper()
+
+	return filepath.Join(t.TempDir(), "db.sqlite")
+}
+
+// openTestDB creates a fresh scan database in a temp dir.
+func openTestDB(t *testing.T) *sql.DB {
+	t.Helper()
+
+	db, err := openScanDatabase(testDBPath(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() { _ = db.Close() })
+
+	return db
+}
+
+func TestDatabasePath(t *testing.T) {
+	t.Setenv(databaseEnv, "")
+
+	if got := databasePath(); got != defaultDatabasePath {
+		t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
+	}
+
+	t.Setenv(databaseEnv, "/custom/place.sqlite")
+
+	if got := databasePath(); got != "/custom/place.sqlite" {
+		t.Errorf("databasePath() = %q, want the env override", got)
+	}
+}
+
+func TestOpenScanDatabaseCreates(t *testing.T) {
+	t.Parallel()
+
+	// The parent directory does not exist yet; scan must create it.
+	path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("openScanDatabase: %v", err)
+	}
+
+	v, err := userVersion(db)
+	if err != nil || v != schemaVersion {
+		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
+	}
+
+	_ = db.Close()
+
+	// Reopening an existing database must succeed and find the schema.
+	db, err = openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+
+	defer func() { _ = db.Close() }()
+
+	recs, err := loadFileRows(db)
+	if err != nil || len(recs) != 0 {
+		t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err)
+	}
+}
+
+func TestOpenReportDatabaseMissing(t *testing.T) {
+	t.Parallel()
+
+	_, err := openReportDatabase(testDBPath(t))
+	if !errors.Is(err, errNoDatabase) {
+		t.Fatalf("err = %v, want errNoDatabase", err)
+	}
+}
+
+func TestOpenReportDatabaseVersionMismatch(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	_, err = openReportDatabase(path)
+	if !errors.Is(err, errSchemaVersion) {
+		t.Fatalf("err = %v, want errSchemaVersion", err)
+	}
+}
+
+func TestOpenReportDatabaseOK(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	db, err = openReportDatabase(path)
+	if err != nil {
+		t.Fatalf("openReportDatabase: %v", err)
+	}
+
+	_ = db.Close()
+}
+
+func TestApplyChangesRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// Paths may contain tabs and newlines; the database must store
+	// them byte-exactly.
+	recs := []scanRec{
+		{size: 2, mtime: 20, head: "h2", tail: "t2", path: "/a/tab\tnew\nline"},
+		{size: 1, mtime: 10, head: "h1", tail: "t1", path: "/a/x"},
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	slices.SortFunc(got, func(a, b scanRec) int {
+		return strings.Compare(a.path, b.path)
+	})
+
+	if !slices.Equal(got, recs) {
+		t.Fatalf("rows = %+v, want %+v", got, recs)
+	}
+
+	// An upsert for an existing path updates in place; a delete
+	// removes exactly its path.
+	upd := scanRec{size: 3, mtime: 30, head: "h3", tail: "t3", path: "/a/x"}
+
+	err = applyChanges(db, []scanRec{upd},
+		[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(got) != 1 || got[0] != upd {
+		t.Fatalf("rows = %+v, want just %+v", got, upd)
+	}
+}
+
+func TestApplyChangesBatching(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// One more change than the batch size, so the update spans two
+	// transactions.
+	n := updateBatchSize + 1
+
+	recs := make([]scanRec, 0, n)
+	for i := range n {
+		recs = append(recs, scanRec{
+			size: int64(i), mtime: 1, head: "h", tail: "t",
+			path: fmt.Sprintf("/batch/%07d", i),
+		})
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", int64(n)))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil || len(got) != n {
+		t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n)
+	}
+
+	deletes := make([]string, 0, n)
+	for _, r := range recs {
+		deletes = append(deletes, r.path)
+	}
+
+	err = applyChanges(db, nil, deletes, newProgress("update", int64(n)))
+	if err != nil {
+		t.Fatalf("applyChanges deletes: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil || len(got) != 0 {
+		t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err)
+	}
+}

go.mod

@@ -5,13 +5,22 @@ go 1.25.7
 require (
 	github.com/schollz/progressbar/v3 v3.19.1
 	github.com/spf13/cobra v1.10.2
+	modernc.org/sqlite v1.54.0
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.22 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/term v0.44.0 // indirect
+	modernc.org/libc v1.74.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -3,14 +3,28 @@ github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
+github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -23,10 +37,44 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
 golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
+modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog=
+modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

main.go

@@ -2,13 +2,15 @@
 // very large filesystems without reading full file contents. Files are
 // considered duplicates when they have identical size, identical SHA-256
 // of their first 1024 bytes, and identical SHA-256 of their last 1024
-// bytes.
+// bytes. scan maintains a persistent SQLite database of file signatures
+// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
+// reporting subcommands read.
 //
 // Usage:
 //
-//	sfdupes scan [--workers N] [-x] PATH... > files.dat
+//	sfdupes scan [--workers N] [-x] PATH...
-//	sfdupes report [files.dat|-] > dupes.tsv
+//	sfdupes report > dupes.tsv
-//	sfdupes trees [files.dat|-] > dupetrees.tsv
+//	sfdupes trees > dupetrees.tsv
 //
 // See README.md for the complete specification.
 package main
@@ -60,32 +62,32 @@ func main() {
 
 	scanCmd := &cobra.Command{
 		Use:   "scan [--workers N] [-x] PATH...",
-		Short: "Walk trees and emit one record per regular file on stdout",
+		Short: "Walk trees and synchronize the scan database",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(_ *cobra.Command, args []string) {
 			runScan(args, scanWorkers, scanOneFS)
 		},
 	}
 	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
-		"concurrent workers for the stat and hash passes")
+		"concurrent workers for the walk and hash phases")
 	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
 		"do not cross filesystem boundaries")
 
 	reportCmd := &cobra.Command{
-		Use:   "report [files.dat|-]",
+		Use:   "report",
-		Short: "Read a scan stream and print the file-level duplicates report",
+		Short: "Read the scan database and print the file-level duplicates report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runReport(args)
+			runReport()
 		},
 	}
 
 	treesCmd := &cobra.Command{
-		Use:   "trees [files.dat|-]",
+		Use:   "trees",
-		Short: "Read a scan stream and print the duplicate-tree report",
+		Short: "Read the scan database and print the duplicate-tree report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runTrees(args)
+			runTrees()
 		},
 	}
 

progress.go

@@ -38,7 +38,10 @@ func stderrIsTTY() bool {
 // stderr is not a TTY it emits no ANSI redraws: it prints a plain
 // one-line update no more often than every plainInterval.
 //
-// All methods must be called from the main goroutine only.
+// All methods must be called from the main goroutine only. A nil
+// *progress is a valid no-display receiver: every method is a no-op,
+// so batched database flushes during the streaming pass can reuse the
+// update-pass helpers without rendering anything.
 type progress struct {
 	label string
 	total int64 // -1 when unknown (walk pass)
@@ -62,6 +65,9 @@ func newProgress(label string, total int64) *progress {
 		progressbar.OptionSetItsString("files"),
 		progressbar.OptionSetElapsedTime(true),
 		progressbar.OptionThrottle(barThrottle),
+		// Render at zero immediately: a phase must be visible the
+		// moment it starts, even before its first item completes.
+		progressbar.OptionSetRenderBlankState(true),
 	}
 	if total >= 0 {
 		opts = append(opts,
@@ -82,6 +88,10 @@ func newProgress(label string, total int64) *progress {
 
 // increment records one completed item and refreshes the display.
 func (p *progress) increment() {
+	if p == nil {
+		return
+	}
+
 	p.count++
 	if p.bar != nil {
 		_ = p.bar.Add(1)
@@ -97,6 +107,10 @@ func (p *progress) increment() {
 
 // warnf prints a one-line warning to stderr without corrupting the bar.
 func (p *progress) warnf(format string, args ...any) {
+	if p == nil {
+		return
+	}
+
 	if p.bar != nil {
 		_ = p.bar.Clear()
 	}
@@ -106,6 +120,10 @@ func (p *progress) warnf(format string, args ...any) {
 
 // finish terminates the pass's display.
 func (p *progress) finish() {
+	if p == nil {
+		return
+	}
+
 	if p.bar != nil {
 		_ = p.bar.Finish()
 

report.go

@@ -2,106 +2,49 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"slices"
-	"strconv"
 	"strings"
 )
 
-// ioBufSize is the buffer size for the buffered scan-stream reader and
+// ioBufSize is the buffer size for the buffered stdout writers.
-// the buffered stdout writers.
 const ioBufSize = 1 << 20
 
-// recordFieldCount is the number of tab-separated fields in a scan
-// record: size, mtime, head hash, tail hash, path.
-const recordFieldCount = 5
-
-// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
-// to read scan records; a record longer than scanMaxRecordSize is a
-// fatal read error.
-const (
-	scanInitBufSize   = 64 << 10
-	scanMaxRecordSize = 4 << 20
-)
-
 // minGroupSize is the smallest number of members that makes a
 // duplicate group.
 const minGroupSize = 2
 
-// scanRec is one well-formed record parsed from a scan stream. The
+// scanRec is one file record from the database. The signature (size,
-// signature (size, head, tail) is the duplicate key; mtime is
+// head, tail) is the duplicate key; mtime is informational only and
-// informational and not retained.
+// used by scan for change detection.
 type scanRec struct {
-	size int64
+	size  int64
-	head string
+	mtime int64
-	tail string
+	head  string
-	path string
+	tail  string
+	path  string
 }
 
-// openScanInput resolves the analysis-mode input: the file named by the
+// loadRecords opens the database and reads every file record for the
-// single optional positional argument, or stdin when it is absent or
+// report and trees subcommands. Any database problem — including a
-// "-". The returned closer must be called when reading is done.
+// missing database — is fatal.
-func openScanInput(args []string) (io.Reader, string, func()) {
+func loadRecords() []scanRec {
-	if len(args) == 1 && args[0] != "-" {
+	dbPath := databasePath()
-		f, err := os.Open(args[0])
-		if err != nil {
-			fatalf("open %s: %v", args[0], err)
-		}
 
-		return f, args[0], func() { _ = f.Close() }
+	db, err := openReportDatabase(dbPath)
-	}
-
-	return os.Stdin, "stdin", func() {}
-}
-
-// parseScanStream reads every NUL-terminated record from in. A record
-// that does not have exactly 5 fields or whose size is non-numeric is
-// counted as malformed and skipped. Total records read is
-// len(recs) + malformed.
-func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
-	sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
-	sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
-	sc.Split(splitNUL)
-
-	var (
-		recs      []scanRec
-		malformed int
-	)
-
-	for sc.Scan() {
-		// The path is the last field and may itself contain tabs,
-		// so split into at most recordFieldCount fields.
-		fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
-		if len(fields) != recordFieldCount {
-			malformed++
-
-			continue
-		}
-
-		size, err := strconv.ParseInt(fields[0], 10, 64)
-		if err != nil {
-			malformed++
-
-			continue
-		}
-
-		recs = append(recs, scanRec{
-			size: size,
-			head: fields[2],
-			tail: fields[3],
-			path: fields[4],
-		})
-	}
-
-	err := sc.Err()
 	if err != nil {
-		fatalf("read %s: %v", name, err)
+		fatalf("%v", err)
 	}
 
-	return recs, malformed
+	defer func() { _ = db.Close() }()
+
+	recs, err := loadFileRows(db)
+	if err != nil {
+		fatalf("database %s: %v", dbPath, err)
+	}
+
+	return recs
 }
 
 // dupeGroup is one set of candidate-duplicate files: identical size,
@@ -112,18 +55,12 @@ type dupeGroup struct {
 	paths []string
 }
 
-// runReport implements the report subcommand: it reads a scan stream
+// runReport implements the report subcommand: it reads every record
-// from the named file (or stdin when absent or "-") and prints the
+// from the database and prints the file-level duplicates report as TSV
-// file-level duplicates report as TSV on stdout. It never touches the
+// on stdout. It never touches the scanned filesystem; its only I/O is
-// scanned filesystem; its only I/O is the scan input, stdout, and
+// the database, stdout, and stderr.
-// stderr. args holds the positional arguments already validated by
+func runReport() {
-// cobra (at most one).
+	recs := loadRecords()
-func runReport(args []string) {
-	in, name, closer := openScanInput(args)
-	defer closer()
-
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 	dupes := collectDupeGroups(recs)
 
 	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
@@ -156,9 +93,9 @@ func runReport(args []string) {
 	}
 
 	fmt.Fprintf(os.Stderr,
-		"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
+		"report: %d records read, %d duplicate groups, %d dupe files, "+
-		records, malformedNote(malformed), len(dupes), dupeFiles,
+			"%s reclaimable\n",
-		humanBytes(reclaimable))
+		len(recs), len(dupes), dupeFiles, humanBytes(reclaimable))
 }
 
 // collectDupeGroups groups records by signature and returns every group
@@ -168,6 +105,13 @@ func collectDupeGroups(recs []scanRec) []dupeGroup {
 	groups := make(map[fileSig][]string)
 
 	for _, r := range recs {
+		// A record without hashes (its size was unique when last
+		// scanned) has unknown content and is never reported as a
+		// duplicate.
+		if r.head == "" {
+			continue
+		}
+
 		k := fileSig{size: r.size, head: r.head, tail: r.tail}
 		groups[k] = append(groups[k], r.path)
 	}
@@ -199,30 +143,6 @@ func collectDupeGroups(recs []scanRec) []dupeGroup {
 	return dupes
 }
 
-// malformedNote formats the optional malformed-record note for the
-// stderr summaries.
-func malformedNote(malformed int) string {
-	if malformed == 0 {
-		return ""
-	}
-
-	return fmt.Sprintf(" (%d malformed, skipped)", malformed)
-}
-
-// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
-// data without a terminator at EOF is returned as a final record.
-func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
-	if i := bytes.IndexByte(data, 0); i >= 0 {
-		return i + 1, data[:i], nil
-	}
-
-	if atEOF && len(data) > 0 {
-		return len(data), data, nil
-	}
-
-	return 0, nil, nil
-}
-
 // humanBytes formats a byte count in human units (binary prefixes).
 func humanBytes(n int64) string {
 	const unit = 1024

report_test.go

@@ -1,92 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"fmt"
 	"slices"
-	"strings"
 	"testing"
 )
 
-// mkRecord serializes one scan record in the on-the-wire format.
-func mkRecord(size, mtime int64, head, tail, path string) string {
-	return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00",
-		size, mtime, head, tail, path)
-}
-
-func TestSplitNULScanner(t *testing.T) {
-	t.Parallel()
-
-	sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail"))
-	sc.Split(splitNUL)
-
-	var got []string
-	for sc.Scan() {
-		got = append(got, sc.Text())
-	}
-
-	err := sc.Err()
-	if err != nil {
-		t.Fatalf("scanner error: %v", err)
-	}
-
-	want := []string{"a", "bb", "", "tail"}
-	if !slices.Equal(got, want) {
-		t.Fatalf("tokens = %q, want %q", got, want)
-	}
-}
-
-func TestParseScanStream(t *testing.T) {
-	t.Parallel()
-
-	stream := mkRecord(10, 1, "h1", "t1", "/a/x") +
-		"garbage-without-tabs\x00" +
-		"notanumber\t1\th\tt\t/a/bad\x00" +
-		mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") +
-		"30\t3\th3\tt3\t/trailing/no-nul"
-
-	recs, malformed := parseScanStream(strings.NewReader(stream), "test")
-
-	if malformed != 2 {
-		t.Errorf("malformed = %d, want 2", malformed)
-	}
-
-	want := []scanRec{
-		{size: 10, head: "h1", tail: "t1", path: "/a/x"},
-		{size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"},
-		{size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"},
-	}
-	if !slices.Equal(recs, want) {
-		t.Fatalf("recs = %+v, want %+v", recs, want)
-	}
-}
-
-func TestParseScanStreamPathWithNewline(t *testing.T) {
-	t.Parallel()
-
-	in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline"))
-
-	recs, malformed := parseScanStream(in, "test")
-	if malformed != 0 || len(recs) != 1 {
-		t.Fatalf("got %d recs, %d malformed, want 1, 0",
-			len(recs), malformed)
-	}
-
-	if recs[0].path != "/a/new\nline" {
-		t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline")
-	}
-}
-
-func TestParseScanStreamEmpty(t *testing.T) {
-	t.Parallel()
-
-	recs, malformed := parseScanStream(strings.NewReader(""), "test")
-	if len(recs) != 0 || malformed != 0 {
-		t.Fatalf("got %d recs, %d malformed, want 0, 0",
-			len(recs), malformed)
-	}
-}
-
 func TestCollectDupeGroups(t *testing.T) {
 	t.Parallel()
 
@@ -120,6 +38,22 @@ func TestCollectDupeGroups(t *testing.T) {
 	}
 }
 
+func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
+	t.Parallel()
+
+	// mtime is informational only; records differing only in mtime
+	// still group together.
+	recs := []scanRec{
+		{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
+		{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 {
+		t.Fatalf("len(groups) = %d, want 1", len(groups))
+	}
+}
+
 func TestCollectDupeGroupsTieBreak(t *testing.T) {
 	t.Parallel()
 
@@ -190,15 +124,3 @@ func TestHumanBytes(t *testing.T) {
 		}
 	}
 }
-
-func TestMalformedNote(t *testing.T) {
-	t.Parallel()
-
-	if got := malformedNote(0); got != "" {
-		t.Errorf("malformedNote(0) = %q, want empty", got)
-	}
-
-	if got := malformedNote(3); got != " (3 malformed, skipped)" {
-		t.Errorf("malformedNote(3) = %q", got)
-	}
-}

scan_test.go

@@ -1,15 +1,17 @@
 package main
 
 import (
-	"bytes"
+	"context"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
-	"io"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 // writeFile creates a file with the given content and returns its path.
@@ -98,6 +100,14 @@ func TestHashHeadTailErrors(t *testing.T) {
 		t.Error("no error for a missing file")
 	}
 
+	// A zero-length file has constant hashes and is never opened: even
+	// a missing path succeeds.
+	head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0)
+	if err != nil || head != emptyHash || tail != emptyHash {
+		t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes",
+			head, tail, err)
+	}
+
 	// A file that shrank between the stat and hash passes: reading at
 	// the stat-reported size must fail rather than emit wrong hashes.
 	p := writeFile(t, dir, "shrunk", []byte("tiny"))
@@ -108,14 +118,51 @@ func TestHashHeadTailErrors(t *testing.T) {
 	}
 }
 
-func TestWalkPass(t *testing.T) {
+// collectWalk runs a walk over roots and returns the emitted records
+// and the number of warning events.
+func collectWalk(t *testing.T, roots []string, oneFS bool,
+	workers int,
+) ([]fileRec, int) {
+	t.Helper()
+
+	var (
+		recs []fileRec
+		errs int
+	)
+
+	for ev := range startWalk(roots, oneFS, workers) {
+		if ev.fail {
+			errs++
+
+			continue
+		}
+
+		recs = append(recs, ev.rec)
+	}
+
+	return recs, errs
+}
+
+// walkedPaths returns the sorted paths of the walked records.
+func walkedPaths(recs []fileRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+func TestWalk(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
 	want := []string{
 		writeFile(t, dir, "a.txt", []byte("a")),
-		writeFile(t, dir, "sub/b.txt", []byte("b")),
+		writeFile(t, dir, "sub/b.txt", []byte("bb")),
-		writeFile(t, dir, "sub/deeper/c.txt", []byte("c")),
+		writeFile(t, dir, "sub/deeper/c.txt", []byte("ccc")),
 	}
 
 	slices.Sort(want)
@@ -129,19 +176,57 @@ func TestWalkPass(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	paths, errs := walkPass([]string{dir}, false)
+	recs, errs := collectWalk(t, []string{dir}, false, 4)
 	if errs != 0 {
 		t.Fatalf("errs = %d, want 0", errs)
 	}
 
-	slices.Sort(paths)
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
 
-	if !slices.Equal(paths, want) {
+	// The walk stats each file as it is discovered: every record must
-		t.Fatalf("paths = %q, want %q", paths, want)
+	// carry the real size and a plausible mtime.
+	for _, r := range recs {
+		if r.size < 1 || r.size > 3 {
+			t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
+		}
+
+		if r.mtime <= 0 {
+			t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
+		}
 	}
 }
 
-func TestWalkPassMultipleRoots(t *testing.T) {
+func TestWalkDeepAndWide(t *testing.T) {
+	t.Parallel()
+
+	// Exercise the dispatcher with more directories than workers and
+	// with nesting deeper than the worker count.
+	dir := t.TempDir()
+	deep := "deep" + strings.Repeat("/d", 30)
+
+	want := make([]string, 0, 41)
+	want = append(want, writeFile(t, dir, deep+"/f", []byte("x")))
+
+	for i := range 40 {
+		want = append(want, writeFile(t, dir,
+			fmt.Sprintf("wide/%02d/f", i), []byte("y")))
+	}
+
+	slices.Sort(want)
+
+	recs, errs := collectWalk(t, []string{dir}, false, 8)
+	if errs != 0 {
+		t.Fatalf("errs = %d, want 0", errs)
+	}
+
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("walked %d paths, want %d", len(got), len(want))
+	}
+}
+
+func TestWalkMultipleRoots(t *testing.T) {
 	t.Parallel()
 
 	rootA := t.TempDir()
@@ -152,18 +237,21 @@ func TestWalkPassMultipleRoots(t *testing.T) {
 		writeFile(t, rootB, "b1", []byte("3")),
 	}
 
-	paths, errs := walkPass([]string{rootA, rootB}, false)
+	slices.Sort(want)
+
+	// Operands are enumerated concurrently by the shared pool; order
+	// is unspecified.
+	recs, errs := collectWalk(t, []string{rootA, rootB}, false, 4)
 	if errs != 0 {
 		t.Fatalf("errs = %d, want 0", errs)
 	}
 
-	// Operands are walked in the order given.
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
-	if !slices.Equal(paths, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
-		t.Fatalf("paths = %q, want %q", paths, want)
 	}
 }
 
-func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
+func TestWalkFileAndSymlinkOperands(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
@@ -176,20 +264,20 @@ func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// A regular-file operand is emitted as itself.
+	// A regular-file operand is emitted as itself, statted.
-	paths, errs := walkPass([]string{f}, false)
+	recs, errs := collectWalk(t, []string{f}, false, 2)
-	if errs != 0 || !slices.Equal(paths, []string{f}) {
+	if errs != 0 || len(recs) != 1 || recs[0].path != f || recs[0].size != 4 {
-		t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
+		t.Fatalf("file operand: recs = %+v, errs = %d", recs, errs)
 	}
 
 	// A symlink operand is not followed and yields nothing.
-	paths, errs = walkPass([]string{link}, false)
+	recs, errs = collectWalk(t, []string{link}, false, 2)
-	if errs != 0 || len(paths) != 0 {
+	if errs != 0 || len(recs) != 0 {
-		t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
+		t.Fatalf("symlink operand: recs = %+v, errs = %d", recs, errs)
 	}
 }
 
-func TestWalkPassOneFilesystemSameFS(t *testing.T) {
+func TestWalkOneFilesystemSameFS(t *testing.T) {
 	t.Parallel()
 
 	// Everything in one filesystem: -x must not skip anything.
@@ -199,91 +287,38 @@ func TestWalkPassOneFilesystemSameFS(t *testing.T) {
 		writeFile(t, dir, "sub/deep/b", []byte("b")),
 	}
 
-	paths, errs := walkPass([]string{dir}, true)
+	recs, errs := collectWalk(t, []string{dir}, true, 4)
 	if errs != 0 {
 		t.Fatalf("errs = %d, want 0", errs)
 	}
 
-	slices.Sort(paths)
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
-
+		t.Fatalf("paths = %q, want %q", got, want)
-	if !slices.Equal(paths, want) {
-		t.Fatalf("paths = %q, want %q", paths, want)
 	}
 }
 
-func TestDeviceOf(t *testing.T) {
+func TestDeviceOfInfo(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
 
-	dev1, ok1 := deviceOf(dir)
+	fi1, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	dev2, ok2 := deviceOf(dir)
+	fi2, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev1, ok1 := deviceOfInfo(fi1)
+
+	dev2, ok2 := deviceOfInfo(fi2)
 	if !ok1 || !ok2 || dev1 != dev2 {
-		t.Fatalf("deviceOf unstable: %d/%v vs %d/%v", dev1, ok1, dev2, ok2)
+		t.Fatalf("deviceOfInfo unstable: %d/%v vs %d/%v",
+			dev1, ok1, dev2, ok2)
 	}
-
-	if _, ok := deviceOf(filepath.Join(dir, "missing")); ok {
-		t.Fatal("deviceOf reported ok for a missing path")
-	}
-}
-
-func TestStatPass(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	a := writeFile(t, dir, "a", pattern(1, 10))
-	b := writeFile(t, dir, "b", pattern(2, 20))
-	missing := filepath.Join(dir, "vanished")
-
-	recs, errs := statPass([]string{a, b, missing}, 2)
-	if errs != 1 {
-		t.Fatalf("errs = %d, want 1 for the vanished file", errs)
-	}
-
-	slices.SortFunc(recs, func(x, y fileRec) int {
-		return strings.Compare(x.path, y.path)
-	})
-
-	if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 {
-		t.Fatalf("recs = %+v, want sizes 10 and 20", recs)
-	}
-
-	if recs[0].mtime <= 0 || recs[1].mtime <= 0 {
-		t.Fatalf("recs = %+v, want positive mtimes", recs)
-	}
-}
-
-// captureStdout runs fn with os.Stdout redirected to a temp file and
-// returns everything fn wrote to it.
-func captureStdout(t *testing.T, fn func()) []byte {
-	t.Helper()
-
-	f, err := os.CreateTemp(t.TempDir(), "stdout")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	orig := os.Stdout
-	os.Stdout = f
-
-	defer func() { os.Stdout = orig }()
-
-	fn()
-
-	_, err = f.Seek(0, io.SeekStart)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	data, err := io.ReadAll(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_ = f.Close()
-
-	return data
 }
 
 // buildSmokeTree recreates the README smoke-test filesystem layout
@@ -316,45 +351,65 @@ func buildSmokeTree(t *testing.T) string {
 	return dir
 }
 
-// scanToRecords runs the walk, stat, and hash passes over dir and
+// smokeTreeFiles is the number of regular files buildSmokeTree creates.
-// parses the emitted stream back into records.
+const smokeTreeFiles = 15
-func scanToRecords(t *testing.T, dir string) []scanRec {
+
+// syncTree synchronizes the database with the given roots and returns
+// the scan stats.
+func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats {
 	t.Helper()
 
-	paths, walkErrs := walkPass([]string{dir}, false)
+	st, err := syncScan(db, roots, 4, false)
-	if walkErrs != 0 {
+	if err != nil {
-		t.Fatalf("walk errors: %d", walkErrs)
+		t.Fatalf("syncScan: %v", err)
 	}
 
-	recs, statErrs := statPass(paths, 4)
+	return st
-	if statErrs != 0 {
-		t.Fatalf("stat errors: %d", statErrs)
-	}
-
-	var emitted, hashErrs int
-
-	out := captureStdout(t, func() {
-		emitted, hashErrs = hashPass(recs, 4)
-	})
-
-	if emitted != len(paths) || hashErrs != 0 {
-		t.Fatalf("emitted %d of %d, %d hash errors",
-			emitted, len(paths), hashErrs)
-	}
-
-	parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe")
-	if malformed != 0 || len(parsed) != len(paths) {
-		t.Fatalf("parsed %d records, %d malformed, want %d, 0",
-			len(parsed), malformed, len(paths))
-	}
-
-	return parsed
 }
 
-//nolint:paralleltest // redirects the process-wide os.Stdout
+// dbRecords returns every record currently in the database.
-func TestScanPipeline(t *testing.T) {
+func dbRecords(t *testing.T, db *sql.DB) []scanRec {
-	dir := buildSmokeTree(t)
+	t.Helper()
-	parsed := scanToRecords(t, dir)
+
+	recs, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return recs
+}
+
+// recordByPath finds the record with the given path.
+func recordByPath(t *testing.T, recs []scanRec, path string) scanRec {
+	t.Helper()
+
+	for _, r := range recs {
+		if r.path == path {
+			return r
+		}
+	}
+
+	t.Fatalf("no record for %q", path)
+
+	return scanRec{}
+}
+
+// recordPaths returns the sorted paths of recs.
+func recordPaths(recs []scanRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+// assertSmokeDupeGroups checks the file-level duplicate groups for the
+// smoke tree rooted at dir.
+func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
 
 	groups := collectDupeGroups(parsed)
 	if len(groups) != 5 {
@@ -377,6 +432,12 @@ func TestScanPipeline(t *testing.T) {
 	if !slices.Equal(groups[0].paths, wantF1) {
 		t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
 	}
+}
+
+// assertSmokeTreeGroups checks the duplicate-tree groups for the smoke
+// tree rooted at dir.
+func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
 
 	super, dirs := buildHierarchy(parsed)
 	super.compute()
@@ -396,3 +457,451 @@ func TestScanPipeline(t *testing.T) {
 			tg[0][0].fileCount, tg[0][0].totalSize)
 	}
 }
+
+func TestScanPipeline(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: smokeTreeFiles}) {
+		t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles)
+	}
+
+	parsed := dbRecords(t, db)
+	if len(parsed) != smokeTreeFiles {
+		t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles)
+	}
+
+	assertSmokeDupeGroups(t, dir, parsed)
+	assertSmokeTreeGroups(t, dir, parsed)
+}
+
+func TestSyncScanUnchangedReuse(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("first scan stats = %+v, want 2 added", st)
+	}
+
+	// An immediate rescan reuses every record without reading file
+	// contents. Prove the files are not re-read by corrupting a stored
+	// hash and observing that it survives the rescan.
+	_, err := db.ExecContext(context.Background(),
+		"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{unchanged: 2}) {
+		t.Fatalf("rescan stats = %+v, want 2 unchanged", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
+		t.Fatalf("head = %q, want sentinel (file must not be re-read)",
+			r.head)
+	}
+}
+
+func TestSyncScanMtimeBump(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	syncTree(t, db, dir)
+
+	// Bump the mtime forward: the file must be re-hashed even though
+	// its size is unchanged.
+	future := time.Now().Add(time.Hour)
+
+	err := os.Chtimes(a, future, future)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{updated: 1}) {
+		t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
+		t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
+	}
+}
+
+func TestSyncScanAddRemove(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+	syncTree(t, db, dir)
+
+	// Add one file, remove another.
+	c := writeFile(t, dir, "c.bin", pattern(3, 700))
+
+	err := os.Remove(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 1, removed: 1, unchanged: 1}) {
+		t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged",
+			st)
+	}
+
+	want := []string{filepath.Join(dir, "b.bin"), c}
+	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+}
+
+func TestSyncScanSizeChange(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 100))
+
+	syncTree(t, db, dir)
+
+	// Rewrite with a different size but force the mtime back to the
+	// recorded value: the size mismatch alone must trigger a re-hash.
+	old := recordByPath(t, dbRecords(t, db), p)
+
+	writeFile(t, dir, "f", pattern(1, 200))
+
+	mt := time.Unix(old.mtime, 0)
+
+	err := os.Chtimes(p, mt, mt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.updated != 1 {
+		t.Fatalf("stats = %+v, want 1 updated", st)
+	}
+
+	if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 {
+		t.Fatalf("size = %d, want 200", got.size)
+	}
+}
+
+func TestSyncScanScope(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "a/keep", pattern(1, 10))
+
+	gone := writeFile(t, dir, "b/gone", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Deleting a file outside the rescanned root must not remove its
+	// record: records outside the scanned operands are untouched.
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, filepath.Join(dir, "a"))
+	if st.removed != 0 || st.unchanged != 1 {
+		t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 2 {
+		t.Fatalf("records = %q, want both retained", got)
+	}
+
+	// Rescanning the parent now removes the vanished file's record.
+	st = syncTree(t, db, dir)
+	if st.removed != 1 {
+		t.Fatalf("parent stats = %+v, want 1 removed", st)
+	}
+}
+
+func TestSyncScanRemovesNonRegular(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 10))
+	keep := writeFile(t, dir, "g", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Replace the file with a symlink: it is no longer walked, so its
+	// record must be deleted.
+	err := os.Remove(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Symlink(keep, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.removed != 1 || st.unchanged != 1 {
+		t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st)
+	}
+}
+
+func TestSyncScanOverlappingRoots(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	// A file reachable via two overlapping operands is deduplicated
+	// by path in the shared walk and processed once.
+	st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
+	if st != (scanStats{added: 1}) {
+		t.Fatalf("stats = %+v, want 1 added", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 1 {
+		t.Fatalf("records = %q, want exactly one", got)
+	}
+}
+
+func TestScanSkipsUniqueSizes(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	// Neither size is shared, so neither file is read: both records
+	// are written without hashes and no duplicates are reported.
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("stats = %+v, want 2 added", st)
+	}
+
+	recs := dbRecords(t, db)
+	for _, r := range recs {
+		if r.head != "" || r.tail != "" {
+			t.Errorf("%s: head = %q tail = %q, want unhashed",
+				r.path, r.head, r.tail)
+		}
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 0 {
+		t.Fatalf("groups = %+v, want none from unhashed records", groups)
+	}
+
+	// A new same-size file makes 500 a shared size: the next scan
+	// hashes both the new file and the previously unhashed unchanged
+	// one, and they group as duplicates.
+	c := writeFile(t, dir, "c.bin", pattern(1, 500))
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{added: 1, updated: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 added 1 updated 1 unchanged",
+			st)
+	}
+
+	groups := collectDupeGroups(dbRecords(t, db))
+	if len(groups) != 1 {
+		t.Fatalf("groups = %+v, want the a/c pair", groups)
+	}
+
+	if want := []string{a, c}; !slices.Equal(groups[0].paths, want) {
+		t.Fatalf("group paths = %q, want %q", groups[0].paths, want)
+	}
+}
+
+func TestTreesUnhashedNeverEqual(t *testing.T) {
+	t.Parallel()
+
+	// Two trees identical except for unhashed same-name, same-size
+	// files (possible when the trees were scanned separately) must not
+	// compare equal: unhashed content is unknown.
+	shared := pattern(1, 100)
+	recs := []scanRec{
+		{path: "/x/t1/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
+		{path: "/x/t2/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
+		{path: "/x/t1/u", size: 50},
+		{path: "/x/t2/u", size: 50},
+	}
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
+		t.Fatalf("tree groups = %d, want 0 (unhashed files differ)",
+			len(tg))
+	}
+}
+
+func TestScanHardlinksReadOnce(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 300))
+	b := filepath.Join(dir, "b.bin")
+
+	err := os.Link(a, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("stats = %+v, want 2 added", st)
+	}
+
+	// Both paths share the single read's hashes and group together.
+	recs := dbRecords(t, db)
+
+	ra := recordByPath(t, recs, a)
+	rb := recordByPath(t, recs, b)
+
+	if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail {
+		t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 1 {
+		t.Fatalf("groups = %+v, want the hardlink pair", groups)
+	}
+}
+
+func TestScanHardlinkRunFailsTogether(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 300))
+
+	err := os.Link(a, filepath.Join(dir, "b.bin"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unreadable inode: the run's single read fails, so both paths are
+	// skipped — proof that hard links are read once, not per path.
+	err = os.Chmod(a, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.skipped != 2 || st.added != 0 {
+		t.Fatalf("stats = %+v, want both hardlink paths skipped", st)
+	}
+}
+
+func TestHashRuns(t *testing.T) {
+	t.Parallel()
+
+	rec := func(path string, dev, ino uint64) fileRec {
+		return fileRec{path: path, dev: dev, ino: ino}
+	}
+
+	runs := hashRuns([]fileRec{
+		rec("/c", 1, 7),
+		rec("/a", 1, 7),
+		rec("/b", 1, 9),
+		// No inode identity: never merged, even with matching zeros.
+		rec("/z1", 0, 0),
+		rec("/z2", 0, 0),
+	})
+
+	got := make([][]string, 0, len(runs))
+	for _, run := range runs {
+		paths := make([]string, 0, len(run))
+		for _, r := range run {
+			paths = append(paths, r.path)
+		}
+
+		got = append(got, paths)
+	}
+
+	want := [][]string{{"/z1"}, {"/z2"}, {"/a", "/c"}, {"/b"}}
+	if !slices.EqualFunc(got, want, slices.Equal) {
+		t.Fatalf("runs = %v, want %v", got, want)
+	}
+}
+
+func TestPruneRoots(t *testing.T) {
+	t.Parallel()
+
+	// Duplicates and operands under other operands are dropped; /cc is
+	// not under /c (sibling with a shared prefix).
+	got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
+
+	want := []string{"/a", "/c", "/cc"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("pruneRoots = %q, want %q", got, want)
+	}
+}
+
+func TestReportsNeverTouchFilesystem(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	syncTree(t, db, dir)
+
+	// Remove the scanned tree entirely; the analysis must be
+	// unaffected because it reads the database alone.
+	err := os.RemoveAll(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	recs := dbRecords(t, db)
+
+	assertSmokeDupeGroups(t, dir, recs)
+	assertSmokeTreeGroups(t, dir, recs)
+}
+
+func TestUnderRoot(t *testing.T) {
+	t.Parallel()
+
+	const abRoot = "/a/b"
+
+	cases := []struct {
+		path string
+		root string
+		want bool
+	}{
+		{"/a/b/c", abRoot, true},
+		{abRoot, abRoot, true},
+		{"/a/bc", abRoot, false},
+		{"/a", abRoot, false},
+		{"/x/y", "/", true},
+		{"/", "/", true},
+	}
+	for _, c := range cases {
+		if got := underRoot(c.path, c.root); got != c.want {
+			t.Errorf("underRoot(%q, %q) = %v, want %v",
+				c.path, c.root, got, c.want)
+		}
+	}
+}

script/bootstrap

@@ -0,0 +1,79 @@
+#!/bin/sh
+# script/bootstrap: install all dependencies needed to build and develop
+# this repo. Idempotent: every install is guarded by a check so already
+# installed tools are skipped. Base tooling comes from nix, apt, brew,
+# or apk (detected in that order); assumes nothing is present.
+# golangci-lint is installed via `go install` pinned to the same version
+# the Dockerfile lint stage uses (never "latest").
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# Pinned versions, 2026-08-07 (same version as the Dockerfile lint stage).
+# golangci-lint v2.12.2
+GOLANGCI_LINT_REF="github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2"
+
+PKGMGR=""
+SUDO=""
+APT_UPDATED=""
+
+detect_pkgmgr() {
+    [ -n "$PKGMGR" ] && return 0
+    if command -v nix-env >/dev/null 2>&1; then
+        PKGMGR="nix"
+    elif command -v apt-get >/dev/null 2>&1; then
+        PKGMGR="apt"
+    elif command -v brew >/dev/null 2>&1; then
+        PKGMGR="brew"
+    elif command -v apk >/dev/null 2>&1; then
+        PKGMGR="apk"
+    else
+        echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
+        exit 1
+    fi
+    if [ "$PKGMGR" = "apt" ]; then
+        export DEBIAN_FRONTEND=noninteractive
+        if [ "$(id -u)" != "0" ]; then
+            SUDO="sudo"
+        fi
+    fi
+}
+
+# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
+pkg_install() {
+    detect_pkgmgr
+    case "$PKGMGR" in
+        nix) nix-env -iA "nixpkgs.$1" ;;
+        apt)
+            if [ -z "$APT_UPDATED" ]; then
+                $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update
+                APT_UPDATED=1
+            fi
+            $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2"
+            ;;
+        brew) brew install "$3" ;;
+        apk) apk add --no-cache "$4" ;;
+    esac
+}
+
+missing() {
+    ! command -v "$1" >/dev/null 2>&1
+}
+
+main() {
+    cd "$ROOT"
+
+    if missing git; then pkg_install git git git git; fi
+    if missing make; then pkg_install gnumake make make make; fi
+    if missing go; then pkg_install go golang go go; fi
+
+    # Lint tooling, pinned via go install (installs into
+    # "$(go env GOPATH)/bin"; ensure that is on your PATH).
+    if missing golangci-lint; then go install "$GOLANGCI_LINT_REF"; fi
+
+    go mod download
+
+    echo "bootstrap complete"
+}
+
+main "$@"

script/check

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/check: run all checks (test, lint, fmt-check). Our own
+# extension to scripts-to-rule-them-all. Must not modify any files.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/test"
+    "$SCRIPT_DIR/lint"
+    "$SCRIPT_DIR/fmt-check"
+}
+
+main "$@"

script/cibuild

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/cibuild: run the CI build. The Dockerfile runs make check (via
+# script/check), so a successful build implies all checks pass. The
+# Gitea workflow runs this on push.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    docker build .
+}
+
+main "$@"

script/docker

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/docker: build the Docker image tagged with the project name.
+# Identical in all repos; the tag comes from script/projectname.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    docker build -t "$("$SCRIPT_DIR/projectname")" .
+}
+
+main "$@"

script/fmt

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/fmt: format all files (writes).
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    gofmt -s -w .
+}
+
+main "$@"

script/fmt-check

@@ -0,0 +1,18 @@
+#!/bin/sh
+# script/fmt-check: check formatting (read-only). Same scope as
+# script/fmt, but fails instead of writing.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    files="$(gofmt -s -l .)"
+    if [ -n "$files" ]; then
+        echo "gofmt: files not formatted:" >&2
+        echo "$files" >&2
+        exit 1
+    fi
+}
+
+main "$@"

script/install-precommit

@@ -0,0 +1,20 @@
+#!/bin/sh
+# script/install-precommit: install the git pre-commit hook that runs
+# script/precommit. Our own extension to scripts-to-rule-them-all.
+# Hooks are shared between the main checkout and all worktrees, so
+# resolve the common git dir instead of assuming .git is a directory.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    hooks_dir="$(git rev-parse --git-common-dir)/hooks"
+    mkdir -p "$hooks_dir"
+    hook="$hooks_dir/pre-commit"
+    printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
+    chmod +x "$hook"
+    echo "pre-commit hook installed: runs script/precommit"
+}
+
+main "$@"

script/lint

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/lint: run the linter.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    golangci-lint run --config .golangci.yml ./...
+}
+
+main "$@"

script/precommit

@@ -0,0 +1,21 @@
+#!/bin/sh
+# script/precommit: run by the git pre-commit hook; fails the commit if
+# checks fail. Our own extension to scripts-to-rule-them-all. Go extra:
+# go mod tidy must be a no-op before the checks run.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go mod tidy
+    if ! git diff --exit-code -- go.mod go.sum; then
+        echo "precommit: go mod tidy changed go.mod/go.sum;" \
+            "stage the changes and retry" >&2
+        exit 1
+    fi
+    "$SCRIPT_DIR/check"
+}
+
+main "$@"

script/projectname

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/projectname: output the name of this project. Our own
+# extension to scripts-to-rule-them-all. Other scripts that need the
+# name (e.g. script/docker) call this, so they can stay identical
+# across all repos.
+set -eu
+
+main() {
+    echo "sfdupes"
+}
+
+main "$@"

script/setup

@@ -0,0 +1,13 @@
+#!/bin/sh
+# script/setup: set up the repo for development after a fresh clone:
+# installs dependencies (script/bootstrap) and the git pre-commit hook.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/bootstrap"
+    "$SCRIPT_DIR/install-precommit"
+}
+
+main "$@"

script/test

@@ -0,0 +1,17 @@
+#!/bin/sh
+# script/test: run the test suite. Reruns verbosely on failure so CI
+# logs show which test failed.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go test -timeout 30s -cover ./... || {
+        echo "--- Rerunning with -v for details ---"
+        go test -timeout 30s -v ./...
+        exit 1
+    }
+}
+
+main "$@"

trees.go

@@ -28,19 +28,14 @@ type treeNode struct {
 	totalSize int64
 }
 
-// runTrees implements the trees subcommand: it reads a scan stream from
+// runTrees implements the trees subcommand: it reads every record from
-// the named file (or stdin when absent or "-"), reconstructs the
+// the database, reconstructs the directory hierarchy from the record
-// directory hierarchy from the record paths, computes a Merkle-style
+// paths, computes a Merkle-style digest per directory, and prints
-// digest per directory, and prints maximal duplicate-tree groups as TSV
+// maximal duplicate-tree groups as TSV on stdout. It never touches the
-// on stdout. It never touches the scanned filesystem; its only I/O is
+// scanned filesystem; its only I/O is the database, stdout, and
-// the scan input, stdout, and stderr. args holds the positional
+// stderr.
-// arguments already validated by cobra (at most one).
+func runTrees() {
-func runTrees(args []string) {
+	recs := loadRecords()
-	in, name, closer := openScanInput(args)
-	defer closer()
-
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 
 	super, allDirs := buildHierarchy(recs)
 	super.compute()
@@ -78,9 +73,9 @@ func runTrees(args []string) {
 	}
 
 	fmt.Fprintf(os.Stderr,
-		"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
-		records, malformedNote(malformed), len(dupes), dupeTrees,
+			"%s reclaimable\n",
-		humanBytes(reclaimable))
+		len(recs), len(dupes), dupeTrees, humanBytes(reclaimable))
 }
 
 // buildHierarchy reconstructs the directory hierarchy from the record
@@ -121,9 +116,17 @@ func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
 			node.files = make(map[string]fileSig)
 		}
 
-		node.files[comps[len(comps)-1]] = fileSig{
+		sig := fileSig{size: r.size, head: r.head, tail: r.tail}
-			size: r.size, head: r.head, tail: r.tail,
+
+		// An unhashed record (its size was unique when last scanned)
+		// has unknown content: give it a signature no other file can
+		// share, so trees containing it never compare equal. Real
+		// heads are hex, so the NUL-prefixed form cannot collide.
+		if sig.head == "" {
+			sig.head = "unhashed\x00" + r.path
 		}
+
+		node.files[comps[len(comps)-1]] = sig
 	}
 
 	return super, allDirs