Merge branch 'golangci-v2.12.2': golangci-lint v2.12.2 and canonical config (closes #3)

Bumps the pinned golangci-lint from v2.12.1 to v2.12.2 in the Dockerfile
lint stage and script/bootstrap, and replaces .golangci.yml with the
canonical org-standard file (sha256
021cc83f4e6fc7c31b95b34b846723dfcf20b66b7baeea1dc40406e643346bcb).

The material change is that the lll, funlen, cyclop and dupl thresholds
move from the v1-style top-level linters-settings key, which
golangci-lint v2 silently ignores, to linters.settings, where they are
actually enforced. Independent review proved the migration claim with a
controlled experiment and confirmed the code passes the now-live
thresholds with zero findings.

Reviewed independently; make check and make docker both green.

.claude/settings.json

@@ -0,0 +1,5 @@
+{
+    "worktree": {
+        "bgIsolation": "none"
+    }
+}

.gitea/workflows/check.yml

@@ -6,4 +6,4 @@ jobs:
         steps:
             # actions/checkout v4.2.2, 2026-02-22
             - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-            - run: docker build .
+            - run: script/cibuild

.gitignore

@@ -28,6 +28,9 @@ node_modules/
 
 # Local scan data
 files.dat
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
 
 # Agent worktrees
 .claude/worktrees/

.golangci.yml

@@ -1,5 +1,9 @@
 version: "2"
 
+# Config schema uses the golangci-lint v2 layout (settings live under
+# linters.settings, not top-level linters-settings) so that the
+# thresholds below are actually applied by golangci-lint >= v2.
+
 run:
   timeout: 5m
   modules-download-mode: readonly
@@ -14,8 +18,7 @@ linters:
     - wsl              # Deprecated, replaced by wsl_v5
     - wrapcheck        # Too verbose for internal packages
     - varnamelen       # Short names like db, id are idiomatic Go
-
+  settings:
-linters-settings:
     lll:
       line-length: 88
     funlen:
@@ -27,6 +30,5 @@ linters-settings:
       threshold: 100
 
 issues:
-  exclude-use-default: false
   max-issues-per-linter: 0
   max-same-issues: 0

Dockerfile

@@ -1,6 +1,6 @@
 # Lint stage — fast feedback on formatting and lint issues
-# golangci/golangci-lint:v2.12.1, 2026-07-23
+# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
-FROM golangci/golangci-lint@sha256:c9843d374ca80ecbac86081ec4dd7fe2bb6187b03224f59a0cc2f80759e1845b AS lint
+FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
@@ -14,6 +14,14 @@ FROM golang@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c
 
 RUN apk add --no-cache make
 
+# We never build or run as root. Create an unprivileged user and point
+# HOME and the Go caches at its home so go build/test and golangci-lint
+# can write their caches when we drop to it below.
+RUN adduser -D -u 1000 builder
+ENV HOME=/home/builder
+ENV GOPATH=/home/builder/go
+ENV GOCACHE=/home/builder/.cache/go-build
+
 WORKDIR /src
 
 # Reuse the linter binary from the lint stage; the copy also forces
@@ -24,7 +32,14 @@ COPY go.mod go.sum ./
 RUN go mod download
 COPY . .
 
-# Fail the build unless the branch is green.
+# Hand the sources and caches to the unprivileged user, then drop root
+# before running any checks or builds.
+RUN chown -R builder:builder /src /home/builder
+USER builder
+
+# Fail the build unless the branch is green. Runs as non-root so the
+# permission-denied test paths are exercised legitimately (root would
+# bypass the chmod(0) the tests rely on).
 RUN make check
 
 RUN make build

Makefile

@@ -6,43 +6,44 @@ BINARY := sfdupes
 VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 LDFLAGS := -X main.Version=$(VERSION)
 
-.PHONY: all build test lint fmt fmt-check check docker hooks clean
+.PHONY: sfdupes build bootstrap setup test lint fmt fmt-check check docker hooks clean
 
-all: check build
+# Standard targets are thin shims; the implementations live in script/
+# per the scripts-to-rule-them-all pattern.
 
-build:
+# Default target: build the binary. Phony so go build (which has its
+# own build cache) always decides what to recompile.
+sfdupes:
 	go build -ldflags "$(LDFLAGS)" -o $(BINARY)
 
+build: sfdupes
+
+bootstrap:
+	@script/bootstrap
+
+setup:
+	@script/setup
+
 test:
-	@go test -timeout 30s -cover ./... || \
+	@script/test
-		{ echo "--- Rerunning with -v for details ---"; \
-		  go test -timeout 30s -v ./...; exit 1; }
 
 lint:
-	golangci-lint run --config .golangci.yml ./...
+	@script/lint
 
 fmt:
-	gofmt -s -w .
+	@script/fmt
 
 fmt-check:
-	@files="$$(gofmt -l -s .)"; if [ -n "$$files" ]; then \
+	@script/fmt-check
-		echo "gofmt: files not formatted:"; echo "$$files"; exit 1; fi
 
-check: test lint fmt-check
+check:
+	@script/check
 
 docker:
-	docker build -t $(BINARY) .
+	@script/docker
-
-# Hooks are shared between the main checkout and all worktrees, so
-# resolve the common git dir instead of assuming .git is a directory.
-HOOKS_DIR := $(shell git rev-parse --git-common-dir)/hooks
 
 hooks:
-	@printf '#!/bin/sh\nset -e\n' > $(HOOKS_DIR)/pre-commit
+	@script/install-precommit
-	@printf 'go mod tidy\ngo fmt ./...\n' >> $(HOOKS_DIR)/pre-commit
-	@printf 'git diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> $(HOOKS_DIR)/pre-commit
-	@printf 'make check\n' >> $(HOOKS_DIR)/pre-commit
-	@chmod +x $(HOOKS_DIR)/pre-commit
 
 clean:
 	rm -f $(BINARY) files.dat

README.md

@@ -12,7 +12,12 @@ SHA-256 of their first 1024 bytes, and identical SHA-256 of their last
 content (the middle of the file is never read); the intended use is
 finding duplicate downloads and duplicated directory trees on
 multi-terabyte ZFS servers where reading every byte is prohibitively
-expensive.
+expensive. `scan` maintains a persistent SQLite database of file
+signatures that survives between runs, so it can be run from cron and
+the reports can be generated at any time from the most recent scan.
+
+This tool was created by [@sneak](https://sneak.berlin) to scratch an itch,
+using Claude Code/Fable.
 
 This README is the complete and authoritative specification.
 
@@ -20,29 +25,41 @@ This README is the complete and authoritative specification.
 
 ```sh
 make build
-./sfdupes scan /srv > files.dat
+export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
-./sfdupes report files.dat > dupes.tsv
+./sfdupes scan /srv
-./sfdupes trees files.dat > dupetrees.tsv
+./sfdupes report > dupes.tsv
+./sfdupes trees > dupetrees.tsv
 ```
 
-`scan` walks one or more filesystem trees and emits one record per
+`scan` walks one or more filesystem trees and maintains one database
-regular file (path, size, mtime, head hash, tail hash). `report`
+record per regular file (path, size, mtime, head hash, tail hash). The
-ingests that stream and prints the file-level duplicates report.
+database persists between runs; a rescan only hashes files that are new
-`trees` ingests the same stream and prints the duplicate-tree report. A
+or changed, and removes records for files that no longer exist.
-missing/invalid subcommand — or a `scan` invocation with no `PATH`
+`report` reads the database and prints the file-level duplicates
-operand — prints a usage message and exits 2.
+report. `trees` reads the same database and prints the duplicate-tree
+report. A missing/invalid subcommand — or a `scan` invocation with no
+`PATH` operand — prints a usage message and exits 2.
+
+The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed
+anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a
+daily `sfdupes scan` cron job, with the reporting commands run
+interactively whenever needed; their results are as fresh as the last
+completed scan.
 
 ## Rationale
 
 Duplicate finders that hash entire files do not scale to the target
 environment: ~10 million files and ~150 TB on possibly slow or busy
-disks (a ZFS pool under resilver). Reading at most 2 KiB per file makes
+disks (a ZFS pool under resilver). Reading at most 2 KiB per file — and
-a full-filesystem sweep tractable, and the resulting scan stream is
+only from files whose size at least one other file shares, since a
-self-contained, so the expensive filesystem pass runs exactly once and
+size-unique file cannot be a duplicate — makes a full-filesystem sweep
-all analysis happens offline. The end goal is not individual files but
+tractable, and the signatures are kept in a persistent database, so
-whole duplicated trees — duplicate extractions, duplicate downloads,
+the expensive filesystem pass is incremental: a rescan re-hashes only
-copied project trees — which an operator can consider removing as a
+files whose recorded mtime or size changed, and all analysis happens
-unit.
+offline from the database alone. The end goal is
+not individual files but whole duplicated trees — duplicate
+extractions, duplicate downloads, copied project trees — which an
+operator can consider removing as a unit.
 
 ## Design
 
@@ -55,14 +72,19 @@ Goals, in order:
     removing an entire subtree at once. File-level duplicate detection is
     the foundation; tree-level detection is built on top of it.
 2.  **Never read full file contents.** At most 2 KiB is read per file
-    (first and last 1024 bytes). Scale target: ~10 million files, ~150 TB
+    (first and last 1024 bytes), and only files whose size at least
+    one other file shares are read at all — a size-unique file cannot
+    be a duplicate. Scale target: tens of millions of files, ~150 TB
     filesystem, possibly slow or busy disks (ZFS pool under resilver).
-    Holding the full file list in memory is acceptable; reading file
+    Holding one small record (path, size, mtime) per file in memory
-    contents beyond 2 KiB per file is not.
+    during a scan is acceptable; holding every file's hashes is not
-3.  **Scan once, analyze offline.** The expensive filesystem scan
+    (they stay in the database).
-    produces a self-contained stream; all analysis (`report`, `trees`)
+3.  **Scan incrementally, analyze offline.** The expensive filesystem
-    works from that stream alone and must never touch the scanned
+    scan maintains a persistent database; an unchanged file is never
-    filesystem again.
+    read again on a rescan. All analysis (`report`, `trees`) works from
+    the database alone and must never touch the scanned filesystem
+    again. `scan` is designed to be cronned; the reports run at any
+    time against the last completed scan.
 4.  **Clean stream separation.** Everything on stdout is machine-readable
     data. All progress, warnings, and summaries go to stderr. Never mix
     them.
@@ -71,54 +93,162 @@ Goals, in order:
 
 -   Language: Go (module `sneak.berlin/go/sfdupes`). Binary name:
     `sfdupes`.
--   Dependencies: standard library, `github.com/spf13/cobra` for the CLI,
+-   Dependencies: standard library, `github.com/spf13/cobra` for the
-    and **one progress-bar library**
+    CLI, **one progress-bar library**
-    (`github.com/schollz/progressbar/v3`). `github.com/spf13/viper` is
+    (`github.com/schollz/progressbar/v3`), and **one SQLite driver**
-    permitted if configuration-file support is ever needed, but is not
+    (`modernc.org/sqlite`, pure Go, so builds keep cgo disabled).
-    currently used. No other third-party deps.
+    `github.com/spf13/viper` is permitted if configuration-file support
+    is ever needed, but is not currently used. No other third-party
+    deps.
 -   Cross-compilation is not a concern. Builds run with cgo disabled (the
     `Makefile` exports `CGO_ENABLED=0`); the code must remain pure Go.
 -   Analysis modes (`report`, `trees`) must be deterministic: identical
-    input stream, identical output, regardless of record order.
+    database contents, identical output, regardless of the order in
+    which records were inserted.
 
 ### Subcommands
 
 Three subcommands, all implemented:
 
-1.  `scan` — walk the filesystem and emit one signature record per
+1.  `scan` — walk the filesystem and synchronize the database: one
-    regular file.
+    signature record per regular file.
-2.  `report` — file-level duplicate report from the scan stream.
+2.  `report` — file-level duplicate report from the database.
 3.  `trees` — tree-level duplicate report: reconstruct the directory
-    hierarchy from the scan stream, compute a Merkle-style digest per
+    hierarchy from the database records, compute a Merkle-style digest
-    directory, and report maximal groups of identical trees.
+    per directory, and report maximal groups of identical trees.
 
 ```
-sfdupes scan [--workers N] [-x] PATH... > files.dat
+sfdupes scan [--workers N] [-x] PATH...
-sfdupes report [files.dat|-] > dupes.tsv
+sfdupes report > dupes.tsv
-sfdupes trees [files.dat|-] > dupetrees.tsv
+sfdupes trees > dupetrees.tsv
 ```
 
+### Database
+
+All three subcommands operate on a single SQLite database file:
+
+-   Location: the value of the `SFDUPES_DATABASE` environment variable
+    when set and non-empty, otherwise `/var/lib/sfdupes/db.sqlite`.
+    There is no command-line flag.
+-   `scan` creates the database (and its parent directory) on first
+    use. `report` and `trees` require an existing database; a missing
+    database file is a fatal error (exit 1) telling the user to run
+    `scan` first.
+-   The database uses WAL journal mode and a busy timeout, so running a
+    report while a cron `scan` is in progress is safe. The filesystem
+    is authoritative; the database is an eventually-consistent
+    reflection of it. Hashed records are committed in batched
+    transactions while the scan is still running (keeping the WAL
+    small and letting concurrent reports observe progress), so a
+    report may see a scan's changes partially applied, and a scan
+    that dies partway leaves a valid database holding everything
+    hashed so far; the next scan skips those records and converges
+    toward the filesystem.
+-   Schema (`PRAGMA user_version` is the schema version, currently 1; a
+    database with any other version is a fatal error):
+
+    ```sql
+    CREATE TABLE files (
+        path  BLOB PRIMARY KEY,  -- absolute path, raw bytes
+        size  INTEGER NOT NULL,  -- bytes, from lstat
+        mtime INTEGER NOT NULL,  -- Unix seconds, from lstat
+        head  TEXT NOT NULL,     -- lowercase-hex SHA-256, first 1 KiB
+        tail  TEXT NOT NULL      -- lowercase-hex SHA-256, last 1 KiB
+    ) WITHOUT ROWID;
+    ```
+
+    Paths are stored as BLOBs because Unix paths are raw bytes, not
+    guaranteed UTF-8. `mtime` is used only for change detection; it is
+    not part of the duplicate key. `head` and `tail` are empty strings
+    when the file has never been hashed because its size was unique as
+    of the last scan that covered it; such records still define the
+    file for tree reconstruction but never participate in duplicate
+    groups.
+
 ### `scan` mode
 
 `scan` requires one or more `PATH` operands naming the trees to scan.
 There is no default path; invoking `scan` with no operand is a usage
 error (usage message on stderr, exit 2). An operand may be a directory
 or a regular file; an operand that does not exist is a fatal error
-(exit 1). Operands are walked in the order given; overlapping operands
+(exit 1). Because database records persist between runs and are keyed
-(one containing another) emit their common files once per operand, so
+by absolute path, each operand is resolved to an absolute, lexically
-callers should pass disjoint paths.
+cleaned path (symlinks are not resolved) before walking, so results do
+not depend on the working directory. All operands belong to a single
+scan and are enumerated concurrently: every operand seeds the shared
+walk worker pool. Overlapping operands are harmless — an operand that
+duplicates another or lies under another is dropped before walking,
+so every file is reached exactly once and produces one database
+record.
 
-`scan` runs **three sequential passes**, in this order, so that every
+`scan` synchronizes the database with the filesystem state under the
-expensive pass has an exact total for meaningful progress and ETA:
+scanned operands:
 
-1.  **walk** — recursively enumerate the tree under each `PATH` in
+-   Only a file whose size at least one other file shares is ever
-    turn, collecting the list of regular-file paths. Total unknown
+    read: a size-unique file cannot be a duplicate, so it is recorded
-    while running: show a live count, not a percentage.
+    without hashes (`head` and `tail` empty). The size census covers
-2.  **stat** — `lstat` every collected path, recording size and mtime.
+    every file walked this scan plus every database record outside
-3.  **hash** — for each file, read the first `min(1024, size)` bytes and
+    the scanned operands, so a possible duplicate of a separately
-    the last `min(1024, size)` bytes (the two reads overlap when
+    scanned tree is still recognized.
-    `size < 2048`; for `size == 0` hash the empty input) and compute the
+-   A file not yet in the database is inserted: hashed when its size
-    SHA-256 of each. Emit the output record.
+    is shared, without hashes otherwise.
+-   A file already in the database is **skipped without reading its
+    contents** when its lstat size equals the recorded size and its
+    lstat mtime is not newer than the recorded mtime. This is what
+    makes a daily rescan cheap. Exception: an unchanged file whose
+    record lacks hashes is hashed — and its record updated — once its
+    size becomes shared, so hashing deferred by size-uniqueness
+    happens as soon as it could matter.
+-   A file whose mtime is newer than recorded, or whose size differs,
+    is processed as if new: re-hashed, or recorded without hashes,
+    per the shared-size rule.
+-   A database record whose path lies under one of the scanned operands
+    but was not successfully processed this run is deleted. This
+    removes records for deleted files. It also removes records for
+    paths that failed to stat or hash this run: the database only ever
+    contains signatures verified by the most recent scan that covered
+    them (a subsequent successful scan re-adds such files).
+-   Database records outside the scanned operands are untouched, so
+    disjoint trees can be scanned on different schedules into the same
+    database.
+
+`scan` runs **three sequential phases over the whole scan**.
+Parallelism lives inside each phase; batched database writes begin
+during the hash phase:
+
+1.  **walk + stat** — enumerate the trees under all `PATH` operands
+    concurrently with the walk worker pool: every operand seeds the
+    shared queue, and each worker reads one directory at a time,
+    handing discovered subdirectories back to the queue and running
+    `lstat` on each regular file as it is discovered (while the
+    directory's metadata is still hot). Sequential directory
+    enumeration is metadata-latency-bound and takes hours at tens of
+    millions of files; per-directory parallelism is what makes the
+    walk tractable on large or busy pools. The walk builds the size
+    census and resolves unchanged already-hashed files on the fly;
+    every other file is carried to the hash phase as a (path, size,
+    mtime) record.
+2.  **hash** — with the census complete, each carried file's size
+    decides its fate. Size-unique files are never read: new or
+    changed ones are recorded without hashes in the update phase,
+    unchanged unhashed ones simply keep their records. Every file
+    with a shared size is hashed by the worker pool: read the first
+    `min(1024, size)` bytes and the last `min(1024, size)` bytes
+    (one read when `size <= 1024`, since the two windows coincide)
+    and compute the SHA-256 of each. Zero-length files have constant
+    hashes and are never opened. Files are hashed in **inode order**
+    (minimizing seeks on spinning disks), and paths that are hard
+    links to the same inode are **read once**, all sharing the one
+    result — a hard-link backup farm costs one read per inode, not
+    per path. The phase total counts actual reads, so progress and
+    ETA are meaningful. Completed records are committed in batched
+    transactions **while hashing runs**, so a scan interrupted after
+    hours keeps everything hashed so far and the next scan resumes
+    cheaply, skipping records already written.
+3.  **update** — commit the final partial batch, the hash-less
+    records for size-unique new and changed files, and the deletions
+    for records the scan did not verify (vanished files, plus paths
+    that failed to stat or hash).
 
 Rules for the walk:
 
@@ -134,52 +264,55 @@ Rules for the walk:
 -   On any per-path error (permission denied, file vanished between
     passes, unreadable): print a one-line warning to stderr, skip the
     path, and continue. Per-file errors never abort the run; the final
-    summary reports how many were skipped.
+    summary reports how many were skipped. As specified above, a
+    skipped path that has a database record from an earlier scan loses
+    that record; an unreadable directory subtree likewise loses its
+    records (accepted: the database mirrors what the latest scan could
+    actually verify).
 
-Concurrency: the stat and hash passes use a worker pool (`--workers`,
+Concurrency: the walk phase (which also stats files) and the hash
-default `runtime.NumCPU()`). The main goroutine owns stdout writing and
+phase each use a worker pool of `--workers` workers (default
-progress rendering; progress display must never block the workers.
+`runtime.NumCPU()`); the walk parallelizes across directories,
+hashing across files. Both phases are seek-bound on spinning disks,
+so raising `--workers` well past the core count can help on pools
+with many spindles. The main goroutine owns partitioning, database
+writes, and progress rendering; progress display must never block
+the workers.
 
-#### Output record format
+`scan` writes nothing to stdout. The summary line on stderr reports the
-
+files seen this run broken down by disposition, plus skips:
-One record per file on stdout, NUL-terminated (`\x00`), with
-tab-separated fields, **path last** so tabs or newlines embedded in
-paths cannot corrupt the record structure:
 
 ```
-<size>\t<mtime_unix>\t<sha256_first1k_hex>\t<sha256_last1k_hex>\t<path>\x00
+scan: 123456 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged), 3 skipped
 ```
 
--   `size`: decimal bytes, from the stat pass.
+(`removed` counts deleted database records, which are not part of the
--   `mtime_unix`: decimal Unix seconds. Informational only; not part of
+files-seen total.)
-    the duplicate key.
--   Hashes: lowercase hex, 64 chars each.
--   Record order is unspecified (workers complete out of order); the
-    analysis modes must not depend on ordering.
 
 ### `report` mode
 
-`report` reads the scan stream from the file named in its first
+`report` reads every record from the database and takes no positional
-positional argument, or from stdin if the argument is absent or `-`.
+arguments.
 
 **`report` must never touch the filesystem being analyzed.** It does not
 stat, open, or otherwise access any path that appears in the records; its
-only I/O is reading the scan file/stdin and writing stdout/stderr. It must
+only I/O is reading the database and writing stdout/stderr. It must
 produce identical output whether or not the scanned filesystem is still
 mounted.
 
 Processing:
 
--   Parse records; a record that does not have exactly 5 fields or whose
+-   Records without hashes (size-unique when last scanned) are
-    size is non-numeric is counted as malformed and skipped (warn once
+    excluded: their content is unknown, so they are never reported as
-    with the total malformed count in the summary, not per record).
+    duplicates.
--   Group records by the key `(size, head_hash, tail_hash)`.
+-   Group the remaining records by the key
+    `(size, head_hash, tail_hash)`.
 -   Every group with two or more paths is a duplicate group.
 -   Within each group, sort paths lexicographically (byte order). The
     first path is the group's `first`; every other path is a `dupe`.
 -   Order groups by size descending (biggest reclaimable space first),
     tie-broken by `first` path ascending. Output must be fully
-    deterministic for a given input.
+    deterministic for a given database state.
 
 #### Report output format
 
@@ -192,16 +325,16 @@ first	dupe	size
 /srv/a/big.iso	/srv/c/big-copy2.iso	4294967296
 ```
 
-Summary to stderr: records read, malformed count (if any), number of
+Summary to stderr: records read, number of duplicate groups, number of
-duplicate groups, number of dupe files, and total reclaimable bytes
+dupe files, and total reclaimable bytes (sum of `size` over all dupe
-(sum of `size` over all dupe rows) in human units.
+rows) in human units.
 
 ### `trees` mode
 
-`trees` reads the same scan stream as `report` (same argument handling,
+`trees` reads the same database as `report` (no positional arguments)
-same parsing and malformed-record rules) and reports **entire duplicate
+and reports **entire duplicate directory trees**: directories under
-directory trees**: directories under which the exact same set of relative
+which the exact same set of relative paths exists with the exact same
-paths exists with the exact same file signatures.
+file signatures.
 
 **`trees` must never touch the filesystem being analyzed** — the same
 rule as `report`. The directory hierarchy is reconstructed purely from
@@ -210,7 +343,10 @@ the paths in the records, split on `/`.
 Definitions:
 
 -   A file's **signature** is `(size, head_hash, tail_hash)` — mtime is
-    informational and excluded.
+    informational and excluded. An unhashed record (empty hashes) has
+    unknown content: its signature is treated as unique to that file,
+    so a tree containing an unhashed file never compares equal to any
+    other tree.
 -   A directory's **digest** is a SHA-256 Merkle digest computed
     bottom-up: serialize the directory's child entries — for a file
     child, its name and signature; for a subdirectory child, its name
@@ -222,10 +358,16 @@ Definitions:
     equal. Equal digests imply equal recursive file count and equal
     total byte size.
 
-Known limitation (accepted): only regular files that appear in the scan
+Known limitation (accepted): hard-linked paths are reported as
-stream define a tree. Empty directories are invisible, and a file skipped
+duplicates by `report` and count toward duplicate trees — their
-during the scan (e.g. permission error) in one copy but not the other
+content is genuinely identical — even though they share storage, so
-will make otherwise-identical trees compare as different.
+removing one reclaims no space. Inode identity is used during the
+scan to avoid redundant reads but is not persisted in the database.
+
+Known limitation (accepted): only regular files that appear in the
+database define a tree. Empty directories are invisible, and a file
+skipped during the scan (e.g. permission error) in one copy but not the
+other will make otherwise-identical trees compare as different.
 
 Processing:
 
@@ -254,32 +396,35 @@ first	dupe	files	size
 /srv/a/project	/srv/backup/project	3417	104857600
 ```
 
-Summary to stderr: records read, malformed count (if any), number of
+Summary to stderr: records read, number of duplicate-tree groups,
-duplicate-tree groups, number of dupe trees, and total reclaimable bytes
+number of dupe trees, and total reclaimable bytes (sum of `size` over
-(sum of `size` over all dupe rows) in human units.
+all dupe rows) in human units.
 
 ### Progress
 
-Use the progress-bar library for all scan-pass progress; rendering in the
+Use the progress-bar library for all scan progress; rendering in the
 style of `pv` is the model. All progress goes to stderr.
 
-Each scan pass gets its own bar. Required elements for the stat and hash
+Each phase gets its own display, rendered the moment the phase
-passes (known totals):
+starts — a scan must never look hung. Loading the existing-record
+index (`load`) and the walk have no known totals while running: show
+a live count, rate, and elapsed time (spinner-style, no percentage or
+ETA). The hash and update phases
+have exact totals — only files that actually need hashing appear in
+the hash total, so its ETA is meaningful. Required elements for the
+bars with known totals:
 
 -   elapsed time
 -   estimated time remaining
--   a `[m/n] x%` display (files processed / total files, percent)
+-   a `[m/n] x%` display (items processed / total items, percent)
--   current rate (files/s)
+-   current rate (items/s)
 
 Example shape (exact layout is flexible, content is not):
 
 ```
-hash: [1234567/9876543] 12% |████      | 8123 files/s elapsed 2:32 eta 17:54
+hash: [12345/98765] 12% |████      | 92 files/s elapsed 2:32 eta 17:54
 ```
 
-The walk pass has no known total: show a live file count and elapsed time
-(spinner-style, no percentage or ETA).
-
 Additional requirements:
 
 -   When stderr is not a TTY, do not emit ANSI redraws: print a plain
@@ -292,15 +437,18 @@ Additional requirements:
 ### Error handling and exit codes
 
 -   `0`: success, even if individual files were skipped with warnings.
--   `1`: fatal error (e.g., a `PATH` operand does not exist, cannot
+-   `1`: fatal error (e.g., a `PATH` operand does not exist, the
-    read the scan input, stdout write failure).
+    database cannot be created/opened/read/written, a missing database
--   `2`: usage error (including `scan` with no `PATH` operand).
+    for `report`/`trees`, stdout write failure).
+-   `2`: usage error (including `scan` with no `PATH` operand and
+    `report`/`trees` with any positional argument).
 
 ## Build
 
 The `Makefile` is the single source of truth for all operations:
 
--   `make build` — build the `sfdupes` binary (cgo disabled).
+-   `make` / `make build` — build the `sfdupes` binary (cgo
+    disabled); building is the default target.
 -   `make test` — run the test suite (30-second timeout; reruns with
     `-v` on failure).
 -   `make lint` — run `golangci-lint` with the repo config.
@@ -310,7 +458,7 @@ The `Makefile` is the single source of truth for all operations:
 -   `make docker` — build the Docker image, which runs `make check` as
     a build stage.
 -   `make hooks` — install the pre-commit hook.
--   `make clean` — remove the binary and any local `files.dat`.
+-   `make clean` — remove the binary and any legacy local `files.dat`.
 
 ### Definition of done
 
@@ -323,6 +471,7 @@ All of the following, run in this directory, must pass:
 
     ```sh
     d=$(mktemp -d)
+    export SFDUPES_DATABASE="$d/db.sqlite"
     mkdir -p "$d/a" "$d/b"
     head -c 2000 /dev/urandom > "$d/a/one.bin"
     cp "$d/a/one.bin" "$d/b/copy.bin"
@@ -339,28 +488,41 @@ All of the following, run in this directory, must pass:
     cp "$d/t1/sub/f2" "$d/t2/sub/f2"
     cp "$d/t1/f1" "$d/t3/f1"
     cp "$d/t1/sub/f2" "$d/t3/sub/f2renamed"
-    ./sfdupes scan "$d" > files.dat
+    ./sfdupes scan "$d"
-    ./sfdupes report files.dat
+    ./sfdupes report
-    ./sfdupes trees files.dat
+    ./sfdupes trees
+    # incremental behavior (scan a subtree; records elsewhere persist):
+    ./sfdupes scan "$d/a"                  # everything unchanged, nothing hashed
+    printf 'z' >> "$d/a/one.bin"           # modify: next scan re-hashes it
+    rm "$d/a/unique.bin"                   # delete: next scan removes its record
+    ./sfdupes scan "$d/a"                  # 1 updated, 1 removed
+    ./sfdupes report
     ```
 
-    Expected from `report`: `one.bin`/`copy.bin`/`copy2.bin` form one
+    (The scan database lives inside `$d` here purely for test hygiene;
-    group (two dupe rows, `first` is the lexicographically smallest
+    scanning `$d` therefore also records the SQLite file itself, which
-    path); `t1/f1`/`t2/f1`/`t3/f1` form one group; `t1/sub/f2`/
+    is harmless.)
-    `t2/sub/f2`/`t3/sub/f2renamed` form one group; `tiny1`/`tiny2` pair;
+
-    `empty1`/`empty2` pair; `unique.bin` and `tiny3` appear nowhere;
+    Expected from the first `report`: `one.bin`/`copy.bin`/`copy2.bin`
-    groups ordered by size descending; piping scan directly into report
+    form one group (two dupe rows, `first` is the lexicographically
-    (`./sfdupes scan "$d" | ./sfdupes report`) gives the same
+    smallest path); `t1/f1`/`t2/f1`/`t3/f1` form one group;
-    rows.
+    `t1/sub/f2`/ `t2/sub/f2`/`t3/sub/f2renamed` form one group;
+    `tiny1`/`tiny2` pair; `empty1`/`empty2` pair; `unique.bin` and
+    `tiny3` appear nowhere; groups ordered by size descending.
 
     Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe`
     `$d/t2`, 2 files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is
     suppressed as non-maximal (implied by the `t1`/`t2` group), and `t3`
     appears nowhere (its file set differs by name).
 
+    Expected from the second `report` (after the modify/delete rescan):
+    `one.bin` has left its group (its content changed), so
+    `copy.bin`/`copy2.bin` remain as one pair, and `unique.bin` is
+    gone from the database.
+
     The test suite automates this scenario (see `scan_test.go`), plus a
-    negative check: `report` and `trees` operate on the captured stream
+    negative check: `report` and `trees` operate on the database alone
-    alone and never touch the scanned filesystem.
+    and never touch the scanned filesystem.
 
 ## TODO
 
@@ -371,7 +533,9 @@ Tracked in [TODO.md](TODO.md).
 -   No full-content verification, no byte-for-byte compare, no deletion
     or linking of duplicates. The reports are advisory; acting on them is
     the user's job.
--   No persistence formats beyond the scan stream described above.
+-   No persistence beyond the SQLite database described above; no
+    export/import formats.
+-   No daemon or filesystem watcher; scheduling rescans is cron's job.
 
 ## License
 

TODO.md

@@ -14,10 +14,62 @@
 
 # Next Step
 
-- add a remote on git.eeqj.de and push (`main` plus tags)
+- convert Makefile targets to scripts-to-rule-them-all `script/`
+  entrypoints like the other managed repos
 
 # Completed Steps
 
+- update golangci-lint to v2.12.2 with the canonical config
+  (2026-08-07, branch `golangci-v2.12.2`): bumped the pinned linter
+  in the `Dockerfile` lint stage and `script/bootstrap` from v2.12.1
+  to v2.12.2, and replaced `.golangci.yml` with the canonical file —
+  the linter settings (`lll`, `funlen`, `cyclop`, `dupl` thresholds)
+  now live under `linters.settings` per the v2 schema, so they are
+  actually applied; no new lint findings surfaced
+- make the binary the default Make target (2026-07-24, branch
+  `make-default-target`): plain `make` now builds `sfdupes`
+  (previously it ran `check` plus `build`); `make build` remains as
+  an alias
+- scan-wide phases, concurrent operands, batched updates (2026-07-24,
+  branch `scan-wide-phases`): all operands seed the shared walk pool
+  and every pass runs once over the whole scan, so totals and ETAs
+  are scan-global; the per-operand walk/hash/update cycles and their
+  stderr announcements are gone; the update pass commits in batched
+  transactions — the filesystem is authoritative and the database an
+  eventually-consistent reflection, so scan-level atomicity is not
+  required
+- split the stat pass back out of the walk (2026-07-24, branch
+  `parallel-phases`): phases are strictly sequential again — walk,
+  stat, hash, update per operand — with parallelism only inside each
+  phase; the walk enumerates paths with per-directory workers and the
+  stat pass lstats them with per-file workers, restoring the exact
+  total/ETA stat bar
+- announce each operand on stderr before its passes (2026-07-24,
+  branch `scan-operand-progress`): with per-operand walk/hash/update
+  cycles, a multi-operand run (e.g. `scan /srv/*`) showed pass totals
+  that looked like the whole run's — an operator watching operand 3 of
+  14 hash 300k files concluded 20M files were being skipped
+- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass
+  was a single goroutine and took hours at ~20M files on a busy pool
+  (observed: 22M files in 4h on a ZFS server); it is now a
+  per-directory worker-pool traversal that records size/mtime during
+  the walk (folding away the separate stat pass, halving metadata
+  I/O), and each `PATH` operand commits in its own transaction so an
+  interrupted scan keeps completed operands
+
+- persistent scan database (2026-07-24, branch `persistent-database`):
+  `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure
+  Go, cgo stays disabled) keyed by absolute path that survives between
+  runs — a rescan hashes only new or changed files (by mtime/size),
+  deletes records for files vanished from under the scanned operands,
+  and leaves records outside them untouched, so `scan` can be cronned
+  daily; `report` and `trees` read the database (no positional
+  arguments) instead of a scan stream. Database at
+  `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`;
+  WAL journaling plus a single-transaction update keep a report run
+  during a scan safe
+- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag
+  `v0.0.1`, and push `main` plus tags (2026-07-23)
 - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
   `PATH...` operands via cobra flags replacing the `/srv` `-root`
   default; new `-x`/`--one-file-system` flag (GNU convention) to stop
@@ -30,9 +82,6 @@
 
 # Future Steps
 
-- convert Makefile targets to scripts-to-rule-them-all `script/`
-  entrypoints like the other managed repos
-- tag `v0.0.1` once the compliance branch is merged
 - possible later features (explicitly out of scope per README):
   full-content verification of candidates, removal-script helpers
 

db.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+
+	// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
+	// disabled.
+	_ "modernc.org/sqlite"
+)
+
+// defaultDatabasePath is where the persistent scan database lives when
+// SFDUPES_DATABASE is not set.
+const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite"
+
+// databaseEnv is the environment variable that overrides the database
+// path.
+const databaseEnv = "SFDUPES_DATABASE"
+
+// schemaVersion is the database schema version this build reads and
+// writes, stored in PRAGMA user_version.
+const schemaVersion = 1
+
+// dbDirPerm is the mode for a database parent directory created by
+// scan.
+const dbDirPerm = 0o755
+
+// createTableSQL is the schema applied to a fresh database. Paths are
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+const createTableSQL = `
+CREATE TABLE files (
+    path  BLOB PRIMARY KEY,
+    size  INTEGER NOT NULL,
+    mtime INTEGER NOT NULL,
+    head  TEXT NOT NULL,
+    tail  TEXT NOT NULL
+) WITHOUT ROWID
+`
+
+// upsertSQL inserts one file record, replacing any existing record for
+// the same path.
+const upsertSQL = `
+INSERT INTO files (path, size, mtime, head, tail)
+VALUES (?, ?, ?, ?, ?)
+ON CONFLICT (path) DO UPDATE SET
+    size = excluded.size, mtime = excluded.mtime,
+    head = excluded.head, tail = excluded.tail
+`
+
+// errNoDatabase reports a missing database file for report/trees.
+var errNoDatabase = errors.New(
+	"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
+
+// errSchemaVersion reports a database whose schema version this build
+// does not understand.
+var errSchemaVersion = errors.New("unsupported database schema version")
+
+// databasePath resolves the database location: SFDUPES_DATABASE when
+// set and non-empty, the compiled-in default otherwise.
+func databasePath() string {
+	if p := os.Getenv(databaseEnv); p != "" {
+		return p
+	}
+
+	return defaultDatabasePath
+}
+
+// openDB opens the SQLite database at path with WAL journaling and a
+// busy timeout, so a report can run while a cron scan is in progress.
+// It does not create or verify the schema.
+func openDB(path string) (*sql.DB, error) {
+	dsn := "file:" + path +
+		"?_pragma=busy_timeout(10000)" +
+		"&_pragma=journal_mode(WAL)" +
+		"&_pragma=synchronous(NORMAL)"
+
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, fmt.Errorf("open database %s: %w", path, err)
+	}
+
+	// A single connection avoids SQLITE_BUSY between this process's
+	// own connections; concurrency lives in the worker pools, not in
+	// parallel database access.
+	db.SetMaxOpenConns(1)
+
+	return db, nil
+}
+
+// openScanDatabase opens the database for the scan subcommand, creating
+// the file, its parent directory, and the schema as needed.
+func openScanDatabase(path string) (*sql.DB, error) {
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
+	if err != nil {
+		return nil, fmt.Errorf("create database directory: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = initSchema(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	return db, nil
+}
+
+// openReportDatabase opens an existing database for the report and
+// trees subcommands. A missing database file is an error directing the
+// user to run scan first; the schema version must match exactly.
+func openReportDatabase(path string) (*sql.DB, error) {
+	_, err := os.Stat(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, fmt.Errorf("%s: %w", path, errNoDatabase)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("database: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	v, err := userVersion(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	if v != schemaVersion {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: version %d, want %d: %w",
+			path, v, schemaVersion, errSchemaVersion)
+	}
+
+	return db, nil
+}
+
+// initSchema creates the schema on a fresh database and verifies the
+// schema version on an existing one.
+func initSchema(db *sql.DB) error {
+	v, err := userVersion(db)
+	if err != nil {
+		return err
+	}
+
+	switch v {
+	case 0:
+		return createSchema(db)
+	case schemaVersion:
+		return nil
+	default:
+		return fmt.Errorf("version %d, want %d: %w",
+			v, schemaVersion, errSchemaVersion)
+	}
+}
+
+// createSchema applies the schema to a fresh database and stamps the
+// schema version.
+func createSchema(db *sql.DB) error {
+	ctx := context.Background()
+
+	_, err := db.ExecContext(ctx, createTableSQL)
+	if err != nil {
+		return fmt.Errorf("create schema: %w", err)
+	}
+
+	_, err = db.ExecContext(ctx,
+		"PRAGMA user_version = "+strconv.Itoa(schemaVersion))
+	if err != nil {
+		return fmt.Errorf("set schema version: %w", err)
+	}
+
+	return nil
+}
+
+// userVersion reads the database's PRAGMA user_version.
+func userVersion(db *sql.DB) (int, error) {
+	var v int
+
+	err := db.QueryRowContext(context.Background(),
+		"PRAGMA user_version").Scan(&v)
+	if err != nil {
+		return 0, fmt.Errorf("read schema version: %w", err)
+	}
+
+	return v, nil
+}
+
+// loadFileRows reads every record from the files table.
+func loadFileRows(db *sql.DB) ([]scanRec, error) {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head, tail FROM files")
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var recs []scanRec
+
+	for rows.Next() {
+		var (
+			path []byte
+			r    scanRec
+		)
+
+		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail)
+		if err != nil {
+			return nil, fmt.Errorf("read record: %w", err)
+		}
+
+		r.path = string(path)
+		recs = append(recs, r)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	return recs, nil
+}
+
+// loadFileMeta streams every record's path, size, mtime, and whether
+// it carries hashes to fn. Scan change detection needs no hash
+// values, and skipping the hash columns keeps the scan's in-memory
+// index small on multi-million-file databases.
+func loadFileMeta(db *sql.DB,
+	fn func(path string, size, mtime int64, hashed bool),
+) error {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head <> '' FROM files")
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var (
+			path        []byte
+			size, mtime int64
+			hashed      int64
+		)
+
+		err = rows.Scan(&path, &size, &mtime, &hashed)
+		if err != nil {
+			return fmt.Errorf("read record: %w", err)
+		}
+
+		fn(string(path), size, mtime, hashed != 0)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	return nil
+}
+
+// updateBatchSize is the number of record changes committed per
+// transaction during the update pass. The filesystem is authoritative
+// and the database an eventually-consistent reflection of it, so
+// scan-level atomicity is not required; smaller transactions keep the
+// WAL small and let concurrent reports observe progress.
+const updateBatchSize = 10000
+
+// applyChanges writes one scan's database changes — upserts for new and
+// changed files, deletes for vanished ones — in batched transactions.
+// Progress is rendered on prog (one increment per change).
+func applyChanges(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	for batch := range slices.Chunk(upserts, updateBatchSize) {
+		err := applyBatch(db, batch, nil, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	for batch := range slices.Chunk(deletes, updateBatchSize) {
+		err := applyBatch(db, nil, batch, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// applyBatch commits one batch of upserts and deletes in a single
+// transaction.
+func applyBatch(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	ctx := context.Background()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+
+	defer func() { _ = tx.Rollback() }()
+
+	err = execUpserts(ctx, tx, upserts, prog)
+	if err != nil {
+		return err
+	}
+
+	err = execDeletes(ctx, tx, deletes, prog)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("commit: %w", err)
+	}
+
+	return nil
+}
+
+// execUpserts inserts or updates one record per new or changed file.
+func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, upsertSQL)
+	if err != nil {
+		return fmt.Errorf("prepare upsert: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, r := range upserts {
+		_, err = st.ExecContext(ctx,
+			[]byte(r.path), r.size, r.mtime, r.head, r.tail)
+		if err != nil {
+			return fmt.Errorf("upsert %s: %w", r.path, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}
+
+// execDeletes removes the records for paths no longer present.
+func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, "DELETE FROM files WHERE path = ?")
+	if err != nil {
+		return fmt.Errorf("prepare delete: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, p := range deletes {
+		_, err = st.ExecContext(ctx, []byte(p))
+		if err != nil {
+			return fmt.Errorf("delete %s: %w", p, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}

db_test.go

@@ -0,0 +1,224 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// testDBPath returns a database path inside a fresh temp dir.
+func testDBPath(t *testing.T) string {
+	t.Helper()
+
+	return filepath.Join(t.TempDir(), "db.sqlite")
+}
+
+// openTestDB creates a fresh scan database in a temp dir.
+func openTestDB(t *testing.T) *sql.DB {
+	t.Helper()
+
+	db, err := openScanDatabase(testDBPath(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() { _ = db.Close() })
+
+	return db
+}
+
+func TestDatabasePath(t *testing.T) {
+	t.Setenv(databaseEnv, "")
+
+	if got := databasePath(); got != defaultDatabasePath {
+		t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
+	}
+
+	t.Setenv(databaseEnv, "/custom/place.sqlite")
+
+	if got := databasePath(); got != "/custom/place.sqlite" {
+		t.Errorf("databasePath() = %q, want the env override", got)
+	}
+}
+
+func TestOpenScanDatabaseCreates(t *testing.T) {
+	t.Parallel()
+
+	// The parent directory does not exist yet; scan must create it.
+	path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("openScanDatabase: %v", err)
+	}
+
+	v, err := userVersion(db)
+	if err != nil || v != schemaVersion {
+		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
+	}
+
+	_ = db.Close()
+
+	// Reopening an existing database must succeed and find the schema.
+	db, err = openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+
+	defer func() { _ = db.Close() }()
+
+	recs, err := loadFileRows(db)
+	if err != nil || len(recs) != 0 {
+		t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err)
+	}
+}
+
+func TestOpenReportDatabaseMissing(t *testing.T) {
+	t.Parallel()
+
+	_, err := openReportDatabase(testDBPath(t))
+	if !errors.Is(err, errNoDatabase) {
+		t.Fatalf("err = %v, want errNoDatabase", err)
+	}
+}
+
+func TestOpenReportDatabaseVersionMismatch(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	_, err = openReportDatabase(path)
+	if !errors.Is(err, errSchemaVersion) {
+		t.Fatalf("err = %v, want errSchemaVersion", err)
+	}
+}
+
+func TestOpenReportDatabaseOK(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	db, err = openReportDatabase(path)
+	if err != nil {
+		t.Fatalf("openReportDatabase: %v", err)
+	}
+
+	_ = db.Close()
+}
+
+func TestApplyChangesRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// Paths may contain tabs and newlines; the database must store
+	// them byte-exactly.
+	recs := []scanRec{
+		{size: 2, mtime: 20, head: "h2", tail: "t2", path: "/a/tab\tnew\nline"},
+		{size: 1, mtime: 10, head: "h1", tail: "t1", path: "/a/x"},
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	slices.SortFunc(got, func(a, b scanRec) int {
+		return strings.Compare(a.path, b.path)
+	})
+
+	if !slices.Equal(got, recs) {
+		t.Fatalf("rows = %+v, want %+v", got, recs)
+	}
+
+	// An upsert for an existing path updates in place; a delete
+	// removes exactly its path.
+	upd := scanRec{size: 3, mtime: 30, head: "h3", tail: "t3", path: "/a/x"}
+
+	err = applyChanges(db, []scanRec{upd},
+		[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(got) != 1 || got[0] != upd {
+		t.Fatalf("rows = %+v, want just %+v", got, upd)
+	}
+}
+
+func TestApplyChangesBatching(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// One more change than the batch size, so the update spans two
+	// transactions.
+	n := updateBatchSize + 1
+
+	recs := make([]scanRec, 0, n)
+	for i := range n {
+		recs = append(recs, scanRec{
+			size: int64(i), mtime: 1, head: "h", tail: "t",
+			path: fmt.Sprintf("/batch/%07d", i),
+		})
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", int64(n)))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil || len(got) != n {
+		t.Fatalf("loadFileRows = %d rows, %v; want %d", len(got), err, n)
+	}
+
+	deletes := make([]string, 0, n)
+	for _, r := range recs {
+		deletes = append(deletes, r.path)
+	}
+
+	err = applyChanges(db, nil, deletes, newProgress("update", int64(n)))
+	if err != nil {
+		t.Fatalf("applyChanges deletes: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil || len(got) != 0 {
+		t.Fatalf("loadFileRows = %d rows, %v; want 0", len(got), err)
+	}
+}

go.mod

@@ -5,13 +5,22 @@ go 1.25.7
 require (
 	github.com/schollz/progressbar/v3 v3.19.1
 	github.com/spf13/cobra v1.10.2
+	modernc.org/sqlite v1.54.0
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.22 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/term v0.44.0 // indirect
+	modernc.org/libc v1.74.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -3,14 +3,28 @@ github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
+github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -23,10 +37,44 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
 golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
+modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog=
+modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

main.go

@@ -2,13 +2,15 @@
 // very large filesystems without reading full file contents. Files are
 // considered duplicates when they have identical size, identical SHA-256
 // of their first 1024 bytes, and identical SHA-256 of their last 1024
-// bytes.
+// bytes. scan maintains a persistent SQLite database of file signatures
+// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
+// reporting subcommands read.
 //
 // Usage:
 //
-//	sfdupes scan [--workers N] [-x] PATH... > files.dat
+//	sfdupes scan [--workers N] [-x] PATH...
-//	sfdupes report [files.dat|-] > dupes.tsv
+//	sfdupes report > dupes.tsv
-//	sfdupes trees [files.dat|-] > dupetrees.tsv
+//	sfdupes trees > dupetrees.tsv
 //
 // See README.md for the complete specification.
 package main
@@ -60,32 +62,32 @@ func main() {
 
 	scanCmd := &cobra.Command{
 		Use:   "scan [--workers N] [-x] PATH...",
-		Short: "Walk trees and emit one record per regular file on stdout",
+		Short: "Walk trees and synchronize the scan database",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(_ *cobra.Command, args []string) {
 			runScan(args, scanWorkers, scanOneFS)
 		},
 	}
 	scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(),
-		"concurrent workers for the stat and hash passes")
+		"concurrent workers for the walk and hash phases")
 	scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false,
 		"do not cross filesystem boundaries")
 
 	reportCmd := &cobra.Command{
-		Use:   "report [files.dat|-]",
+		Use:   "report",
-		Short: "Read a scan stream and print the file-level duplicates report",
+		Short: "Read the scan database and print the file-level duplicates report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runReport(args)
+			runReport()
 		},
 	}
 
 	treesCmd := &cobra.Command{
-		Use:   "trees [files.dat|-]",
+		Use:   "trees",
-		Short: "Read a scan stream and print the duplicate-tree report",
+		Short: "Read the scan database and print the duplicate-tree report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runTrees(args)
+			runTrees()
 		},
 	}
 

progress.go

@@ -38,7 +38,10 @@ func stderrIsTTY() bool {
 // stderr is not a TTY it emits no ANSI redraws: it prints a plain
 // one-line update no more often than every plainInterval.
 //
-// All methods must be called from the main goroutine only.
+// All methods must be called from the main goroutine only. A nil
+// *progress is a valid no-display receiver: every method is a no-op,
+// so batched database flushes during the streaming pass can reuse the
+// update-pass helpers without rendering anything.
 type progress struct {
 	label string
 	total int64 // -1 when unknown (walk pass)
@@ -62,6 +65,9 @@ func newProgress(label string, total int64) *progress {
 		progressbar.OptionSetItsString("files"),
 		progressbar.OptionSetElapsedTime(true),
 		progressbar.OptionThrottle(barThrottle),
+		// Render at zero immediately: a phase must be visible the
+		// moment it starts, even before its first item completes.
+		progressbar.OptionSetRenderBlankState(true),
 	}
 	if total >= 0 {
 		opts = append(opts,
@@ -82,6 +88,10 @@ func newProgress(label string, total int64) *progress {
 
 // increment records one completed item and refreshes the display.
 func (p *progress) increment() {
+	if p == nil {
+		return
+	}
+
 	p.count++
 	if p.bar != nil {
 		_ = p.bar.Add(1)
@@ -97,6 +107,10 @@ func (p *progress) increment() {
 
 // warnf prints a one-line warning to stderr without corrupting the bar.
 func (p *progress) warnf(format string, args ...any) {
+	if p == nil {
+		return
+	}
+
 	if p.bar != nil {
 		_ = p.bar.Clear()
 	}
@@ -106,6 +120,10 @@ func (p *progress) warnf(format string, args ...any) {
 
 // finish terminates the pass's display.
 func (p *progress) finish() {
+	if p == nil {
+		return
+	}
+
 	if p.bar != nil {
 		_ = p.bar.Finish()
 

report.go

@@ -2,106 +2,49 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"slices"
-	"strconv"
 	"strings"
 )
 
-// ioBufSize is the buffer size for the buffered scan-stream reader and
+// ioBufSize is the buffer size for the buffered stdout writers.
-// the buffered stdout writers.
 const ioBufSize = 1 << 20
 
-// recordFieldCount is the number of tab-separated fields in a scan
-// record: size, mtime, head hash, tail hash, path.
-const recordFieldCount = 5
-
-// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
-// to read scan records; a record longer than scanMaxRecordSize is a
-// fatal read error.
-const (
-	scanInitBufSize   = 64 << 10
-	scanMaxRecordSize = 4 << 20
-)
-
 // minGroupSize is the smallest number of members that makes a
 // duplicate group.
 const minGroupSize = 2
 
-// scanRec is one well-formed record parsed from a scan stream. The
+// scanRec is one file record from the database. The signature (size,
-// signature (size, head, tail) is the duplicate key; mtime is
+// head, tail) is the duplicate key; mtime is informational only and
-// informational and not retained.
+// used by scan for change detection.
 type scanRec struct {
 	size  int64
+	mtime int64
 	head  string
 	tail  string
 	path  string
 }
 
-// openScanInput resolves the analysis-mode input: the file named by the
+// loadRecords opens the database and reads every file record for the
-// single optional positional argument, or stdin when it is absent or
+// report and trees subcommands. Any database problem — including a
-// "-". The returned closer must be called when reading is done.
+// missing database — is fatal.
-func openScanInput(args []string) (io.Reader, string, func()) {
+func loadRecords() []scanRec {
-	if len(args) == 1 && args[0] != "-" {
+	dbPath := databasePath()
-		f, err := os.Open(args[0])
+
+	db, err := openReportDatabase(dbPath)
 	if err != nil {
-			fatalf("open %s: %v", args[0], err)
+		fatalf("%v", err)
 	}
 
-		return f, args[0], func() { _ = f.Close() }
+	defer func() { _ = db.Close() }()
-	}
 
-	return os.Stdin, "stdin", func() {}
+	recs, err := loadFileRows(db)
-}
-
-// parseScanStream reads every NUL-terminated record from in. A record
-// that does not have exactly 5 fields or whose size is non-numeric is
-// counted as malformed and skipped. Total records read is
-// len(recs) + malformed.
-func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
-	sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
-	sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
-	sc.Split(splitNUL)
-
-	var (
-		recs      []scanRec
-		malformed int
-	)
-
-	for sc.Scan() {
-		// The path is the last field and may itself contain tabs,
-		// so split into at most recordFieldCount fields.
-		fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
-		if len(fields) != recordFieldCount {
-			malformed++
-
-			continue
-		}
-
-		size, err := strconv.ParseInt(fields[0], 10, 64)
 	if err != nil {
-			malformed++
+		fatalf("database %s: %v", dbPath, err)
-
-			continue
 	}
 
-		recs = append(recs, scanRec{
+	return recs
-			size: size,
-			head: fields[2],
-			tail: fields[3],
-			path: fields[4],
-		})
-	}
-
-	err := sc.Err()
-	if err != nil {
-		fatalf("read %s: %v", name, err)
-	}
-
-	return recs, malformed
 }
 
 // dupeGroup is one set of candidate-duplicate files: identical size,
@@ -112,18 +55,12 @@ type dupeGroup struct {
 	paths []string
 }
 
-// runReport implements the report subcommand: it reads a scan stream
+// runReport implements the report subcommand: it reads every record
-// from the named file (or stdin when absent or "-") and prints the
+// from the database and prints the file-level duplicates report as TSV
-// file-level duplicates report as TSV on stdout. It never touches the
+// on stdout. It never touches the scanned filesystem; its only I/O is
-// scanned filesystem; its only I/O is the scan input, stdout, and
+// the database, stdout, and stderr.
-// stderr. args holds the positional arguments already validated by
+func runReport() {
-// cobra (at most one).
+	recs := loadRecords()
-func runReport(args []string) {
-	in, name, closer := openScanInput(args)
-	defer closer()
-
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 	dupes := collectDupeGroups(recs)
 
 	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
@@ -156,9 +93,9 @@ func runReport(args []string) {
 	}
 
 	fmt.Fprintf(os.Stderr,
-		"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
+		"report: %d records read, %d duplicate groups, %d dupe files, "+
-		records, malformedNote(malformed), len(dupes), dupeFiles,
+			"%s reclaimable\n",
-		humanBytes(reclaimable))
+		len(recs), len(dupes), dupeFiles, humanBytes(reclaimable))
 }
 
 // collectDupeGroups groups records by signature and returns every group
@@ -168,6 +105,13 @@ func collectDupeGroups(recs []scanRec) []dupeGroup {
 	groups := make(map[fileSig][]string)
 
 	for _, r := range recs {
+		// A record without hashes (its size was unique when last
+		// scanned) has unknown content and is never reported as a
+		// duplicate.
+		if r.head == "" {
+			continue
+		}
+
 		k := fileSig{size: r.size, head: r.head, tail: r.tail}
 		groups[k] = append(groups[k], r.path)
 	}
@@ -199,30 +143,6 @@ func collectDupeGroups(recs []scanRec) []dupeGroup {
 	return dupes
 }
 
-// malformedNote formats the optional malformed-record note for the
-// stderr summaries.
-func malformedNote(malformed int) string {
-	if malformed == 0 {
-		return ""
-	}
-
-	return fmt.Sprintf(" (%d malformed, skipped)", malformed)
-}
-
-// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
-// data without a terminator at EOF is returned as a final record.
-func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
-	if i := bytes.IndexByte(data, 0); i >= 0 {
-		return i + 1, data[:i], nil
-	}
-
-	if atEOF && len(data) > 0 {
-		return len(data), data, nil
-	}
-
-	return 0, nil, nil
-}
-
 // humanBytes formats a byte count in human units (binary prefixes).
 func humanBytes(n int64) string {
 	const unit = 1024

report_test.go

@@ -1,92 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"fmt"
 	"slices"
-	"strings"
 	"testing"
 )
 
-// mkRecord serializes one scan record in the on-the-wire format.
-func mkRecord(size, mtime int64, head, tail, path string) string {
-	return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00",
-		size, mtime, head, tail, path)
-}
-
-func TestSplitNULScanner(t *testing.T) {
-	t.Parallel()
-
-	sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail"))
-	sc.Split(splitNUL)
-
-	var got []string
-	for sc.Scan() {
-		got = append(got, sc.Text())
-	}
-
-	err := sc.Err()
-	if err != nil {
-		t.Fatalf("scanner error: %v", err)
-	}
-
-	want := []string{"a", "bb", "", "tail"}
-	if !slices.Equal(got, want) {
-		t.Fatalf("tokens = %q, want %q", got, want)
-	}
-}
-
-func TestParseScanStream(t *testing.T) {
-	t.Parallel()
-
-	stream := mkRecord(10, 1, "h1", "t1", "/a/x") +
-		"garbage-without-tabs\x00" +
-		"notanumber\t1\th\tt\t/a/bad\x00" +
-		mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") +
-		"30\t3\th3\tt3\t/trailing/no-nul"
-
-	recs, malformed := parseScanStream(strings.NewReader(stream), "test")
-
-	if malformed != 2 {
-		t.Errorf("malformed = %d, want 2", malformed)
-	}
-
-	want := []scanRec{
-		{size: 10, head: "h1", tail: "t1", path: "/a/x"},
-		{size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"},
-		{size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"},
-	}
-	if !slices.Equal(recs, want) {
-		t.Fatalf("recs = %+v, want %+v", recs, want)
-	}
-}
-
-func TestParseScanStreamPathWithNewline(t *testing.T) {
-	t.Parallel()
-
-	in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline"))
-
-	recs, malformed := parseScanStream(in, "test")
-	if malformed != 0 || len(recs) != 1 {
-		t.Fatalf("got %d recs, %d malformed, want 1, 0",
-			len(recs), malformed)
-	}
-
-	if recs[0].path != "/a/new\nline" {
-		t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline")
-	}
-}
-
-func TestParseScanStreamEmpty(t *testing.T) {
-	t.Parallel()
-
-	recs, malformed := parseScanStream(strings.NewReader(""), "test")
-	if len(recs) != 0 || malformed != 0 {
-		t.Fatalf("got %d recs, %d malformed, want 0, 0",
-			len(recs), malformed)
-	}
-}
-
 func TestCollectDupeGroups(t *testing.T) {
 	t.Parallel()
 
@@ -120,6 +38,22 @@ func TestCollectDupeGroups(t *testing.T) {
 	}
 }
 
+func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
+	t.Parallel()
+
+	// mtime is informational only; records differing only in mtime
+	// still group together.
+	recs := []scanRec{
+		{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
+		{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 {
+		t.Fatalf("len(groups) = %d, want 1", len(groups))
+	}
+}
+
 func TestCollectDupeGroupsTieBreak(t *testing.T) {
 	t.Parallel()
 
@@ -190,15 +124,3 @@ func TestHumanBytes(t *testing.T) {
 		}
 	}
 }
-
-func TestMalformedNote(t *testing.T) {
-	t.Parallel()
-
-	if got := malformedNote(0); got != "" {
-		t.Errorf("malformedNote(0) = %q, want empty", got)
-	}
-
-	if got := malformedNote(3); got != " (3 malformed, skipped)" {
-		t.Errorf("malformedNote(3) = %q", got)
-	}
-}

scan_test.go

@@ -1,15 +1,17 @@
 package main
 
 import (
-	"bytes"
+	"context"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
-	"io"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 // writeFile creates a file with the given content and returns its path.
@@ -98,6 +100,14 @@ func TestHashHeadTailErrors(t *testing.T) {
 		t.Error("no error for a missing file")
 	}
 
+	// A zero-length file has constant hashes and is never opened: even
+	// a missing path succeeds.
+	head, tail, err := hashHeadTail(filepath.Join(dir, "missing"), 0)
+	if err != nil || head != emptyHash || tail != emptyHash {
+		t.Errorf("empty: head=%q tail=%q err=%v, want constant hashes",
+			head, tail, err)
+	}
+
 	// A file that shrank between the stat and hash passes: reading at
 	// the stat-reported size must fail rather than emit wrong hashes.
 	p := writeFile(t, dir, "shrunk", []byte("tiny"))
@@ -108,14 +118,51 @@ func TestHashHeadTailErrors(t *testing.T) {
 	}
 }
 
-func TestWalkPass(t *testing.T) {
+// collectWalk runs a walk over roots and returns the emitted records
+// and the number of warning events.
+func collectWalk(t *testing.T, roots []string, oneFS bool,
+	workers int,
+) ([]fileRec, int) {
+	t.Helper()
+
+	var (
+		recs []fileRec
+		errs int
+	)
+
+	for ev := range startWalk(roots, oneFS, workers) {
+		if ev.fail {
+			errs++
+
+			continue
+		}
+
+		recs = append(recs, ev.rec)
+	}
+
+	return recs, errs
+}
+
+// walkedPaths returns the sorted paths of the walked records.
+func walkedPaths(recs []fileRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+func TestWalk(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
 	want := []string{
 		writeFile(t, dir, "a.txt", []byte("a")),
-		writeFile(t, dir, "sub/b.txt", []byte("b")),
+		writeFile(t, dir, "sub/b.txt", []byte("bb")),
-		writeFile(t, dir, "sub/deeper/c.txt", []byte("c")),
+		writeFile(t, dir, "sub/deeper/c.txt", []byte("ccc")),
 	}
 
 	slices.Sort(want)
@@ -129,19 +176,57 @@ func TestWalkPass(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	paths, errs := walkPass([]string{dir}, false)
+	recs, errs := collectWalk(t, []string{dir}, false, 4)
 	if errs != 0 {
 		t.Fatalf("errs = %d, want 0", errs)
 	}
 
-	slices.Sort(paths)
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
 
-	if !slices.Equal(paths, want) {
+	// The walk stats each file as it is discovered: every record must
-		t.Fatalf("paths = %q, want %q", paths, want)
+	// carry the real size and a plausible mtime.
+	for _, r := range recs {
+		if r.size < 1 || r.size > 3 {
+			t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
+		}
+
+		if r.mtime <= 0 {
+			t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
+		}
 	}
 }
 
-func TestWalkPassMultipleRoots(t *testing.T) {
+func TestWalkDeepAndWide(t *testing.T) {
+	t.Parallel()
+
+	// Exercise the dispatcher with more directories than workers and
+	// with nesting deeper than the worker count.
+	dir := t.TempDir()
+	deep := "deep" + strings.Repeat("/d", 30)
+
+	want := make([]string, 0, 41)
+	want = append(want, writeFile(t, dir, deep+"/f", []byte("x")))
+
+	for i := range 40 {
+		want = append(want, writeFile(t, dir,
+			fmt.Sprintf("wide/%02d/f", i), []byte("y")))
+	}
+
+	slices.Sort(want)
+
+	recs, errs := collectWalk(t, []string{dir}, false, 8)
+	if errs != 0 {
+		t.Fatalf("errs = %d, want 0", errs)
+	}
+
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
+		t.Fatalf("walked %d paths, want %d", len(got), len(want))
+	}
+}
+
+func TestWalkMultipleRoots(t *testing.T) {
 	t.Parallel()
 
 	rootA := t.TempDir()
@@ -152,18 +237,21 @@ func TestWalkPassMultipleRoots(t *testing.T) {
 		writeFile(t, rootB, "b1", []byte("3")),
 	}
 
-	paths, errs := walkPass([]string{rootA, rootB}, false)
+	slices.Sort(want)
+
+	// Operands are enumerated concurrently by the shared pool; order
+	// is unspecified.
+	recs, errs := collectWalk(t, []string{rootA, rootB}, false, 4)
 	if errs != 0 {
 		t.Fatalf("errs = %d, want 0", errs)
 	}
 
-	// Operands are walked in the order given.
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
-	if !slices.Equal(paths, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
-		t.Fatalf("paths = %q, want %q", paths, want)
 	}
 }
 
-func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
+func TestWalkFileAndSymlinkOperands(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
@@ -176,20 +264,20 @@ func TestWalkPassFileAndSymlinkOperands(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// A regular-file operand is emitted as itself.
+	// A regular-file operand is emitted as itself, statted.
-	paths, errs := walkPass([]string{f}, false)
+	recs, errs := collectWalk(t, []string{f}, false, 2)
-	if errs != 0 || !slices.Equal(paths, []string{f}) {
+	if errs != 0 || len(recs) != 1 || recs[0].path != f || recs[0].size != 4 {
-		t.Fatalf("file operand: paths = %q, errs = %d", paths, errs)
+		t.Fatalf("file operand: recs = %+v, errs = %d", recs, errs)
 	}
 
 	// A symlink operand is not followed and yields nothing.
-	paths, errs = walkPass([]string{link}, false)
+	recs, errs = collectWalk(t, []string{link}, false, 2)
-	if errs != 0 || len(paths) != 0 {
+	if errs != 0 || len(recs) != 0 {
-		t.Fatalf("symlink operand: paths = %q, errs = %d", paths, errs)
+		t.Fatalf("symlink operand: recs = %+v, errs = %d", recs, errs)
 	}
 }
 
-func TestWalkPassOneFilesystemSameFS(t *testing.T) {
+func TestWalkOneFilesystemSameFS(t *testing.T) {
 	t.Parallel()
 
 	// Everything in one filesystem: -x must not skip anything.
@@ -199,91 +287,38 @@ func TestWalkPassOneFilesystemSameFS(t *testing.T) {
 		writeFile(t, dir, "sub/deep/b", []byte("b")),
 	}
 
-	paths, errs := walkPass([]string{dir}, true)
+	recs, errs := collectWalk(t, []string{dir}, true, 4)
 	if errs != 0 {
 		t.Fatalf("errs = %d, want 0", errs)
 	}
 
-	slices.Sort(paths)
+	if got := walkedPaths(recs); !slices.Equal(got, want) {
-
+		t.Fatalf("paths = %q, want %q", got, want)
-	if !slices.Equal(paths, want) {
-		t.Fatalf("paths = %q, want %q", paths, want)
 	}
 }
 
-func TestDeviceOf(t *testing.T) {
+func TestDeviceOfInfo(t *testing.T) {
 	t.Parallel()
 
 	dir := t.TempDir()
 
-	dev1, ok1 := deviceOf(dir)
+	fi1, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	dev2, ok2 := deviceOf(dir)
+	fi2, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev1, ok1 := deviceOfInfo(fi1)
+
+	dev2, ok2 := deviceOfInfo(fi2)
 	if !ok1 || !ok2 || dev1 != dev2 {
-		t.Fatalf("deviceOf unstable: %d/%v vs %d/%v", dev1, ok1, dev2, ok2)
+		t.Fatalf("deviceOfInfo unstable: %d/%v vs %d/%v",
+			dev1, ok1, dev2, ok2)
 	}
-
-	if _, ok := deviceOf(filepath.Join(dir, "missing")); ok {
-		t.Fatal("deviceOf reported ok for a missing path")
-	}
-}
-
-func TestStatPass(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	a := writeFile(t, dir, "a", pattern(1, 10))
-	b := writeFile(t, dir, "b", pattern(2, 20))
-	missing := filepath.Join(dir, "vanished")
-
-	recs, errs := statPass([]string{a, b, missing}, 2)
-	if errs != 1 {
-		t.Fatalf("errs = %d, want 1 for the vanished file", errs)
-	}
-
-	slices.SortFunc(recs, func(x, y fileRec) int {
-		return strings.Compare(x.path, y.path)
-	})
-
-	if len(recs) != 2 || recs[0].size != 10 || recs[1].size != 20 {
-		t.Fatalf("recs = %+v, want sizes 10 and 20", recs)
-	}
-
-	if recs[0].mtime <= 0 || recs[1].mtime <= 0 {
-		t.Fatalf("recs = %+v, want positive mtimes", recs)
-	}
-}
-
-// captureStdout runs fn with os.Stdout redirected to a temp file and
-// returns everything fn wrote to it.
-func captureStdout(t *testing.T, fn func()) []byte {
-	t.Helper()
-
-	f, err := os.CreateTemp(t.TempDir(), "stdout")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	orig := os.Stdout
-	os.Stdout = f
-
-	defer func() { os.Stdout = orig }()
-
-	fn()
-
-	_, err = f.Seek(0, io.SeekStart)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	data, err := io.ReadAll(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_ = f.Close()
-
-	return data
 }
 
 // buildSmokeTree recreates the README smoke-test filesystem layout
@@ -316,45 +351,65 @@ func buildSmokeTree(t *testing.T) string {
 	return dir
 }
 
-// scanToRecords runs the walk, stat, and hash passes over dir and
+// smokeTreeFiles is the number of regular files buildSmokeTree creates.
-// parses the emitted stream back into records.
+const smokeTreeFiles = 15
-func scanToRecords(t *testing.T, dir string) []scanRec {
+
+// syncTree synchronizes the database with the given roots and returns
+// the scan stats.
+func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats {
 	t.Helper()
 
-	paths, walkErrs := walkPass([]string{dir}, false)
+	st, err := syncScan(db, roots, 4, false)
-	if walkErrs != 0 {
+	if err != nil {
-		t.Fatalf("walk errors: %d", walkErrs)
+		t.Fatalf("syncScan: %v", err)
 	}
 
-	recs, statErrs := statPass(paths, 4)
+	return st
-	if statErrs != 0 {
-		t.Fatalf("stat errors: %d", statErrs)
 }
 
-	var emitted, hashErrs int
+// dbRecords returns every record currently in the database.
+func dbRecords(t *testing.T, db *sql.DB) []scanRec {
+	t.Helper()
 
-	out := captureStdout(t, func() {
+	recs, err := loadFileRows(db)
-		emitted, hashErrs = hashPass(recs, 4)
+	if err != nil {
-	})
+		t.Fatal(err)
-
-	if emitted != len(paths) || hashErrs != 0 {
-		t.Fatalf("emitted %d of %d, %d hash errors",
-			emitted, len(paths), hashErrs)
 	}
 
-	parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe")
+	return recs
-	if malformed != 0 || len(parsed) != len(paths) {
-		t.Fatalf("parsed %d records, %d malformed, want %d, 0",
-			len(parsed), malformed, len(paths))
 }
 
-	return parsed
+// recordByPath finds the record with the given path.
+func recordByPath(t *testing.T, recs []scanRec, path string) scanRec {
+	t.Helper()
+
+	for _, r := range recs {
+		if r.path == path {
+			return r
+		}
 	}
 
-//nolint:paralleltest // redirects the process-wide os.Stdout
+	t.Fatalf("no record for %q", path)
-func TestScanPipeline(t *testing.T) {
+
-	dir := buildSmokeTree(t)
+	return scanRec{}
-	parsed := scanToRecords(t, dir)
+}
+
+// recordPaths returns the sorted paths of recs.
+func recordPaths(recs []scanRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+// assertSmokeDupeGroups checks the file-level duplicate groups for the
+// smoke tree rooted at dir.
+func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
 
 	groups := collectDupeGroups(parsed)
 	if len(groups) != 5 {
@@ -377,6 +432,12 @@ func TestScanPipeline(t *testing.T) {
 	if !slices.Equal(groups[0].paths, wantF1) {
 		t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
 	}
+}
+
+// assertSmokeTreeGroups checks the duplicate-tree groups for the smoke
+// tree rooted at dir.
+func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
 
 	super, dirs := buildHierarchy(parsed)
 	super.compute()
@@ -396,3 +457,451 @@ func TestScanPipeline(t *testing.T) {
 			tg[0][0].fileCount, tg[0][0].totalSize)
 	}
 }
+
+func TestScanPipeline(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: smokeTreeFiles}) {
+		t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles)
+	}
+
+	parsed := dbRecords(t, db)
+	if len(parsed) != smokeTreeFiles {
+		t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles)
+	}
+
+	assertSmokeDupeGroups(t, dir, parsed)
+	assertSmokeTreeGroups(t, dir, parsed)
+}
+
+func TestSyncScanUnchangedReuse(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("first scan stats = %+v, want 2 added", st)
+	}
+
+	// An immediate rescan reuses every record without reading file
+	// contents. Prove the files are not re-read by corrupting a stored
+	// hash and observing that it survives the rescan.
+	_, err := db.ExecContext(context.Background(),
+		"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{unchanged: 2}) {
+		t.Fatalf("rescan stats = %+v, want 2 unchanged", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
+		t.Fatalf("head = %q, want sentinel (file must not be re-read)",
+			r.head)
+	}
+}
+
+func TestSyncScanMtimeBump(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	syncTree(t, db, dir)
+
+	// Bump the mtime forward: the file must be re-hashed even though
+	// its size is unchanged.
+	future := time.Now().Add(time.Hour)
+
+	err := os.Chtimes(a, future, future)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{updated: 1}) {
+		t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
+		t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
+	}
+}
+
+func TestSyncScanAddRemove(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+	syncTree(t, db, dir)
+
+	// Add one file, remove another.
+	c := writeFile(t, dir, "c.bin", pattern(3, 700))
+
+	err := os.Remove(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 1, removed: 1, unchanged: 1}) {
+		t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged",
+			st)
+	}
+
+	want := []string{filepath.Join(dir, "b.bin"), c}
+	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+}
+
+func TestSyncScanSizeChange(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 100))
+
+	syncTree(t, db, dir)
+
+	// Rewrite with a different size but force the mtime back to the
+	// recorded value: the size mismatch alone must trigger a re-hash.
+	old := recordByPath(t, dbRecords(t, db), p)
+
+	writeFile(t, dir, "f", pattern(1, 200))
+
+	mt := time.Unix(old.mtime, 0)
+
+	err := os.Chtimes(p, mt, mt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.updated != 1 {
+		t.Fatalf("stats = %+v, want 1 updated", st)
+	}
+
+	if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 {
+		t.Fatalf("size = %d, want 200", got.size)
+	}
+}
+
+func TestSyncScanScope(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "a/keep", pattern(1, 10))
+
+	gone := writeFile(t, dir, "b/gone", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Deleting a file outside the rescanned root must not remove its
+	// record: records outside the scanned operands are untouched.
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, filepath.Join(dir, "a"))
+	if st.removed != 0 || st.unchanged != 1 {
+		t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 2 {
+		t.Fatalf("records = %q, want both retained", got)
+	}
+
+	// Rescanning the parent now removes the vanished file's record.
+	st = syncTree(t, db, dir)
+	if st.removed != 1 {
+		t.Fatalf("parent stats = %+v, want 1 removed", st)
+	}
+}
+
+func TestSyncScanRemovesNonRegular(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 10))
+	keep := writeFile(t, dir, "g", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Replace the file with a symlink: it is no longer walked, so its
+	// record must be deleted.
+	err := os.Remove(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Symlink(keep, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.removed != 1 || st.unchanged != 1 {
+		t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st)
+	}
+}
+
+func TestSyncScanOverlappingRoots(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	// A file reachable via two overlapping operands is deduplicated
+	// by path in the shared walk and processed once.
+	st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
+	if st != (scanStats{added: 1}) {
+		t.Fatalf("stats = %+v, want 1 added", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 1 {
+		t.Fatalf("records = %q, want exactly one", got)
+	}
+}
+
+func TestScanSkipsUniqueSizes(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	// Neither size is shared, so neither file is read: both records
+	// are written without hashes and no duplicates are reported.
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("stats = %+v, want 2 added", st)
+	}
+
+	recs := dbRecords(t, db)
+	for _, r := range recs {
+		if r.head != "" || r.tail != "" {
+			t.Errorf("%s: head = %q tail = %q, want unhashed",
+				r.path, r.head, r.tail)
+		}
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 0 {
+		t.Fatalf("groups = %+v, want none from unhashed records", groups)
+	}
+
+	// A new same-size file makes 500 a shared size: the next scan
+	// hashes both the new file and the previously unhashed unchanged
+	// one, and they group as duplicates.
+	c := writeFile(t, dir, "c.bin", pattern(1, 500))
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{added: 1, updated: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 added 1 updated 1 unchanged",
+			st)
+	}
+
+	groups := collectDupeGroups(dbRecords(t, db))
+	if len(groups) != 1 {
+		t.Fatalf("groups = %+v, want the a/c pair", groups)
+	}
+
+	if want := []string{a, c}; !slices.Equal(groups[0].paths, want) {
+		t.Fatalf("group paths = %q, want %q", groups[0].paths, want)
+	}
+}
+
+func TestTreesUnhashedNeverEqual(t *testing.T) {
+	t.Parallel()
+
+	// Two trees identical except for unhashed same-name, same-size
+	// files (possible when the trees were scanned separately) must not
+	// compare equal: unhashed content is unknown.
+	shared := pattern(1, 100)
+	recs := []scanRec{
+		{path: "/x/t1/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
+		{path: "/x/t2/f1", size: 100, head: hexSum(shared), tail: hexSum(shared)},
+		{path: "/x/t1/u", size: 50},
+		{path: "/x/t2/u", size: 50},
+	}
+
+	super, dirs := buildHierarchy(recs)
+	super.compute()
+
+	if tg := collectTreeGroups(dirs, super); len(tg) != 0 {
+		t.Fatalf("tree groups = %d, want 0 (unhashed files differ)",
+			len(tg))
+	}
+}
+
+func TestScanHardlinksReadOnce(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 300))
+	b := filepath.Join(dir, "b.bin")
+
+	err := os.Link(a, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("stats = %+v, want 2 added", st)
+	}
+
+	// Both paths share the single read's hashes and group together.
+	recs := dbRecords(t, db)
+
+	ra := recordByPath(t, recs, a)
+	rb := recordByPath(t, recs, b)
+
+	if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail {
+		t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
+	}
+
+	if groups := collectDupeGroups(recs); len(groups) != 1 {
+		t.Fatalf("groups = %+v, want the hardlink pair", groups)
+	}
+}
+
+func TestScanHardlinkRunFailsTogether(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 300))
+
+	err := os.Link(a, filepath.Join(dir, "b.bin"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unreadable inode: the run's single read fails, so both paths are
+	// skipped — proof that hard links are read once, not per path.
+	err = os.Chmod(a, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.skipped != 2 || st.added != 0 {
+		t.Fatalf("stats = %+v, want both hardlink paths skipped", st)
+	}
+}
+
+func TestHashRuns(t *testing.T) {
+	t.Parallel()
+
+	rec := func(path string, dev, ino uint64) fileRec {
+		return fileRec{path: path, dev: dev, ino: ino}
+	}
+
+	runs := hashRuns([]fileRec{
+		rec("/c", 1, 7),
+		rec("/a", 1, 7),
+		rec("/b", 1, 9),
+		// No inode identity: never merged, even with matching zeros.
+		rec("/z1", 0, 0),
+		rec("/z2", 0, 0),
+	})
+
+	got := make([][]string, 0, len(runs))
+	for _, run := range runs {
+		paths := make([]string, 0, len(run))
+		for _, r := range run {
+			paths = append(paths, r.path)
+		}
+
+		got = append(got, paths)
+	}
+
+	want := [][]string{{"/z1"}, {"/z2"}, {"/a", "/c"}, {"/b"}}
+	if !slices.EqualFunc(got, want, slices.Equal) {
+		t.Fatalf("runs = %v, want %v", got, want)
+	}
+}
+
+func TestPruneRoots(t *testing.T) {
+	t.Parallel()
+
+	// Duplicates and operands under other operands are dropped; /cc is
+	// not under /c (sibling with a shared prefix).
+	got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
+
+	want := []string{"/a", "/c", "/cc"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("pruneRoots = %q, want %q", got, want)
+	}
+}
+
+func TestReportsNeverTouchFilesystem(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	syncTree(t, db, dir)
+
+	// Remove the scanned tree entirely; the analysis must be
+	// unaffected because it reads the database alone.
+	err := os.RemoveAll(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	recs := dbRecords(t, db)
+
+	assertSmokeDupeGroups(t, dir, recs)
+	assertSmokeTreeGroups(t, dir, recs)
+}
+
+func TestUnderRoot(t *testing.T) {
+	t.Parallel()
+
+	const abRoot = "/a/b"
+
+	cases := []struct {
+		path string
+		root string
+		want bool
+	}{
+		{"/a/b/c", abRoot, true},
+		{abRoot, abRoot, true},
+		{"/a/bc", abRoot, false},
+		{"/a", abRoot, false},
+		{"/x/y", "/", true},
+		{"/", "/", true},
+	}
+	for _, c := range cases {
+		if got := underRoot(c.path, c.root); got != c.want {
+			t.Errorf("underRoot(%q, %q) = %v, want %v",
+				c.path, c.root, got, c.want)
+		}
+	}
+}

script/bootstrap

@@ -0,0 +1,79 @@
+#!/bin/sh
+# script/bootstrap: install all dependencies needed to build and develop
+# this repo. Idempotent: every install is guarded by a check so already
+# installed tools are skipped. Base tooling comes from nix, apt, brew,
+# or apk (detected in that order); assumes nothing is present.
+# golangci-lint is installed via `go install` pinned to the same version
+# the Dockerfile lint stage uses (never "latest").
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# Pinned versions, 2026-08-07 (same version as the Dockerfile lint stage).
+# golangci-lint v2.12.2
+GOLANGCI_LINT_REF="github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2"
+
+PKGMGR=""
+SUDO=""
+APT_UPDATED=""
+
+detect_pkgmgr() {
+    [ -n "$PKGMGR" ] && return 0
+    if command -v nix-env >/dev/null 2>&1; then
+        PKGMGR="nix"
+    elif command -v apt-get >/dev/null 2>&1; then
+        PKGMGR="apt"
+    elif command -v brew >/dev/null 2>&1; then
+        PKGMGR="brew"
+    elif command -v apk >/dev/null 2>&1; then
+        PKGMGR="apk"
+    else
+        echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
+        exit 1
+    fi
+    if [ "$PKGMGR" = "apt" ]; then
+        export DEBIAN_FRONTEND=noninteractive
+        if [ "$(id -u)" != "0" ]; then
+            SUDO="sudo"
+        fi
+    fi
+}
+
+# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
+pkg_install() {
+    detect_pkgmgr
+    case "$PKGMGR" in
+        nix) nix-env -iA "nixpkgs.$1" ;;
+        apt)
+            if [ -z "$APT_UPDATED" ]; then
+                $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update
+                APT_UPDATED=1
+            fi
+            $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2"
+            ;;
+        brew) brew install "$3" ;;
+        apk) apk add --no-cache "$4" ;;
+    esac
+}
+
+missing() {
+    ! command -v "$1" >/dev/null 2>&1
+}
+
+main() {
+    cd "$ROOT"
+
+    if missing git; then pkg_install git git git git; fi
+    if missing make; then pkg_install gnumake make make make; fi
+    if missing go; then pkg_install go golang go go; fi
+
+    # Lint tooling, pinned via go install (installs into
+    # "$(go env GOPATH)/bin"; ensure that is on your PATH).
+    if missing golangci-lint; then go install "$GOLANGCI_LINT_REF"; fi
+
+    go mod download
+
+    echo "bootstrap complete"
+}
+
+main "$@"

script/check

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/check: run all checks (test, lint, fmt-check). Our own
+# extension to scripts-to-rule-them-all. Must not modify any files.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/test"
+    "$SCRIPT_DIR/lint"
+    "$SCRIPT_DIR/fmt-check"
+}
+
+main "$@"

script/cibuild

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/cibuild: run the CI build. The Dockerfile runs make check (via
+# script/check), so a successful build implies all checks pass. The
+# Gitea workflow runs this on push.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    docker build .
+}
+
+main "$@"

script/docker

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/docker: build the Docker image tagged with the project name.
+# Identical in all repos; the tag comes from script/projectname.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    docker build -t "$("$SCRIPT_DIR/projectname")" .
+}
+
+main "$@"

script/fmt

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/fmt: format all files (writes).
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    gofmt -s -w .
+}
+
+main "$@"

script/fmt-check

@@ -0,0 +1,18 @@
+#!/bin/sh
+# script/fmt-check: check formatting (read-only). Same scope as
+# script/fmt, but fails instead of writing.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    files="$(gofmt -s -l .)"
+    if [ -n "$files" ]; then
+        echo "gofmt: files not formatted:" >&2
+        echo "$files" >&2
+        exit 1
+    fi
+}
+
+main "$@"

script/install-precommit

@@ -0,0 +1,20 @@
+#!/bin/sh
+# script/install-precommit: install the git pre-commit hook that runs
+# script/precommit. Our own extension to scripts-to-rule-them-all.
+# Hooks are shared between the main checkout and all worktrees, so
+# resolve the common git dir instead of assuming .git is a directory.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    hooks_dir="$(git rev-parse --git-common-dir)/hooks"
+    mkdir -p "$hooks_dir"
+    hook="$hooks_dir/pre-commit"
+    printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
+    chmod +x "$hook"
+    echo "pre-commit hook installed: runs script/precommit"
+}
+
+main "$@"

script/lint

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/lint: run the linter.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    golangci-lint run --config .golangci.yml ./...
+}
+
+main "$@"

script/precommit

@@ -0,0 +1,21 @@
+#!/bin/sh
+# script/precommit: run by the git pre-commit hook; fails the commit if
+# checks fail. Our own extension to scripts-to-rule-them-all. Go extra:
+# go mod tidy must be a no-op before the checks run.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go mod tidy
+    if ! git diff --exit-code -- go.mod go.sum; then
+        echo "precommit: go mod tidy changed go.mod/go.sum;" \
+            "stage the changes and retry" >&2
+        exit 1
+    fi
+    "$SCRIPT_DIR/check"
+}
+
+main "$@"

script/projectname

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/projectname: output the name of this project. Our own
+# extension to scripts-to-rule-them-all. Other scripts that need the
+# name (e.g. script/docker) call this, so they can stay identical
+# across all repos.
+set -eu
+
+main() {
+    echo "sfdupes"
+}
+
+main "$@"

script/setup

@@ -0,0 +1,13 @@
+#!/bin/sh
+# script/setup: set up the repo for development after a fresh clone:
+# installs dependencies (script/bootstrap) and the git pre-commit hook.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/bootstrap"
+    "$SCRIPT_DIR/install-precommit"
+}
+
+main "$@"

script/test

@@ -0,0 +1,17 @@
+#!/bin/sh
+# script/test: run the test suite. Reruns verbosely on failure so CI
+# logs show which test failed.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go test -timeout 30s -cover ./... || {
+        echo "--- Rerunning with -v for details ---"
+        go test -timeout 30s -v ./...
+        exit 1
+    }
+}
+
+main "$@"

trees.go

@@ -28,19 +28,14 @@ type treeNode struct {
 	totalSize int64
 }
 
-// runTrees implements the trees subcommand: it reads a scan stream from
+// runTrees implements the trees subcommand: it reads every record from
-// the named file (or stdin when absent or "-"), reconstructs the
+// the database, reconstructs the directory hierarchy from the record
-// directory hierarchy from the record paths, computes a Merkle-style
+// paths, computes a Merkle-style digest per directory, and prints
-// digest per directory, and prints maximal duplicate-tree groups as TSV
+// maximal duplicate-tree groups as TSV on stdout. It never touches the
-// on stdout. It never touches the scanned filesystem; its only I/O is
+// scanned filesystem; its only I/O is the database, stdout, and
-// the scan input, stdout, and stderr. args holds the positional
+// stderr.
-// arguments already validated by cobra (at most one).
+func runTrees() {
-func runTrees(args []string) {
+	recs := loadRecords()
-	in, name, closer := openScanInput(args)
-	defer closer()
-
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 
 	super, allDirs := buildHierarchy(recs)
 	super.compute()
@@ -78,9 +73,9 @@ func runTrees(args []string) {
 	}
 
 	fmt.Fprintf(os.Stderr,
-		"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
-		records, malformedNote(malformed), len(dupes), dupeTrees,
+			"%s reclaimable\n",
-		humanBytes(reclaimable))
+		len(recs), len(dupes), dupeTrees, humanBytes(reclaimable))
 }
 
 // buildHierarchy reconstructs the directory hierarchy from the record
@@ -121,9 +116,17 @@ func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
 			node.files = make(map[string]fileSig)
 		}
 
-		node.files[comps[len(comps)-1]] = fileSig{
+		sig := fileSig{size: r.size, head: r.head, tail: r.tail}
-			size: r.size, head: r.head, tail: r.tail,
+
+		// An unhashed record (its size was unique when last scanned)
+		// has unknown content: give it a signature no other file can
+		// share, so trees containing it never compare equal. Real
+		// heads are hex, so the NUL-prefixed form cannot collide.
+		if sig.head == "" {
+			sig.head = "unhashed\x00" + r.path
 		}
+
+		node.files[comps[len(comps)-1]] = sig
 	}
 
 	return super, allDirs