Implement persistent SQLite scan database

scan now synchronizes a database that survives between runs
(SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of
emitting a stream: operands are resolved to absolute paths, unchanged
files (same size, mtime not newer than recorded) are never re-read, new
and changed files are hashed, and records under the scanned operands
that were not verified this run are deleted; records outside the
operands are untouched. All changes commit in a single transaction, and
WAL journaling with a busy timeout keeps a report run during a cron
scan safe.

report and trees read the database (no positional arguments); the
NUL-terminated stream format, its parser, and the malformed-record
handling are gone. The driver is modernc.org/sqlite (pure Go), so
builds keep cgo disabled.

.gitignore

@@ -28,6 +28,9 @@ node_modules/
 
 # Local scan data
 files.dat
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
 
 # Agent worktrees
 .claude/worktrees/

TODO.md

@@ -14,44 +14,22 @@
 
 # Next Step
 
-- persistent scan database (branch `persistent-database`): `scan`
+- convert Makefile targets to scripts-to-rule-them-all `script/`
-  maintains a SQLite database that survives between runs so it can be
+  entrypoints like the other managed repos
-  cronned daily; `report` and `trees` read the database instead of a
-  scan stream. Database path defaults to `/var/lib/sfdupes/db.sqlite`,
-  overridable via `SFDUPES_DATABASE`. Plan:
-    - [x] update `README.md` (the authoritative spec) for the database:
-          new Database section, revised `scan`/`report`/`trees` modes,
-          dependency list (`modernc.org/sqlite`, pure Go, cgo stays
-          disabled), exit codes, smoke test with incremental steps
-    - [ ] add `modernc.org/sqlite` (hash-pinned via `go.sum`)
-    - [ ] `db.go`: path resolution (`SFDUPES_DATABASE` env, default
-          `/var/lib/sfdupes/db.sqlite`), open/create with WAL +
-          busy-timeout pragmas and `PRAGMA user_version` schema check,
-          `files` table (path BLOB primary key, size, mtime, head,
-          tail), load-all-rows, and single-transaction apply of
-          upserts/deletes
-    - [ ] `scan.go`: resolve operands to absolute paths; diff stat
-          results against loaded rows (reuse hashes when size matches
-          and mtime is not newer); hash only new/changed files; delete
-          rows under the scanned operands not successfully processed;
-          leave rows outside the operands untouched; new summary line
-          (added/updated/removed/unchanged/skipped); "update" progress
-          pass for the database write
-    - [ ] `report.go`/`trees.go`: read records from the database (no
-          positional args, no stream parsing); drop the NUL-stream
-          parser and malformed-record handling
-    - [ ] `main.go`: updated usage strings; `report`/`trees` take no
-          args (usage error otherwise)
-    - [ ] tests: db open/env-override/schema tests; incremental scan
-          tests (add, mtime update, delete, unchanged-hash-reuse,
-          out-of-scope rows untouched, error paths); rework pipeline
-          test to go through the database; keep walk/stat/hash/grouping
-          unit tests
-    - [ ] `.gitignore`: local `*.sqlite` artifacts
-    - [ ] `make check`, `make docker`, and the README smoke test pass
 
 # Completed Steps
 
+- persistent scan database (2026-07-24, branch `persistent-database`):
+  `scan` now maintains a SQLite database (`modernc.org/sqlite`, pure
+  Go, cgo stays disabled) keyed by absolute path that survives between
+  runs — a rescan hashes only new or changed files (by mtime/size),
+  deletes records for files vanished from under the scanned operands,
+  and leaves records outside them untouched, so `scan` can be cronned
+  daily; `report` and `trees` read the database (no positional
+  arguments) instead of a scan stream. Database at
+  `/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`;
+  WAL journaling plus a single-transaction update keep a report run
+  during a scan safe
 - add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag
   `v0.0.1`, and push `main` plus tags (2026-07-23)
 - `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
@@ -66,8 +44,6 @@
 
 # Future Steps
 
-- convert Makefile targets to scripts-to-rule-them-all `script/`
-  entrypoints like the other managed repos
 - possible later features (explicitly out of scope per README):
   full-content verification of candidates, removal-script helpers
 

db.go

@@ -0,0 +1,319 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
+	// disabled.
+	_ "modernc.org/sqlite"
+)
+
+// defaultDatabasePath is where the persistent scan database lives when
+// SFDUPES_DATABASE is not set.
+const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite"
+
+// databaseEnv is the environment variable that overrides the database
+// path.
+const databaseEnv = "SFDUPES_DATABASE"
+
+// schemaVersion is the database schema version this build reads and
+// writes, stored in PRAGMA user_version.
+const schemaVersion = 1
+
+// dbDirPerm is the mode for a database parent directory created by
+// scan.
+const dbDirPerm = 0o755
+
+// createTableSQL is the schema applied to a fresh database. Paths are
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+const createTableSQL = `
+CREATE TABLE files (
+    path  BLOB PRIMARY KEY,
+    size  INTEGER NOT NULL,
+    mtime INTEGER NOT NULL,
+    head  TEXT NOT NULL,
+    tail  TEXT NOT NULL
+) WITHOUT ROWID
+`
+
+// upsertSQL inserts one file record, replacing any existing record for
+// the same path.
+const upsertSQL = `
+INSERT INTO files (path, size, mtime, head, tail)
+VALUES (?, ?, ?, ?, ?)
+ON CONFLICT (path) DO UPDATE SET
+    size = excluded.size, mtime = excluded.mtime,
+    head = excluded.head, tail = excluded.tail
+`
+
+// errNoDatabase reports a missing database file for report/trees.
+var errNoDatabase = errors.New(
+	"no database (run \"sfdupes scan\" first, or set " + databaseEnv + ")")
+
+// errSchemaVersion reports a database whose schema version this build
+// does not understand.
+var errSchemaVersion = errors.New("unsupported database schema version")
+
+// databasePath resolves the database location: SFDUPES_DATABASE when
+// set and non-empty, the compiled-in default otherwise.
+func databasePath() string {
+	if p := os.Getenv(databaseEnv); p != "" {
+		return p
+	}
+
+	return defaultDatabasePath
+}
+
+// openDB opens the SQLite database at path with WAL journaling and a
+// busy timeout, so a report can run while a cron scan is in progress.
+// It does not create or verify the schema.
+func openDB(path string) (*sql.DB, error) {
+	dsn := "file:" + path +
+		"?_pragma=busy_timeout(10000)" +
+		"&_pragma=journal_mode(WAL)" +
+		"&_pragma=synchronous(NORMAL)"
+
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, fmt.Errorf("open database %s: %w", path, err)
+	}
+
+	// A single connection avoids SQLITE_BUSY between this process's
+	// own connections; concurrency lives in the worker pools, not in
+	// parallel database access.
+	db.SetMaxOpenConns(1)
+
+	return db, nil
+}
+
+// openScanDatabase opens the database for the scan subcommand, creating
+// the file, its parent directory, and the schema as needed.
+func openScanDatabase(path string) (*sql.DB, error) {
+	err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
+	if err != nil {
+		return nil, fmt.Errorf("create database directory: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = initSchema(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	return db, nil
+}
+
+// openReportDatabase opens an existing database for the report and
+// trees subcommands. A missing database file is an error directing the
+// user to run scan first; the schema version must match exactly.
+func openReportDatabase(path string) (*sql.DB, error) {
+	_, err := os.Stat(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, fmt.Errorf("%s: %w", path, errNoDatabase)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("database: %w", err)
+	}
+
+	db, err := openDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	v, err := userVersion(db)
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: %w", path, err)
+	}
+
+	if v != schemaVersion {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("database %s: version %d, want %d: %w",
+			path, v, schemaVersion, errSchemaVersion)
+	}
+
+	return db, nil
+}
+
+// initSchema creates the schema on a fresh database and verifies the
+// schema version on an existing one.
+func initSchema(db *sql.DB) error {
+	v, err := userVersion(db)
+	if err != nil {
+		return err
+	}
+
+	switch v {
+	case 0:
+		return createSchema(db)
+	case schemaVersion:
+		return nil
+	default:
+		return fmt.Errorf("version %d, want %d: %w",
+			v, schemaVersion, errSchemaVersion)
+	}
+}
+
+// createSchema applies the schema to a fresh database and stamps the
+// schema version.
+func createSchema(db *sql.DB) error {
+	ctx := context.Background()
+
+	_, err := db.ExecContext(ctx, createTableSQL)
+	if err != nil {
+		return fmt.Errorf("create schema: %w", err)
+	}
+
+	_, err = db.ExecContext(ctx,
+		"PRAGMA user_version = "+strconv.Itoa(schemaVersion))
+	if err != nil {
+		return fmt.Errorf("set schema version: %w", err)
+	}
+
+	return nil
+}
+
+// userVersion reads the database's PRAGMA user_version.
+func userVersion(db *sql.DB) (int, error) {
+	var v int
+
+	err := db.QueryRowContext(context.Background(),
+		"PRAGMA user_version").Scan(&v)
+	if err != nil {
+		return 0, fmt.Errorf("read schema version: %w", err)
+	}
+
+	return v, nil
+}
+
+// loadFileRows reads every record from the files table.
+func loadFileRows(db *sql.DB) ([]scanRec, error) {
+	rows, err := db.QueryContext(context.Background(),
+		"SELECT path, size, mtime, head, tail FROM files")
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var recs []scanRec
+
+	for rows.Next() {
+		var (
+			path []byte
+			r    scanRec
+		)
+
+		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail)
+		if err != nil {
+			return nil, fmt.Errorf("read record: %w", err)
+		}
+
+		r.path = string(path)
+		recs = append(recs, r)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return nil, fmt.Errorf("read records: %w", err)
+	}
+
+	return recs, nil
+}
+
+// applyChanges writes one scan's database changes — upserts for new and
+// changed files, deletes for vanished ones — in a single transaction,
+// so a concurrent report never observes a half-finished scan. Progress
+// is rendered on prog (one increment per change).
+func applyChanges(db *sql.DB, upserts []scanRec, deletes []string,
+	prog *progress,
+) error {
+	ctx := context.Background()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+
+	defer func() { _ = tx.Rollback() }()
+
+	err = execUpserts(ctx, tx, upserts, prog)
+	if err != nil {
+		return err
+	}
+
+	err = execDeletes(ctx, tx, deletes, prog)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("commit: %w", err)
+	}
+
+	return nil
+}
+
+// execUpserts inserts or updates one record per new or changed file.
+func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, upsertSQL)
+	if err != nil {
+		return fmt.Errorf("prepare upsert: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, r := range upserts {
+		_, err = st.ExecContext(ctx,
+			[]byte(r.path), r.size, r.mtime, r.head, r.tail)
+		if err != nil {
+			return fmt.Errorf("upsert %s: %w", r.path, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}
+
+// execDeletes removes the records for paths no longer present.
+func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx, "DELETE FROM files WHERE path = ?")
+	if err != nil {
+		return fmt.Errorf("prepare delete: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, p := range deletes {
+		_, err = st.ExecContext(ctx, []byte(p))
+		if err != nil {
+			return fmt.Errorf("delete %s: %w", p, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}

db_test.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// testDBPath returns a database path inside a fresh temp dir.
+func testDBPath(t *testing.T) string {
+	t.Helper()
+
+	return filepath.Join(t.TempDir(), "db.sqlite")
+}
+
+// openTestDB creates a fresh scan database in a temp dir.
+func openTestDB(t *testing.T) *sql.DB {
+	t.Helper()
+
+	db, err := openScanDatabase(testDBPath(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() { _ = db.Close() })
+
+	return db
+}
+
+func TestDatabasePath(t *testing.T) {
+	t.Setenv(databaseEnv, "")
+
+	if got := databasePath(); got != defaultDatabasePath {
+		t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
+	}
+
+	t.Setenv(databaseEnv, "/custom/place.sqlite")
+
+	if got := databasePath(); got != "/custom/place.sqlite" {
+		t.Errorf("databasePath() = %q, want the env override", got)
+	}
+}
+
+func TestOpenScanDatabaseCreates(t *testing.T) {
+	t.Parallel()
+
+	// The parent directory does not exist yet; scan must create it.
+	path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("openScanDatabase: %v", err)
+	}
+
+	v, err := userVersion(db)
+	if err != nil || v != schemaVersion {
+		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
+	}
+
+	_ = db.Close()
+
+	// Reopening an existing database must succeed and find the schema.
+	db, err = openScanDatabase(path)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+
+	defer func() { _ = db.Close() }()
+
+	recs, err := loadFileRows(db)
+	if err != nil || len(recs) != 0 {
+		t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err)
+	}
+}
+
+func TestOpenReportDatabaseMissing(t *testing.T) {
+	t.Parallel()
+
+	_, err := openReportDatabase(testDBPath(t))
+	if !errors.Is(err, errNoDatabase) {
+		t.Fatalf("err = %v, want errNoDatabase", err)
+	}
+}
+
+func TestOpenReportDatabaseVersionMismatch(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	_, err = openReportDatabase(path)
+	if !errors.Is(err, errSchemaVersion) {
+		t.Fatalf("err = %v, want errSchemaVersion", err)
+	}
+}
+
+func TestOpenReportDatabaseOK(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	db, err := openScanDatabase(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_ = db.Close()
+
+	db, err = openReportDatabase(path)
+	if err != nil {
+		t.Fatalf("openReportDatabase: %v", err)
+	}
+
+	_ = db.Close()
+}
+
+func TestApplyChangesRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	// Paths may contain tabs and newlines; the database must store
+	// them byte-exactly.
+	recs := []scanRec{
+		{size: 2, mtime: 20, head: "h2", tail: "t2", path: "/a/tab\tnew\nline"},
+		{size: 1, mtime: 10, head: "h1", tail: "t1", path: "/a/x"},
+	}
+
+	err := applyChanges(db, recs, nil, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	slices.SortFunc(got, func(a, b scanRec) int {
+		return strings.Compare(a.path, b.path)
+	})
+
+	if !slices.Equal(got, recs) {
+		t.Fatalf("rows = %+v, want %+v", got, recs)
+	}
+
+	// An upsert for an existing path updates in place; a delete
+	// removes exactly its path.
+	upd := scanRec{size: 3, mtime: 30, head: "h3", tail: "t3", path: "/a/x"}
+
+	err = applyChanges(db, []scanRec{upd},
+		[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
+	if err != nil {
+		t.Fatalf("applyChanges: %v", err)
+	}
+
+	got, err = loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(got) != 1 || got[0] != upd {
+		t.Fatalf("rows = %+v, want just %+v", got, upd)
+	}
+}

go.mod

@@ -5,13 +5,22 @@ go 1.25.7
 require (
 	github.com/schollz/progressbar/v3 v3.19.1
 	github.com/spf13/cobra v1.10.2
+	modernc.org/sqlite v1.54.0
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.22 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/term v0.44.0 // indirect
+	modernc.org/libc v1.74.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -3,14 +3,28 @@ github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
+github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -23,10 +37,44 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
 golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
+modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog=
+modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

main.go

@@ -2,13 +2,15 @@
 // very large filesystems without reading full file contents. Files are
 // considered duplicates when they have identical size, identical SHA-256
 // of their first 1024 bytes, and identical SHA-256 of their last 1024
-// bytes.
+// bytes. scan maintains a persistent SQLite database of file signatures
+// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
+// reporting subcommands read.
 //
 // Usage:
 //
-//	sfdupes scan [--workers N] [-x] PATH... > files.dat
+//	sfdupes scan [--workers N] [-x] PATH...
-//	sfdupes report [files.dat|-] > dupes.tsv
+//	sfdupes report > dupes.tsv
-//	sfdupes trees [files.dat|-] > dupetrees.tsv
+//	sfdupes trees > dupetrees.tsv
 //
 // See README.md for the complete specification.
 package main
@@ -60,7 +62,7 @@ func main() {
 
 	scanCmd := &cobra.Command{
 		Use:   "scan [--workers N] [-x] PATH...",
-		Short: "Walk trees and emit one record per regular file on stdout",
+		Short: "Walk trees and synchronize the scan database",
 		Args:  cobra.MinimumNArgs(1),
 		Run: func(_ *cobra.Command, args []string) {
 			runScan(args, scanWorkers, scanOneFS)
@@ -72,20 +74,20 @@ func main() {
 		"do not cross filesystem boundaries")
 
 	reportCmd := &cobra.Command{
-		Use:   "report [files.dat|-]",
+		Use:   "report",
-		Short: "Read a scan stream and print the file-level duplicates report",
+		Short: "Read the scan database and print the file-level duplicates report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runReport(args)
+			runReport()
 		},
 	}
 
 	treesCmd := &cobra.Command{
-		Use:   "trees [files.dat|-]",
+		Use:   "trees",
-		Short: "Read a scan stream and print the duplicate-tree report",
+		Short: "Read the scan database and print the duplicate-tree report",
-		Args:  cobra.MaximumNArgs(1),
+		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
-			runTrees(args)
+			runTrees()
 		},
 	}
 

report.go

@@ -2,106 +2,49 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"slices"
-	"strconv"
 	"strings"
 )
 
-// ioBufSize is the buffer size for the buffered scan-stream reader and
+// ioBufSize is the buffer size for the buffered stdout writers.
-// the buffered stdout writers.
 const ioBufSize = 1 << 20
 
-// recordFieldCount is the number of tab-separated fields in a scan
-// record: size, mtime, head hash, tail hash, path.
-const recordFieldCount = 5
-
-// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
-// to read scan records; a record longer than scanMaxRecordSize is a
-// fatal read error.
-const (
-	scanInitBufSize   = 64 << 10
-	scanMaxRecordSize = 4 << 20
-)
-
 // minGroupSize is the smallest number of members that makes a
 // duplicate group.
 const minGroupSize = 2
 
-// scanRec is one well-formed record parsed from a scan stream. The
+// scanRec is one file record from the database. The signature (size,
-// signature (size, head, tail) is the duplicate key; mtime is
+// head, tail) is the duplicate key; mtime is informational only and
-// informational and not retained.
+// used by scan for change detection.
 type scanRec struct {
 	size  int64
+	mtime int64
 	head  string
 	tail  string
 	path  string
 }
 
-// openScanInput resolves the analysis-mode input: the file named by the
+// loadRecords opens the database and reads every file record for the
-// single optional positional argument, or stdin when it is absent or
+// report and trees subcommands. Any database problem — including a
-// "-". The returned closer must be called when reading is done.
+// missing database — is fatal.
-func openScanInput(args []string) (io.Reader, string, func()) {
+func loadRecords() []scanRec {
-	if len(args) == 1 && args[0] != "-" {
+	dbPath := databasePath()
-		f, err := os.Open(args[0])
+
+	db, err := openReportDatabase(dbPath)
 	if err != nil {
-			fatalf("open %s: %v", args[0], err)
+		fatalf("%v", err)
 	}
 
-		return f, args[0], func() { _ = f.Close() }
+	defer func() { _ = db.Close() }()
-	}
 
-	return os.Stdin, "stdin", func() {}
+	recs, err := loadFileRows(db)
-}
-
-// parseScanStream reads every NUL-terminated record from in. A record
-// that does not have exactly 5 fields or whose size is non-numeric is
-// counted as malformed and skipped. Total records read is
-// len(recs) + malformed.
-func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
-	sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
-	sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
-	sc.Split(splitNUL)
-
-	var (
-		recs      []scanRec
-		malformed int
-	)
-
-	for sc.Scan() {
-		// The path is the last field and may itself contain tabs,
-		// so split into at most recordFieldCount fields.
-		fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
-		if len(fields) != recordFieldCount {
-			malformed++
-
-			continue
-		}
-
-		size, err := strconv.ParseInt(fields[0], 10, 64)
 	if err != nil {
-			malformed++
+		fatalf("database %s: %v", dbPath, err)
-
-			continue
 	}
 
-		recs = append(recs, scanRec{
+	return recs
-			size: size,
-			head: fields[2],
-			tail: fields[3],
-			path: fields[4],
-		})
-	}
-
-	err := sc.Err()
-	if err != nil {
-		fatalf("read %s: %v", name, err)
-	}
-
-	return recs, malformed
 }
 
 // dupeGroup is one set of candidate-duplicate files: identical size,
@@ -112,18 +55,12 @@ type dupeGroup struct {
 	paths []string
 }
 
-// runReport implements the report subcommand: it reads a scan stream
+// runReport implements the report subcommand: it reads every record
-// from the named file (or stdin when absent or "-") and prints the
+// from the database and prints the file-level duplicates report as TSV
-// file-level duplicates report as TSV on stdout. It never touches the
+// on stdout. It never touches the scanned filesystem; its only I/O is
-// scanned filesystem; its only I/O is the scan input, stdout, and
+// the database, stdout, and stderr.
-// stderr. args holds the positional arguments already validated by
+func runReport() {
-// cobra (at most one).
+	recs := loadRecords()
-func runReport(args []string) {
-	in, name, closer := openScanInput(args)
-	defer closer()
-
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 	dupes := collectDupeGroups(recs)
 
 	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
@@ -156,9 +93,9 @@ func runReport(args []string) {
 	}
 
 	fmt.Fprintf(os.Stderr,
-		"report: %d records read%s, %d duplicate groups, %d dupe files, %s reclaimable\n",
+		"report: %d records read, %d duplicate groups, %d dupe files, "+
-		records, malformedNote(malformed), len(dupes), dupeFiles,
+			"%s reclaimable\n",
-		humanBytes(reclaimable))
+		len(recs), len(dupes), dupeFiles, humanBytes(reclaimable))
 }
 
 // collectDupeGroups groups records by signature and returns every group
@@ -199,30 +136,6 @@ func collectDupeGroups(recs []scanRec) []dupeGroup {
 	return dupes
 }
 
-// malformedNote formats the optional malformed-record note for the
-// stderr summaries.
-func malformedNote(malformed int) string {
-	if malformed == 0 {
-		return ""
-	}
-
-	return fmt.Sprintf(" (%d malformed, skipped)", malformed)
-}
-
-// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
-// data without a terminator at EOF is returned as a final record.
-func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
-	if i := bytes.IndexByte(data, 0); i >= 0 {
-		return i + 1, data[:i], nil
-	}
-
-	if atEOF && len(data) > 0 {
-		return len(data), data, nil
-	}
-
-	return 0, nil, nil
-}
-
 // humanBytes formats a byte count in human units (binary prefixes).
 func humanBytes(n int64) string {
 	const unit = 1024

report_test.go

@@ -1,92 +1,10 @@
 package main
 
 import (
-	"bufio"
-	"fmt"
 	"slices"
-	"strings"
 	"testing"
 )
 
-// mkRecord serializes one scan record in the on-the-wire format.
-func mkRecord(size, mtime int64, head, tail, path string) string {
-	return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00",
-		size, mtime, head, tail, path)
-}
-
-func TestSplitNULScanner(t *testing.T) {
-	t.Parallel()
-
-	sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail"))
-	sc.Split(splitNUL)
-
-	var got []string
-	for sc.Scan() {
-		got = append(got, sc.Text())
-	}
-
-	err := sc.Err()
-	if err != nil {
-		t.Fatalf("scanner error: %v", err)
-	}
-
-	want := []string{"a", "bb", "", "tail"}
-	if !slices.Equal(got, want) {
-		t.Fatalf("tokens = %q, want %q", got, want)
-	}
-}
-
-func TestParseScanStream(t *testing.T) {
-	t.Parallel()
-
-	stream := mkRecord(10, 1, "h1", "t1", "/a/x") +
-		"garbage-without-tabs\x00" +
-		"notanumber\t1\th\tt\t/a/bad\x00" +
-		mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") +
-		"30\t3\th3\tt3\t/trailing/no-nul"
-
-	recs, malformed := parseScanStream(strings.NewReader(stream), "test")
-
-	if malformed != 2 {
-		t.Errorf("malformed = %d, want 2", malformed)
-	}
-
-	want := []scanRec{
-		{size: 10, head: "h1", tail: "t1", path: "/a/x"},
-		{size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"},
-		{size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"},
-	}
-	if !slices.Equal(recs, want) {
-		t.Fatalf("recs = %+v, want %+v", recs, want)
-	}
-}
-
-func TestParseScanStreamPathWithNewline(t *testing.T) {
-	t.Parallel()
-
-	in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline"))
-
-	recs, malformed := parseScanStream(in, "test")
-	if malformed != 0 || len(recs) != 1 {
-		t.Fatalf("got %d recs, %d malformed, want 1, 0",
-			len(recs), malformed)
-	}
-
-	if recs[0].path != "/a/new\nline" {
-		t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline")
-	}
-}
-
-func TestParseScanStreamEmpty(t *testing.T) {
-	t.Parallel()
-
-	recs, malformed := parseScanStream(strings.NewReader(""), "test")
-	if len(recs) != 0 || malformed != 0 {
-		t.Fatalf("got %d recs, %d malformed, want 0, 0",
-			len(recs), malformed)
-	}
-}
-
 func TestCollectDupeGroups(t *testing.T) {
 	t.Parallel()
 
@@ -120,6 +38,22 @@ func TestCollectDupeGroups(t *testing.T) {
 	}
 }
 
+func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
+	t.Parallel()
+
+	// mtime is informational only; records differing only in mtime
+	// still group together.
+	recs := []scanRec{
+		{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
+		{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
+	}
+
+	groups := collectDupeGroups(recs)
+	if len(groups) != 1 {
+		t.Fatalf("len(groups) = %d, want 1", len(groups))
+	}
+}
+
 func TestCollectDupeGroupsTieBreak(t *testing.T) {
 	t.Parallel()
 
@@ -190,15 +124,3 @@ func TestHumanBytes(t *testing.T) {
 		}
 	}
 }
-
-func TestMalformedNote(t *testing.T) {
-	t.Parallel()
-
-	if got := malformedNote(0); got != "" {
-		t.Errorf("malformedNote(0) = %q, want empty", got)
-	}
-
-	if got := malformedNote(3); got != " (3 malformed, skipped)" {
-		t.Errorf("malformedNote(3) = %q", got)
-	}
-}

scan.go

@@ -1,14 +1,16 @@
 package main
 
 import (
-	"bufio"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
+	"strings"
 	"syscall"
 )
 
@@ -30,30 +32,242 @@ type fileRec struct {
 	mtime int64
 }
 
-// runScan implements the scan subcommand: three sequential passes
+// runScan implements the scan subcommand: four sequential passes
-// (walk, stat, hash) over the trees named by the PATH operands,
+// (walk, stat, hash, update) that synchronize the persistent database
-// emitting one NUL-terminated record per regular file on stdout. Flag
+// with the filesystem state under the PATH operands. Flag parsing and
-// parsing and the at-least-one-operand check are done by cobra.
+// the at-least-one-operand check are done by cobra.
 func runScan(roots []string, workers int, oneFS bool) {
 	if workers < 1 {
 		workers = 1
 	}
 
-	// A nonexistent operand is a fatal error before any scanning.
+	roots = resolveRoots(roots)
-	for _, root := range roots {
+	dbPath := databasePath()
-		_, err := os.Lstat(root)
+
+	db, err := openScanDatabase(dbPath)
 	if err != nil {
 		fatalf("%v", err)
 	}
+
+	defer func() { _ = db.Close() }()
+
+	st, err := syncScan(db, roots, workers, oneFS)
+	if err != nil {
+		fatalf("update database %s: %v", dbPath, err)
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"scan: %d files seen (%d added, %d updated, %d removed, "+
+			"%d unchanged), %d skipped\n",
+		st.added+st.updated+st.unchanged, st.added, st.updated,
+		st.removed, st.unchanged, st.skipped)
+}
+
+// resolveRoots converts each PATH operand to an absolute, lexically
+// cleaned path (symlinks are not resolved) and verifies that it
+// exists. Database records are keyed by absolute path, so scan results
+// must not depend on the working directory.
+func resolveRoots(roots []string) []string {
+	abs := make([]string, 0, len(roots))
+
+	for _, root := range roots {
+		a, err := filepath.Abs(root)
+		if err != nil {
+			fatalf("resolve %s: %v", root, err)
+		}
+
+		// A nonexistent operand is a fatal error before any scanning.
+		_, err = os.Lstat(a)
+		if err != nil {
+			fatalf("%v", err)
+		}
+
+		abs = append(abs, a)
+	}
+
+	return abs
+}
+
+// scanStats summarizes one scan's database synchronization for the
+// final stderr summary.
+type scanStats struct {
+	added     int
+	updated   int
+	removed   int
+	unchanged int
+	skipped   int
+}
+
+// syncScan synchronizes the database with the filesystem under roots:
+// walk and stat everything, hash only new or changed files, and apply
+// the resulting record insertions, updates, and deletions in a single
+// transaction. Records outside the roots are never touched.
+func syncScan(db *sql.DB, roots []string, workers int,
+	oneFS bool,
+) (scanStats, error) {
+	var st scanStats
+
+	existing, err := loadScopedRows(db, roots)
+	if err != nil {
+		return st, err
 	}
 
 	paths, walkErrs := walkPass(roots, oneFS)
-	recs, statErrs := statPass(paths, workers)
+	recs, statErrs := statPass(uniquePaths(paths), workers)
-	emitted, hashErrs := hashPass(recs, workers)
+	toHash, unchanged := partitionChanged(recs, existing)
+	hashed, hashErrs := hashPass(toHash, workers)
 
-	skipped := walkErrs + statErrs + hashErrs
+	st.unchanged = len(unchanged)
-	fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n",
+	st.skipped = walkErrs + statErrs + hashErrs
-		emitted, skipped)
+
+	for _, r := range hashed {
+		if _, ok := existing[r.path]; ok {
+			st.updated++
+		} else {
+			st.added++
+		}
+	}
+
+	deletes := collectDeletes(existing, unchanged, hashed)
+	st.removed = len(deletes)
+
+	return st, applyPass(db, hashed, deletes)
+}
+
+// loadScopedRows loads the database records whose paths lie under any
+// of the scan roots, keyed by path. Records outside the roots belong
+// to other trees and are left untouched by this scan.
+func loadScopedRows(db *sql.DB, roots []string) (map[string]scanRec, error) {
+	all, err := loadFileRows(db)
+	if err != nil {
+		return nil, err
+	}
+
+	scoped := make(map[string]scanRec)
+
+	for _, r := range all {
+		if underAnyRoot(r.path, roots) {
+			scoped[r.path] = r
+		}
+	}
+
+	return scoped, nil
+}
+
+// underAnyRoot reports whether path is any of the roots or lies under
+// one of them.
+func underAnyRoot(path string, roots []string) bool {
+	for _, root := range roots {
+		if underRoot(path, root) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// underRoot reports whether path is root itself or lies under it. Both
+// must be absolute and lexically clean.
+func underRoot(path, root string) bool {
+	if path == root {
+		return true
+	}
+
+	prefix := root
+	if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+
+	return strings.HasPrefix(path, prefix)
+}
+
+// uniquePaths deduplicates the walked paths, preserving order.
+// Overlapping operands can reach the same file more than once, but the
+// database keys records by path, so each file is processed once.
+func uniquePaths(paths []string) []string {
+	seen := make(map[string]bool, len(paths))
+	out := make([]string, 0, len(paths))
+
+	for _, p := range paths {
+		if seen[p] {
+			continue
+		}
+
+		seen[p] = true
+
+		out = append(out, p)
+	}
+
+	return out
+}
+
+// partitionChanged splits the stat results into files that must be
+// hashed (new, or changed) and files whose existing records are reused
+// without reading them: a file is unchanged when its statted size
+// equals the recorded size and its statted mtime is not newer than the
+// recorded mtime.
+func partitionChanged(recs []fileRec,
+	existing map[string]scanRec,
+) ([]fileRec, []scanRec) {
+	var (
+		toHash    []fileRec
+		unchanged []scanRec
+	)
+
+	for _, rec := range recs {
+		old, ok := existing[rec.path]
+		if ok && old.size == rec.size && old.mtime >= rec.mtime {
+			unchanged = append(unchanged, old)
+
+			continue
+		}
+
+		toHash = append(toHash, rec)
+	}
+
+	return toHash, unchanged
+}
+
+// collectDeletes returns the existing record paths that were not
+// successfully processed this run: vanished files, plus paths that
+// failed to stat or hash. The database keeps only records verified by
+// the latest scan covering them. The result is sorted so the update
+// pass is deterministic.
+func collectDeletes(existing map[string]scanRec,
+	unchanged, hashed []scanRec,
+) []string {
+	kept := make(map[string]bool, len(unchanged)+len(hashed))
+
+	for _, r := range unchanged {
+		kept[r.path] = true
+	}
+
+	for _, r := range hashed {
+		kept[r.path] = true
+	}
+
+	var deletes []string
+
+	for p := range existing {
+		if !kept[p] {
+			deletes = append(deletes, p)
+		}
+	}
+
+	slices.Sort(deletes)
+
+	return deletes
+}
+
+// applyPass writes the scan's changes to the database under an update
+// progress display (one item per insertion, update, or deletion).
+func applyPass(db *sql.DB, upserts []scanRec, deletes []string) error {
+	prog := newProgress("update", int64(len(upserts)+len(deletes)))
+	err := applyChanges(db, upserts, deletes, prog)
+
+	prog.finish()
+
+	return err
 }
 
 // treeWalker carries the walk-pass state shared by all PATH operands.
@@ -271,15 +485,15 @@ func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
 }
 
 // hashPass hashes the first and last chunk bytes of every file in a
-// worker pool and emits the output records on stdout from the main
+// worker pool and collects the resulting records on the main
 // goroutine. Files that fail to open or read are warned about and
-// dropped.
+// dropped. Only new or changed files reach this pass.
-func hashPass(recs []fileRec, workers int) (int, int) {
+func hashPass(recs []fileRec, workers int) ([]scanRec, int) {
 	results := startHashWorkers(recs, workers)
 	prog := newProgress("hash", int64(len(recs)))
-	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
+	out := make([]scanRec, 0, len(recs))
 
-	var emitted, errs int
+	var errs int
 
 	for range recs {
 		r := <-results
@@ -292,25 +506,20 @@ func hashPass(recs []fileRec, workers int) (int, int) {
 			continue
 		}
 
-		_, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
+		out = append(out, scanRec{
-			r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
+			size:  r.rec.size,
-		if err != nil {
+			mtime: r.rec.mtime,
-			fatalf("write stdout: %v", err)
+			head:  r.head,
-		}
+			tail:  r.tail,
-
+			path:  r.rec.path,
-		emitted++
+		})
 
 		prog.increment()
 	}
 
 	prog.finish()
 
-	err := out.Flush()
+	return out, errs
-	if err != nil {
-		fatalf("write stdout: %v", err)
-	}
-
-	return emitted, errs
 }
 
 // hashHeadTail returns the lowercase-hex SHA-256 of the first

scan_test.go

@@ -1,15 +1,16 @@
 package main
 
 import (
-	"bytes"
+	"context"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
-	"io"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 // writeFile creates a file with the given content and returns its path.
@@ -254,38 +255,6 @@ func TestStatPass(t *testing.T) {
 	}
 }
 
-// captureStdout runs fn with os.Stdout redirected to a temp file and
-// returns everything fn wrote to it.
-func captureStdout(t *testing.T, fn func()) []byte {
-	t.Helper()
-
-	f, err := os.CreateTemp(t.TempDir(), "stdout")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	orig := os.Stdout
-	os.Stdout = f
-
-	defer func() { os.Stdout = orig }()
-
-	fn()
-
-	_, err = f.Seek(0, io.SeekStart)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	data, err := io.ReadAll(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_ = f.Close()
-
-	return data
-}
-
 // buildSmokeTree recreates the README smoke-test filesystem layout
 // with deterministic content and returns the tree root.
 func buildSmokeTree(t *testing.T) string {
@@ -316,45 +285,65 @@ func buildSmokeTree(t *testing.T) string {
 	return dir
 }
 
-// scanToRecords runs the walk, stat, and hash passes over dir and
+// smokeTreeFiles is the number of regular files buildSmokeTree creates.
-// parses the emitted stream back into records.
+const smokeTreeFiles = 15
-func scanToRecords(t *testing.T, dir string) []scanRec {
+
+// syncTree synchronizes the database with the given roots and returns
+// the scan stats.
+func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats {
 	t.Helper()
 
-	paths, walkErrs := walkPass([]string{dir}, false)
+	st, err := syncScan(db, roots, 4, false)
-	if walkErrs != 0 {
+	if err != nil {
-		t.Fatalf("walk errors: %d", walkErrs)
+		t.Fatalf("syncScan: %v", err)
 	}
 
-	recs, statErrs := statPass(paths, 4)
+	return st
-	if statErrs != 0 {
-		t.Fatalf("stat errors: %d", statErrs)
-	}
-
-	var emitted, hashErrs int
-
-	out := captureStdout(t, func() {
-		emitted, hashErrs = hashPass(recs, 4)
-	})
-
-	if emitted != len(paths) || hashErrs != 0 {
-		t.Fatalf("emitted %d of %d, %d hash errors",
-			emitted, len(paths), hashErrs)
-	}
-
-	parsed, malformed := parseScanStream(bytes.NewReader(out), "pipe")
-	if malformed != 0 || len(parsed) != len(paths) {
-		t.Fatalf("parsed %d records, %d malformed, want %d, 0",
-			len(parsed), malformed, len(paths))
-	}
-
-	return parsed
 }
 
-//nolint:paralleltest // redirects the process-wide os.Stdout
+// dbRecords returns every record currently in the database.
-func TestScanPipeline(t *testing.T) {
+func dbRecords(t *testing.T, db *sql.DB) []scanRec {
-	dir := buildSmokeTree(t)
+	t.Helper()
-	parsed := scanToRecords(t, dir)
+
+	recs, err := loadFileRows(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return recs
+}
+
+// recordByPath finds the record with the given path.
+func recordByPath(t *testing.T, recs []scanRec, path string) scanRec {
+	t.Helper()
+
+	for _, r := range recs {
+		if r.path == path {
+			return r
+		}
+	}
+
+	t.Fatalf("no record for %q", path)
+
+	return scanRec{}
+}
+
+// recordPaths returns the sorted paths of recs.
+func recordPaths(recs []scanRec) []string {
+	paths := make([]string, 0, len(recs))
+	for _, r := range recs {
+		paths = append(paths, r.path)
+	}
+
+	slices.Sort(paths)
+
+	return paths
+}
+
+// assertSmokeDupeGroups checks the file-level duplicate groups for the
+// smoke tree rooted at dir.
+func assertSmokeDupeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
 
 	groups := collectDupeGroups(parsed)
 	if len(groups) != 5 {
@@ -377,6 +366,12 @@ func TestScanPipeline(t *testing.T) {
 	if !slices.Equal(groups[0].paths, wantF1) {
 		t.Errorf("groups[0].paths = %q, want %q", groups[0].paths, wantF1)
 	}
+}
+
+// assertSmokeTreeGroups checks the duplicate-tree groups for the smoke
+// tree rooted at dir.
+func assertSmokeTreeGroups(t *testing.T, dir string, parsed []scanRec) {
+	t.Helper()
 
 	super, dirs := buildHierarchy(parsed)
 	super.compute()
@@ -396,3 +391,282 @@ func TestScanPipeline(t *testing.T) {
 			tg[0][0].fileCount, tg[0][0].totalSize)
 	}
 }
+
+func TestScanPipeline(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: smokeTreeFiles}) {
+		t.Fatalf("stats = %+v, want %d added only", st, smokeTreeFiles)
+	}
+
+	parsed := dbRecords(t, db)
+	if len(parsed) != smokeTreeFiles {
+		t.Fatalf("len(records) = %d, want %d", len(parsed), smokeTreeFiles)
+	}
+
+	assertSmokeDupeGroups(t, dir, parsed)
+	assertSmokeTreeGroups(t, dir, parsed)
+}
+
+func TestSyncScanUnchangedReuse(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 2}) {
+		t.Fatalf("first scan stats = %+v, want 2 added", st)
+	}
+
+	// An immediate rescan reuses every record without reading file
+	// contents. Prove the files are not re-read by corrupting a stored
+	// hash and observing that it survives the rescan.
+	_, err := db.ExecContext(context.Background(),
+		"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st = syncTree(t, db, dir)
+	if st != (scanStats{unchanged: 2}) {
+		t.Fatalf("rescan stats = %+v, want 2 unchanged", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
+		t.Fatalf("head = %q, want sentinel (file must not be re-read)",
+			r.head)
+	}
+}
+
+func TestSyncScanMtimeBump(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	syncTree(t, db, dir)
+
+	// Bump the mtime forward: the file must be re-hashed even though
+	// its size is unchanged.
+	future := time.Now().Add(time.Hour)
+
+	err := os.Chtimes(a, future, future)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{updated: 1}) {
+		t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
+	}
+
+	if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
+		t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
+	}
+}
+
+func TestSyncScanAddRemove(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	writeFile(t, dir, "b.bin", pattern(2, 600))
+	syncTree(t, db, dir)
+
+	// Add one file, remove another.
+	c := writeFile(t, dir, "c.bin", pattern(3, 700))
+
+	err := os.Remove(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{added: 1, removed: 1, unchanged: 1}) {
+		t.Fatalf("add/remove stats = %+v, want 1 added 1 removed 1 unchanged",
+			st)
+	}
+
+	want := []string{filepath.Join(dir, "b.bin"), c}
+	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, want) {
+		t.Fatalf("paths = %q, want %q", got, want)
+	}
+}
+
+func TestSyncScanSizeChange(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 100))
+
+	syncTree(t, db, dir)
+
+	// Rewrite with a different size but force the mtime back to the
+	// recorded value: the size mismatch alone must trigger a re-hash.
+	old := recordByPath(t, dbRecords(t, db), p)
+
+	writeFile(t, dir, "f", pattern(1, 200))
+
+	mt := time.Unix(old.mtime, 0)
+
+	err := os.Chtimes(p, mt, mt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.updated != 1 {
+		t.Fatalf("stats = %+v, want 1 updated", st)
+	}
+
+	if got := recordByPath(t, dbRecords(t, db), p); got.size != 200 {
+		t.Fatalf("size = %d, want 200", got.size)
+	}
+}
+
+func TestSyncScanScope(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "a/keep", pattern(1, 10))
+
+	gone := writeFile(t, dir, "b/gone", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Deleting a file outside the rescanned root must not remove its
+	// record: records outside the scanned operands are untouched.
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, filepath.Join(dir, "a"))
+	if st.removed != 0 || st.unchanged != 1 {
+		t.Fatalf("subtree stats = %+v, want 0 removed 1 unchanged", st)
+	}
+
+	if got := recordPaths(dbRecords(t, db)); len(got) != 2 {
+		t.Fatalf("records = %q, want both retained", got)
+	}
+
+	// Rescanning the parent now removes the vanished file's record.
+	st = syncTree(t, db, dir)
+	if st.removed != 1 {
+		t.Fatalf("parent stats = %+v, want 1 removed", st)
+	}
+}
+
+func TestSyncScanRemovesNonRegular(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "f", pattern(1, 10))
+	keep := writeFile(t, dir, "g", pattern(2, 10))
+
+	syncTree(t, db, dir)
+
+	// Replace the file with a symlink: it is no longer walked, so its
+	// record must be deleted.
+	err := os.Remove(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Symlink(keep, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st.removed != 1 || st.unchanged != 1 {
+		t.Fatalf("stats = %+v, want 1 removed 1 unchanged", st)
+	}
+}
+
+func TestSyncScanOverlappingRoots(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	// A file reachable via two overlapping operands yields one record.
+	st := syncTree(t, db, dir, filepath.Join(dir, "sub"))
+	if st.added != 1 {
+		t.Fatalf("stats = %+v, want 1 added", st)
+	}
+}
+
+func TestReportsNeverTouchFilesystem(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	syncTree(t, db, dir)
+
+	// Remove the scanned tree entirely; the analysis must be
+	// unaffected because it reads the database alone.
+	err := os.RemoveAll(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	recs := dbRecords(t, db)
+
+	assertSmokeDupeGroups(t, dir, recs)
+	assertSmokeTreeGroups(t, dir, recs)
+}
+
+func TestUnderRoot(t *testing.T) {
+	t.Parallel()
+
+	const abRoot = "/a/b"
+
+	cases := []struct {
+		path string
+		root string
+		want bool
+	}{
+		{"/a/b/c", abRoot, true},
+		{abRoot, abRoot, true},
+		{"/a/bc", abRoot, false},
+		{"/a", abRoot, false},
+		{"/x/y", "/", true},
+		{"/", "/", true},
+	}
+	for _, c := range cases {
+		if got := underRoot(c.path, c.root); got != c.want {
+			t.Errorf("underRoot(%q, %q) = %v, want %v",
+				c.path, c.root, got, c.want)
+		}
+	}
+}
+
+func TestUniquePaths(t *testing.T) {
+	t.Parallel()
+
+	got := uniquePaths([]string{"/a", "/b", "/a", "/c", "/b"})
+
+	want := []string{"/a", "/b", "/c"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("uniquePaths = %q, want %q", got, want)
+	}
+}

trees.go

@@ -28,19 +28,14 @@ type treeNode struct {
 	totalSize int64
 }
 
-// runTrees implements the trees subcommand: it reads a scan stream from
+// runTrees implements the trees subcommand: it reads every record from
-// the named file (or stdin when absent or "-"), reconstructs the
+// the database, reconstructs the directory hierarchy from the record
-// directory hierarchy from the record paths, computes a Merkle-style
+// paths, computes a Merkle-style digest per directory, and prints
-// digest per directory, and prints maximal duplicate-tree groups as TSV
+// maximal duplicate-tree groups as TSV on stdout. It never touches the
-// on stdout. It never touches the scanned filesystem; its only I/O is
+// scanned filesystem; its only I/O is the database, stdout, and
-// the scan input, stdout, and stderr. args holds the positional
+// stderr.
-// arguments already validated by cobra (at most one).
+func runTrees() {
-func runTrees(args []string) {
+	recs := loadRecords()
-	in, name, closer := openScanInput(args)
-	defer closer()
-
-	recs, malformed := parseScanStream(in, name)
-	records := len(recs) + malformed
 
 	super, allDirs := buildHierarchy(recs)
 	super.compute()
@@ -78,9 +73,9 @@ func runTrees(args []string) {
 	}
 
 	fmt.Fprintf(os.Stderr,
-		"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
-		records, malformedNote(malformed), len(dupes), dupeTrees,
+			"%s reclaimable\n",
-		humanBytes(reclaimable))
+		len(recs), len(dupes), dupeTrees, humanBytes(reclaimable))
 }
 
 // buildHierarchy reconstructs the directory hierarchy from the record