Implement persistent SQLite scan database

scan now synchronizes a database that survives between runs
(SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of
emitting a stream: operands are resolved to absolute paths, unchanged
files (same size, mtime not newer than recorded) are never re-read, new
and changed files are hashed, and records under the scanned operands
that were not verified this run are deleted; records outside the
operands are untouched. All changes commit in a single transaction, and
WAL journaling with a busy timeout keeps a report run during a cron
scan safe.

report and trees read the database (no positional arguments); the
NUL-terminated stream format, its parser, and the malformed-record
handling are gone. The driver is modernc.org/sqlite (pure Go), so
builds keep cgo disabled.

scan.go

@@ -1,14 +1,16 @@
 package main
 
 import (
-	"bufio"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
+	"strings"
 	"syscall"
 )
 
@@ -30,30 +32,242 @@ type fileRec struct {
 	mtime int64
 }
 
-// runScan implements the scan subcommand: three sequential passes
-// (walk, stat, hash) over the trees named by the PATH operands,
-// emitting one NUL-terminated record per regular file on stdout. Flag
-// parsing and the at-least-one-operand check are done by cobra.
+// runScan implements the scan subcommand: four sequential passes
+// (walk, stat, hash, update) that synchronize the persistent database
+// with the filesystem state under the PATH operands. Flag parsing and
+// the at-least-one-operand check are done by cobra.
 func runScan(roots []string, workers int, oneFS bool) {
 	if workers < 1 {
 		workers = 1
 	}
 
-	// A nonexistent operand is a fatal error before any scanning.
+	roots = resolveRoots(roots)
+	dbPath := databasePath()
+
+	db, err := openScanDatabase(dbPath)
+	if err != nil {
+		fatalf("%v", err)
+	}
+
+	defer func() { _ = db.Close() }()
+
+	st, err := syncScan(db, roots, workers, oneFS)
+	if err != nil {
+		fatalf("update database %s: %v", dbPath, err)
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"scan: %d files seen (%d added, %d updated, %d removed, "+
+			"%d unchanged), %d skipped\n",
+		st.added+st.updated+st.unchanged, st.added, st.updated,
+		st.removed, st.unchanged, st.skipped)
+}
+
+// resolveRoots converts each PATH operand to an absolute, lexically
+// cleaned path (symlinks are not resolved) and verifies that it
+// exists. Database records are keyed by absolute path, so scan results
+// must not depend on the working directory.
+func resolveRoots(roots []string) []string {
+	abs := make([]string, 0, len(roots))
+
 	for _, root := range roots {
-		_, err := os.Lstat(root)
+		a, err := filepath.Abs(root)
+		if err != nil {
+			fatalf("resolve %s: %v", root, err)
+		}
+
+		// A nonexistent operand is a fatal error before any scanning.
+		_, err = os.Lstat(a)
 		if err != nil {
 			fatalf("%v", err)
 		}
+
+		abs = append(abs, a)
+	}
+
+	return abs
+}
+
+// scanStats summarizes one scan's database synchronization for the
+// final stderr summary.
+type scanStats struct {
+	added     int
+	updated   int
+	removed   int
+	unchanged int
+	skipped   int
+}
+
+// syncScan synchronizes the database with the filesystem under roots:
+// walk and stat everything, hash only new or changed files, and apply
+// the resulting record insertions, updates, and deletions in a single
+// transaction. Records outside the roots are never touched.
+func syncScan(db *sql.DB, roots []string, workers int,
+	oneFS bool,
+) (scanStats, error) {
+	var st scanStats
+
+	existing, err := loadScopedRows(db, roots)
+	if err != nil {
+		return st, err
 	}
 
 	paths, walkErrs := walkPass(roots, oneFS)
-	recs, statErrs := statPass(paths, workers)
-	emitted, hashErrs := hashPass(recs, workers)
+	recs, statErrs := statPass(uniquePaths(paths), workers)
+	toHash, unchanged := partitionChanged(recs, existing)
+	hashed, hashErrs := hashPass(toHash, workers)
 
-	skipped := walkErrs + statErrs + hashErrs
-	fmt.Fprintf(os.Stderr, "scan: %d files emitted, %d skipped\n",
-		emitted, skipped)
+	st.unchanged = len(unchanged)
+	st.skipped = walkErrs + statErrs + hashErrs
+
+	for _, r := range hashed {
+		if _, ok := existing[r.path]; ok {
+			st.updated++
+		} else {
+			st.added++
+		}
+	}
+
+	deletes := collectDeletes(existing, unchanged, hashed)
+	st.removed = len(deletes)
+
+	return st, applyPass(db, hashed, deletes)
+}
+
+// loadScopedRows loads the database records whose paths lie under any
+// of the scan roots, keyed by path. Records outside the roots belong
+// to other trees and are left untouched by this scan.
+func loadScopedRows(db *sql.DB, roots []string) (map[string]scanRec, error) {
+	all, err := loadFileRows(db)
+	if err != nil {
+		return nil, err
+	}
+
+	scoped := make(map[string]scanRec)
+
+	for _, r := range all {
+		if underAnyRoot(r.path, roots) {
+			scoped[r.path] = r
+		}
+	}
+
+	return scoped, nil
+}
+
+// underAnyRoot reports whether path is any of the roots or lies under
+// one of them.
+func underAnyRoot(path string, roots []string) bool {
+	for _, root := range roots {
+		if underRoot(path, root) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// underRoot reports whether path is root itself or lies under it. Both
+// must be absolute and lexically clean.
+func underRoot(path, root string) bool {
+	if path == root {
+		return true
+	}
+
+	prefix := root
+	if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+
+	return strings.HasPrefix(path, prefix)
+}
+
+// uniquePaths deduplicates the walked paths, preserving order.
+// Overlapping operands can reach the same file more than once, but the
+// database keys records by path, so each file is processed once.
+func uniquePaths(paths []string) []string {
+	seen := make(map[string]bool, len(paths))
+	out := make([]string, 0, len(paths))
+
+	for _, p := range paths {
+		if seen[p] {
+			continue
+		}
+
+		seen[p] = true
+
+		out = append(out, p)
+	}
+
+	return out
+}
+
+// partitionChanged splits the stat results into files that must be
+// hashed (new, or changed) and files whose existing records are reused
+// without reading them: a file is unchanged when its statted size
+// equals the recorded size and its statted mtime is not newer than the
+// recorded mtime.
+func partitionChanged(recs []fileRec,
+	existing map[string]scanRec,
+) ([]fileRec, []scanRec) {
+	var (
+		toHash    []fileRec
+		unchanged []scanRec
+	)
+
+	for _, rec := range recs {
+		old, ok := existing[rec.path]
+		if ok && old.size == rec.size && old.mtime >= rec.mtime {
+			unchanged = append(unchanged, old)
+
+			continue
+		}
+
+		toHash = append(toHash, rec)
+	}
+
+	return toHash, unchanged
+}
+
+// collectDeletes returns the existing record paths that were not
+// successfully processed this run: vanished files, plus paths that
+// failed to stat or hash. The database keeps only records verified by
+// the latest scan covering them. The result is sorted so the update
+// pass is deterministic.
+func collectDeletes(existing map[string]scanRec,
+	unchanged, hashed []scanRec,
+) []string {
+	kept := make(map[string]bool, len(unchanged)+len(hashed))
+
+	for _, r := range unchanged {
+		kept[r.path] = true
+	}
+
+	for _, r := range hashed {
+		kept[r.path] = true
+	}
+
+	var deletes []string
+
+	for p := range existing {
+		if !kept[p] {
+			deletes = append(deletes, p)
+		}
+	}
+
+	slices.Sort(deletes)
+
+	return deletes
+}
+
+// applyPass writes the scan's changes to the database under an update
+// progress display (one item per insertion, update, or deletion).
+func applyPass(db *sql.DB, upserts []scanRec, deletes []string) error {
+	prog := newProgress("update", int64(len(upserts)+len(deletes)))
+	err := applyChanges(db, upserts, deletes, prog)
+
+	prog.finish()
+
+	return err
 }
 
 // treeWalker carries the walk-pass state shared by all PATH operands.
@@ -271,15 +485,15 @@ func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
 }
 
 // hashPass hashes the first and last chunk bytes of every file in a
-// worker pool and emits the output records on stdout from the main
+// worker pool and collects the resulting records on the main
 // goroutine. Files that fail to open or read are warned about and
-// dropped.
-func hashPass(recs []fileRec, workers int) (int, int) {
+// dropped. Only new or changed files reach this pass.
+func hashPass(recs []fileRec, workers int) ([]scanRec, int) {
 	results := startHashWorkers(recs, workers)
 	prog := newProgress("hash", int64(len(recs)))
-	out := bufio.NewWriterSize(os.Stdout, ioBufSize)
+	out := make([]scanRec, 0, len(recs))
 
-	var emitted, errs int
+	var errs int
 
 	for range recs {
 		r := <-results
@@ -292,25 +506,20 @@ func hashPass(recs []fileRec, workers int) (int, int) {
 			continue
 		}
 
-		_, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
-			r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
-		if err != nil {
-			fatalf("write stdout: %v", err)
-		}
-
-		emitted++
+		out = append(out, scanRec{
+			size:  r.rec.size,
+			mtime: r.rec.mtime,
+			head:  r.head,
+			tail:  r.tail,
+			path:  r.rec.path,
+		})
 
 		prog.increment()
 	}
 
 	prog.finish()
 
-	err := out.Flush()
-	if err != nil {
-		fatalf("write stdout: %v", err)
-	}
-
-	return emitted, errs
+	return out, errs
 }
 
 // hashHeadTail returns the lowercase-hex SHA-256 of the first