Merge pull request 'next → main: trial-run fixes (escaping, read-only reports, scan lock, interrupts, streaming, README)' (#69) from next into main

Reviewed-on: #69

README.md

@@ -399,9 +399,10 @@ scan and are enumerated concurrently: every operand seeds the shared walk worker
 pool. Overlapping operands are harmless — an operand that duplicates another or
 lies under another is dropped before walking, so every file is reached exactly
 once and produces one database record. With `-x`, though, an operand under
-another one is kept and walked on its own when it is a directory on a different
+another one is kept and walked on its own when any directory from it (from its
-filesystem than the operand covering it, or a regular file in such a directory,
+directory, for a regular file) up to but not including the operand covering it
-because the walk of the covering operand does not reach it.
+is on a different filesystem than that operand, because that operand's walk does
+not enter such a directory and so does not reach it.
 
 An operand that is a symlink (never followed, not even as an operand), socket,
 FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
@@ -410,9 +411,8 @@ drops it from the scanned operands before reading the database. Another operand
 beneath it is still scanned. The records stored beneath it are not deleted: they
 are treated like any other record outside the scanned operands, including the
 content-phase exception below. If it lies under another operand, they are under
-that operand instead, and are deleted like any other record there that this scan
+that operand instead, and are treated like any other record there. This is not
-did not verify. This is not an error: a scan whose every operand is dropped
+an error: a scan whose every operand is dropped walks nothing and exits 0.
-walks nothing and exits 0.
 
 `scan` synchronizes the database with the filesystem state under the scanned
 operands:
@@ -522,17 +522,19 @@ Rules for the walk:
 - Filesystem boundaries are crossed by default. With `-x` (long form
   `--one-file-system`, following the GNU `du`/`rsync` convention), never descend
   into a directory on a different filesystem than its `PATH` operand; each
-  operand is bounded by its own filesystem. The records below such a directory
+  operand is bounded by its own filesystem. The records below such a directory,
-  count as outside the scanned operands, as described in "`scan` mode" above:
+  unless they lie below an operand at or inside that directory, count as outside
-  they are not deleted, and their sizes join the size census.
+  the scanned operands, as described in "`scan` mode" above: they are not
+  deleted, and their sizes join the size census.
 - On any per-path error (permission denied, file vanished between passes,
   unreadable): print a one-line warning to stderr, skip the path, and continue.
   Per-file errors never abort the run; the final summary reports how many were
   skipped. As specified above, a skipped path that has a database record from an
   earlier scan loses that record, unless it failed only in the content phase, or
-  is an operand dropped before the database was read that lies under no other
+  is an operand dropped before the database was read whose record counts as
-  operand; an unreadable directory subtree likewise loses its records (accepted:
+  outside the scanned operands; an unreadable directory subtree likewise loses
-  the database mirrors what the latest scan could actually verify).
+  its records (accepted: the database mirrors what the latest scan could
+  actually verify).
 
 Concurrency: the walk phase (which also stats files), the hash phase, and the
 content phase each use a worker pool of `--workers` workers (default

TODO.md

@@ -30,8 +30,9 @@
 # Completed Steps
 
 - with `-x`, an operand under another operand is walked on its own instead of
-  being dropped when it is a directory on a different filesystem than that
+  being dropped when a directory from it (from its directory, for a regular
-  operand, or a regular file in such a directory, and the records of files gone
+  file) up to but not including that operand is on a different filesystem than
+  that operand, whose walk then cannot reach it, and the records of files gone
   from below it are deleted (2026-10-14,
   https://git.eeqj.de/sneak/sfdupes/issues/106)
 

scan.go

@@ -204,12 +204,12 @@ func resolveRoots(roots []string) ([]string, error) {
 
 // pruneRoots drops operands already covered by another operand:
 // duplicates and any operand lying under another one. With -x, an
-// operand under another one is kept when device puts it, or for a
+// operand under another one is kept when the walk of the deepest kept
-// regular file its directory, on a different filesystem than the
+// operand above it cannot reach it: when device puts any directory from
-// deepest kept operand above it, because that operand's walk then does
+// it (from its directory, for a regular file) up to but not including
-// not reach it. Every remaining file is then reachable through exactly
+// that operand on a different filesystem than that operand. Every
-// one root, so walked paths are unique without keeping a scan-wide set
+// remaining file is then reachable through exactly one root, so walked
-// of every path seen.
+// paths are unique without keeping a scan-wide set of every path seen.
 func pruneRoots(roots []string, oneFS bool,
 	device func(path string) (uint64, bool),
 ) []string {
@@ -239,10 +239,15 @@ func pruneRoots(roots []string, oneFS bool,
 			continue
 		}
 
+		aboveDev, aboveOK := device(above)
+		if !aboveOK {
+			continue
+		}
+
 		// The walk of above enters a directory only when it is on
 		// above's filesystem, and then reaches every regular file in it
 		// whatever device the file reports, so for a regular-file
-		// operand its directory decides.
+		// operand the directories checked start at its directory.
 		dir := r
 
 		fi, err := os.Lstat(r)
@@ -250,11 +255,13 @@ func pruneRoots(roots []string, oneFS bool,
 			dir = filepath.Dir(r)
 		}
 
-		dev, ok := device(dir)
+		for ; dir != above; dir = filepath.Dir(dir) {
-		aboveDev, aboveOK := device(above)
+			dev, ok := device(dir)
+			if ok && dev != aboveDev {
+				kept = append(kept, r)
 
-		if ok && aboveOK && dev != aboveDev {
+				break
-			kept = append(kept, r)
+			}
 		}
 	}
 
@@ -331,8 +338,8 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
 // size, head, and tail match another record's). Records outside the
 // roots are never touched, except that the content phase fills in
 // their content hash. Operands the walk cannot start from are dropped
-// first, so the records beneath them count as outside the roots unless
+// first, so the records beneath them are treated as if those operands
-// they lie under another root.
+// had not been given.
 func (s *scanState) runPhases(ctx context.Context, roots []string,
 	workers int, oneFS bool,
 ) error {
@@ -374,10 +381,10 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 // walkableRoots returns the operands the walk can start from: regular
 // files, and directories not named .zfs. Every other operand is warned
 // about, counted as skipped, and dropped. A dropped operand is no
-// longer a root, so the records stored beneath it count as outside the
+// longer a root, so the records stored beneath it are treated as if it
-// roots and are not deleted as unverified, unless it lies under another
+// had not been given: those under no other root count as outside the
-// root. An operand that fails lstat here is kept, and the walk warns
+// roots and are not deleted as unverified. An operand that fails lstat
-// about it.
+// here is kept, and the walk warns about it.
 func (s *scanState) walkableRoots(roots []string) []string {
 	kept := make([]string, 0, len(roots))
 
@@ -1032,7 +1039,8 @@ func operandWarning(root string, fi fs.FileInfo) string {
 // operand becomes an initial job. walkableRoots has already dropped
 // every other operand. One that has changed into something else since
 // is warned about and skipped here; it is still a root, so the records
-// stored beneath it are deleted as unverified.
+// stored beneath it, other than those under another root, are deleted
+// as unverified.
 func seedRoot(ctx context.Context, root string,
 	events chan<- walkEvent,
 ) []dirJob {
@@ -1232,7 +1240,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 // .zfs (ZFS snapshot pseudo-dirs would list every file once per
 // snapshot), and with -x never enter a directory on a different
 // filesystem than its operand, sending an event that names it so the
-// scan keeps the records below it.
+// scan keeps the records below it, except those below an operand at or
+// inside it.
 func subdirJob(ctx context.Context, p string, e fs.DirEntry,
 	parent dirJob, oneFS bool, events chan<- walkEvent,
 ) (dirJob, bool) {

scan_test.go

@@ -1136,10 +1136,11 @@ func TestSubdirJobStatError(t *testing.T) {
 }
 
 // syncOneFilesystem scans roots into db with -x and returns the scan
-// stats, taking the device of each operand, and of a regular-file
+// stats. Every device pruneRoots looks up, and the device each
-// operand's directory, from devices instead of from lstat, so a test
+// operand's walk compares its subdirectories with, comes from devices
-// can put an operand on a filesystem that no directory here is on. It
+// instead of from lstat, so a test can put a path on a filesystem that
-// runs the steps of runPhases except the content phase.
+// no directory here is on. Unlike runPhases, it skips walkableRoots,
+// the check for a cancelled walk, and the content phase.
 func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
 	devices map[string]uint64,
 ) scanStats {
@@ -1302,6 +1303,64 @@ func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
 	}
 }
 
+// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that
+// with -x an operand on the same filesystem as the operand covering it
+// is walked on its own when a directory between them is on another
+// filesystem, since the walk of the covering operand does not enter
+// that directory: a file added below the nested operand is recorded,
+// and the record of a file gone from below it is deleted. A record
+// below that directory but outside the nested operand is still kept.
+// Both operands are given one device that no directory here is on,
+// while mnt keeps its own.
+func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	// A scan without -x records both files below mnt.
+	gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10))
+	kept := writeFile(t, dir, "mnt/kept", pattern(2, 20))
+	syncTree(t, db, dir)
+
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	added := writeFile(t, dir, "mnt/back/added", pattern(3, 30))
+
+	info, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev, ok := deviceOfInfo(info)
+	if !ok {
+		t.Skip("platform exposes no device id")
+	}
+
+	mnt := filepath.Join(dir, "mnt")
+	back := filepath.Join(mnt, "back")
+	syncOneFilesystem(t, db, []string{dir, back},
+		map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1})
+
+	paths := recordPaths(dbRecords(t, db))
+
+	if !slices.Contains(paths, added) {
+		t.Errorf("no record for %s: the walk did not reach %s", added, back)
+	}
+
+	if slices.Contains(paths, gone) {
+		t.Errorf("record for %s kept: the file is gone", gone)
+	}
+
+	if !slices.Contains(paths, kept) {
+		t.Errorf("record for %s deleted: the walk of %s did not enter %s",
+			kept, dir, mnt)
+	}
+}
+
 // TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
 // regular-file operand under another operand is dropped when the file
 // reports a different device than that operand but its directory does
@@ -1334,6 +1393,38 @@ func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) {
 	}
 }
 
+// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a
+// regular-file operand under another operand is kept when its directory
+// is on a different filesystem than that operand: the walk of the other
+// operand does not enter the directory, so the file is recorded once,
+// as its own operand. The other operand is given a device that no
+// directory here is on.
+func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	file := writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	info, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev, ok := deviceOfInfo(info)
+	if !ok {
+		t.Skip("platform exposes no device id")
+	}
+
+	st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
+		dir: dev + 1, filepath.Dir(file): dev,
+	})
+
+	if st.added != 1 {
+		t.Errorf("%s recorded %d times, want once", file, st.added)
+	}
+}
+
 // buildSmokeTree recreates the README smoke-test filesystem layout
 // with deterministic content and returns the tree root.
 func buildSmokeTree(t *testing.T) string {