Walk an operand below .zfs or a symlink on its own (closes #142)

An operand under another was dropped as covered by it even when a
directory between them was named .zfs or was a symlink. The walk of
the outer operand enters neither, so the files under the inner operand
went unscanned and their records were deleted. walkReaches now reports
such an operand as not reached, so pruneRoots keeps it and walks it on
its own. README lists both cases with the others.

Judgement call: the new constant zfsDirName replaces the .zfs literal
in operandWarning and subdirJob too, because goconst rejects a third
copy.

Model: opus-5-5

README.md

@@ -455,32 +455,29 @@ scan and are enumerated concurrently: every operand seeds the shared walk worker
 pool. Overlapping operands are harmless — an operand that duplicates another or
 lies under another is dropped before walking, so every file is reached exactly
 once and produces one database record. An operand under another one is kept,
-though, and walked on its own when any directory from it (from its directory,
+though, when the walk of the outer operand does not reach it. It is then walked
-for a regular file) up to but not including the operand covering it is a default
+on its own. The walk of the outer operand does not reach the inner one when it,
-exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with
+or any directory between the two, is:
-`-x`, is on a different filesystem than that operand, because that operand's
-walk does not enter such a directory and so does not reach it.
 
-An operand is not scanned when it is:
+- named `.zfs`;
+- a symlink;
+- a default exclude;
+- not on a persistent filesystem (see "Rules for the walk");
+- with `-x`, on a different filesystem than the outer operand.
 
-- a symlink, which `scan` never follows, not even as an operand;
+For a regular-file operand, only the directories between the two count.
-- a socket, FIFO or device node;
-- at or inside a directory named `.zfs`.
 
-An operand reached through a symlink into a `.zfs` directory is not scanned
+An operand that is a symlink (never followed, not even as an operand), socket,
-either.
+FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
-
+a one-line warning naming the path and what it is, counts it as skipped, and
-This holds even when the operand lies under another operand. `scan` prints a
+drops it from the scanned operands before reading the database. Another operand
-one-line warning naming the path and what it is, counts it as skipped, and drops
+beneath it is still scanned. The records stored beneath it are not deleted: they
-it from the scanned operands before reading the database. An operand beneath a
+are treated like any other record outside the scanned operands. If it lies under
-symlink operand is still scanned. The records stored beneath the dropped operand
+another operand, they are under that operand instead, and are treated like any
-are not deleted: they are treated like any other record outside the scanned
+other record there. This is not an error: a scan whose every operand is dropped
-operands. If it lies under another operand, they are under that operand instead,
+walks nothing and exits 0. An operand that is not on a persistent filesystem
-and are treated like any other record there. This is not an error: a scan whose
+(see "Rules for the walk") is dropped the same way. The line `scan` prints for
-every operand is dropped walks nothing and exits 0. An operand that is not on a
+it is not a warning, though, and it is not counted as skipped.
-persistent filesystem (see "Rules for the walk") is dropped the same way. The
-line `scan` prints for it is not a warning, though, and it is not counted as
-skipped.
 
 `scan` synchronizes the database with the filesystem state under the scanned
 operands:

TODO.md

@@ -29,9 +29,10 @@
 
 # Completed Steps
 
-- an operand at or inside a directory named `.zfs` is never scanned, whether
+- an operand under another is walked on its own when a directory between them,
-  given alone or under another operand. `scan` warns about it, counts it as
+  the inner operand included, is named `.zfs` or is a symlink. The walk of the
-  skipped and does not walk it (2026-10-26,
+  outer operand does not reach it then. `scan` now records the files below it
+  and no longer deletes their records (2026-10-26,
   https://git.eeqj.de/sneak/sfdupes/issues/142)
 
 - a file or directory removed while the scan runs is not an error, in the walk

main_test.go

@@ -928,21 +928,24 @@ func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
 }
 
 func TestRunScanSkipsZFSOperand(t *testing.T) {
-	zfs := filepath.Join(t.TempDir(), ".zfs")
+	path := testDBPath(t)
-	f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
-
-	// No scan records a file inside a .zfs directory, so the record
-	// beneath the operand is written straight into the database.
-	path := seedDatabase(t, []scanRec{{size: 10, path: f}})
 	t.Setenv(databaseEnv, path)
 
+	zfs := filepath.Join(t.TempDir(), ".zfs")
+	snapshot := filepath.Join(zfs, "snapshot", "hourly")
+	f := writeFile(t, snapshot, "f", pattern(1, 10))
+
+	// An operand beneath a .zfs directory is walked, because it is not
+	// itself named .zfs.
+	scanOK(t, snapshot)
+
 	assertOperandSkipped(t, path, zfs, ".zfs directory", f)
 }
 
 // assertOperandSkipped scans operand alone and checks that it is skipped
 // as kind: a warning naming it, one skip in the summary, exit 0, and the
-// record for kept, stored beneath operand before this scan, still in the
+// record for kept, which an earlier scan stored beneath operand, still
-// database at dbPath.
+// in the database at dbPath.
 func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	kept string,
 ) {
@@ -971,121 +974,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	recordByPath(t, dbRecords(t, db), kept)
 }
 
-// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
-// directory, first alone and then under another operand. Each time it
-// is warned about and counted as skipped, and nothing in it is walked.
-// The other operand is scanned as usual.
-func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
-	writeFile(t, hourly, "f", pattern(1, 10))
-	f := writeFile(t, dir, "f", pattern(2, 10))
-
-	warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
-
-	for _, tc := range []struct {
-		operands []string
-		summary  string
-	}{
-		{
-			[]string{hourly},
-			"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-		{
-			[]string{dir, hourly},
-			"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-	} {
-		stderr := scanOK(t, tc.operands...)
-		if !strings.Contains(stderr, warning) ||
-			!strings.Contains(stderr, tc.summary) {
-			t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
-				stderr, warning, tc.summary)
-		}
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
-		t.Errorf("records = %q, want %q", got, []string{f})
-	}
-}
-
-// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
-// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
-// by the path given, counted as skipped, and not walked.
-func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
-	t.Setenv(databaseEnv, testDBPath(t))
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
-
-	snaps := filepath.Join(dir, "snaps")
-
-	err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(snaps, "daily")
-	stderr := scanOK(t, daily)
-
-	warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
-	summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-		"0 unchanged), 1 skipped\n"
-
-	if !strings.Contains(stderr, warning) ||
-		!strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
-	}
-}
-
-// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
-// a symlink to a directory outside any .zfs directory. The operand is
-// scanned as usual, and its file is recorded by the path given.
-func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
-
-	data := filepath.Join(dir, "data")
-
-	err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(data, "daily")
-	stderr := scanOK(t, daily)
-
-	summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-		"0 unchanged), 0 skipped\n"
-	if !strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q", stderr, summary)
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
-}
-
 func TestRunReportSucceeds(t *testing.T) {
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)

scan.go

@@ -48,6 +48,11 @@ const sampleWindow = 1024 * 1024
 // and hash worker pools.
 const workQueueDepth = 1024
 
+// zfsDirName is the name of the directory in which ZFS shows a
+// filesystem's snapshots. The walk never enters one, since it would
+// list every file once per snapshot.
+const zfsDirName = ".zfs"
+
 // errInterrupted reports a scan stopped by SIGINT or SIGTERM. runScan
 // has already printed its line, so run prints nothing more.
 var errInterrupted = errors.New("scan interrupted")
@@ -272,9 +277,10 @@ func pruneRoots(roots []string, oneFS bool, rules walkRules,
 }
 
 // walkReaches reports whether the walk of operand above reaches r, an
-// operand under it. That walk does not enter a default exclude in
+// operand under it. That walk never follows a symlink. It does not
-// rules, nor a directory that device puts on a filesystem in rules that
+// enter a directory named .zfs, nor a default exclude in rules, nor a
-// is not persistent, nor, with -x, one that device puts on a different
+// directory that device puts on a filesystem in rules that is not
+// persistent, nor, with -x, one that device puts on a different
 // filesystem than above, when device knows above's. It reaches every
 // regular file in a directory it enters, so for a regular-file operand
 // the directories checked start at its directory; they end below above.
@@ -295,6 +301,15 @@ func walkReaches(above, r string, oneFS bool, rules walkRules,
 			return false
 		}
 
+		if filepath.Base(dir) == zfsDirName {
+			return false
+		}
+
+		fi, err = os.Lstat(dir)
+		if err == nil && fi.Mode()&fs.ModeSymlink != 0 {
+			return false
+		}
+
 		dev, ok := device(dir)
 		if !ok {
 			continue
@@ -383,8 +398,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 ) error {
 	// Types and filesystems are checked before pruning so that an
 	// operand under a dropped one is still scanned, not dropped as lying
-	// under it, and so that an operand inside a .zfs directory gets its
+	// under it.
-	// warning even when it lies under another operand.
 	roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
 		deviceOfPath)
 
@@ -412,15 +426,15 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 }
 
 // walkableRoots returns the operands the walk can start from: regular
-// files, and directories not named .zfs, none of them inside a
+// files, and directories not named .zfs. Every other operand is warned
-// directory named .zfs. Every other operand is warned about, counted as
+// about, counted as skipped, and dropped. A dropped operand is no
-// skipped, and dropped. A dropped operand is no longer a root, so the
+// longer a root, so the records stored beneath it are treated as if it
-// records stored beneath it are treated as if it had not been given:
+// had not been given: those under no other root count as outside the
-// those under no other root count as outside the roots and are not
+// roots and are not deleted as unverified. An operand on a filesystem
-// deleted as unverified. An operand on a filesystem in rules that is not
+// in rules that is not persistent is dropped the same way, with a line
-// persistent is dropped the same way, with a line saying so that is not
+// saying so that is not a warning, and is not counted as skipped. An
-// a warning, and is not counted as skipped. An operand that fails lstat
+// operand that fails lstat here is kept, and the walk warns about it
-// here is kept, and the walk warns about it unless it no longer exists.
+// unless it no longer exists.
 func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
 	kept := make([]string, 0, len(roots))
 
@@ -875,31 +889,16 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
 
 // operandWarning returns the one-line warning for an operand the walk
 // does not start from, naming the path and what it is, or "" for one it
-// does: a regular file, or a directory not named .zfs, that is not
+// does: a regular file, or a directory not named .zfs. Symlinks are
-// inside a directory named .zfs, neither as given nor with its symlinks
+// never followed, including as operands.
-// resolved. Symlinks are never followed, including as operands.
 func operandWarning(root string, fi fs.FileInfo) string {
-	// The walk never enters a .zfs directory, so it starts from nothing
-	// inside one either, not even through a symlink. An operand whose
-	// symlinks cannot be resolved is checked as given.
-	resolved, err := filepath.EvalSymlinks(root)
-	if err != nil {
-		resolved = root
-	}
-
-	if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
-		slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
-		return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
-			root)
-	}
-
 	var kind string
 
 	switch mode := fi.Mode(); {
 	case mode.IsRegular():
 		return ""
 	case mode.IsDir():
-		if filepath.Base(root) != ".zfs" {
+		if filepath.Base(root) != zfsDirName {
 			return ""
 		}
 
@@ -1169,7 +1168,7 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
 	parent dirJob, oneFS bool, rules walkRules,
 	events chan<- walkEvent,
 ) (dirJob, bool) {
-	if e.Name() == ".zfs" {
+	if e.Name() == zfsDirName {
 		return dirJob{}, false
 	}
 

scan_test.go

@@ -1672,6 +1672,81 @@ func TestSyncScanOverlappingRoots(t *testing.T) {
 	}
 }
 
+// TestScanWalksOperandInZFSDirectory checks that an operand inside a
+// directory named .zfs below another operand is walked on its own,
+// since the walk of the other operand does not enter .zfs. The file
+// under it is recorded, and a second scan keeps the record.
+func TestScanWalksOperandInZFSDirectory(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	f := writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
+	daily := filepath.Dir(f)
+
+	syncTree(t, db, dir, daily)
+	recordByPath(t, dbRecords(t, db), f)
+
+	syncTree(t, db, dir, daily)
+	recordByPath(t, dbRecords(t, db), f)
+}
+
+// TestScanWalksOperandBehindSymlink checks that an operand reached
+// through a symlinked directory below another operand is walked on its
+// own, since the walk of the other operand does not follow the symlink.
+// The file under it is recorded, and a second scan keeps the record.
+func TestScanWalksOperandBehindSymlink(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	target := t.TempDir()
+	db := openTestDB(t)
+	writeFile(t, target, "sub/f", pattern(1, 10))
+
+	link := filepath.Join(dir, "link")
+
+	err := os.Symlink(target, link)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sub := filepath.Join(link, "sub")
+	f := filepath.Join(sub, "f")
+
+	syncTree(t, db, dir, sub)
+	recordByPath(t, dbRecords(t, db), f)
+
+	syncTree(t, db, dir, sub)
+	recordByPath(t, dbRecords(t, db), f)
+}
+
+// TestScanDropsOperandReachedBehindSymlink checks that an operand under
+// another one is still dropped when both lie behind the same symlinked
+// directory: the walk of the other operand starts past the symlink and
+// reaches it, so the file under it is walked once.
+func TestScanDropsOperandReachedBehindSymlink(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	target := t.TempDir()
+	db := openTestDB(t)
+	writeFile(t, target, "sub/deeper/f", pattern(1, 10))
+
+	link := filepath.Join(dir, "link")
+
+	err := os.Symlink(target, link)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sub := filepath.Join(link, "sub")
+
+	st := syncTree(t, db, sub, filepath.Join(sub, "deeper"))
+	if st != (scanStats{walked: 1, added: 1}) {
+		t.Fatalf("stats = %+v, want 1 walked and added", st)
+	}
+}
+
 // TestScanHashesUniqueSize checks that a file whose size no other file
 // shares is hashed like every other file.
 func TestScanHashesUniqueSize(t *testing.T) {
@@ -2085,6 +2160,16 @@ func TestPruneRoots(t *testing.T) {
 		t.Fatalf("pruneRoots with %s not persistent = %q, want %q",
 			pqRoot, got, want)
 	}
+
+	// Nor does it enter a directory named .zfs, so /p/.zfs/s is kept. The
+	// walk of /p/.zfs/s reaches /p/.zfs/s/t, which is dropped.
+	got = pruneRoots([]string{"/p", "/p/.zfs/s", "/p/.zfs/s/t"}, false,
+		walkRules{}, device)
+
+	want = []string{"/p", "/p/.zfs/s"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("pruneRoots with .zfs = %q, want %q", got, want)
+	}
 }
 
 func TestReportsNeverTouchFilesystem(t *testing.T) {