diff --git a/README.md b/README.md index 3a6ce3f..63bfd51 100644 --- a/README.md +++ b/README.md @@ -455,11 +455,17 @@ scan and are enumerated concurrently: every operand seeds the shared walk worker pool. Overlapping operands are harmless — an operand that duplicates another or lies under another is dropped before walking, so every file is reached exactly once and produces one database record. An operand under another one is kept, -though, and walked on its own when any directory from it (from its directory, -for a regular file) up to but not including the operand covering it is a default -exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with -`-x`, is on a different filesystem than that operand, because that operand's -walk does not enter such a directory and so does not reach it. +though, when the walk of the outer operand does not reach it. It is then walked +on its own. The walk of the outer operand does not reach the inner one when it, +or any directory between the two, is: + +- named `.zfs`; +- a symlink; +- a default exclude; +- not on a persistent filesystem (see "Rules for the walk"); +- with `-x`, on a different filesystem than the outer operand. + +For a regular-file operand, only the directories between the two count. An operand that is a symlink (never followed, not even as an operand), socket, FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints diff --git a/TODO.md b/TODO.md index 039cd5e..595a03c 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,12 @@ # Completed Steps +- an operand under another is walked on its own when a directory between them, + the inner operand included, is named `.zfs` or is a symlink. The walk of the + outer operand does not reach it then. `scan` now records the files below it + and no longer deletes their records (2026-10-26, + https://git.eeqj.de/sneak/sfdupes/issues/142) + - a file or directory removed while the scan runs is not an error, in the walk or in the hash phase. The scan prints no line for it and does not count it as skipped. When the first of several hard links to a file is gone, the hash diff --git a/scan.go b/scan.go index 2aed86b..86aca64 100644 --- a/scan.go +++ b/scan.go @@ -48,6 +48,11 @@ const sampleWindow = 1024 * 1024 // and hash worker pools. const workQueueDepth = 1024 +// zfsDirName is the name of the directory in which ZFS shows a +// filesystem's snapshots. The walk never enters one, since it would +// list every file once per snapshot. +const zfsDirName = ".zfs" + // errInterrupted reports a scan stopped by SIGINT or SIGTERM. runScan // has already printed its line, so run prints nothing more. var errInterrupted = errors.New("scan interrupted") @@ -272,9 +277,10 @@ func pruneRoots(roots []string, oneFS bool, rules walkRules, } // walkReaches reports whether the walk of operand above reaches r, an -// operand under it. That walk does not enter a default exclude in -// rules, nor a directory that device puts on a filesystem in rules that -// is not persistent, nor, with -x, one that device puts on a different +// operand under it. That walk never follows a symlink. It does not +// enter a directory named .zfs, nor a default exclude in rules, nor a +// directory that device puts on a filesystem in rules that is not +// persistent, nor, with -x, one that device puts on a different // filesystem than above, when device knows above's. It reaches every // regular file in a directory it enters, so for a regular-file operand // the directories checked start at its directory; they end below above. @@ -295,6 +301,15 @@ func walkReaches(above, r string, oneFS bool, rules walkRules, return false } + if filepath.Base(dir) == zfsDirName { + return false + } + + fi, err = os.Lstat(dir) + if err == nil && fi.Mode()&fs.ModeSymlink != 0 { + return false + } + dev, ok := device(dir) if !ok { continue @@ -883,7 +898,7 @@ func operandWarning(root string, fi fs.FileInfo) string { case mode.IsRegular(): return "" case mode.IsDir(): - if filepath.Base(root) != ".zfs" { + if filepath.Base(root) != zfsDirName { return "" } @@ -1153,7 +1168,7 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry, parent dirJob, oneFS bool, rules walkRules, events chan<- walkEvent, ) (dirJob, bool) { - if e.Name() == ".zfs" { + if e.Name() == zfsDirName { return dirJob{}, false } diff --git a/scan_test.go b/scan_test.go index 58ab1db..95cf9d3 100644 --- a/scan_test.go +++ b/scan_test.go @@ -1672,6 +1672,81 @@ func TestSyncScanOverlappingRoots(t *testing.T) { } } +// TestScanWalksOperandInZFSDirectory checks that an operand inside a +// directory named .zfs below another operand is walked on its own, +// since the walk of the other operand does not enter .zfs. The file +// under it is recorded, and a second scan keeps the record. +func TestScanWalksOperandInZFSDirectory(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + f := writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10)) + daily := filepath.Dir(f) + + syncTree(t, db, dir, daily) + recordByPath(t, dbRecords(t, db), f) + + syncTree(t, db, dir, daily) + recordByPath(t, dbRecords(t, db), f) +} + +// TestScanWalksOperandBehindSymlink checks that an operand reached +// through a symlinked directory below another operand is walked on its +// own, since the walk of the other operand does not follow the symlink. +// The file under it is recorded, and a second scan keeps the record. +func TestScanWalksOperandBehindSymlink(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + target := t.TempDir() + db := openTestDB(t) + writeFile(t, target, "sub/f", pattern(1, 10)) + + link := filepath.Join(dir, "link") + + err := os.Symlink(target, link) + if err != nil { + t.Fatal(err) + } + + sub := filepath.Join(link, "sub") + f := filepath.Join(sub, "f") + + syncTree(t, db, dir, sub) + recordByPath(t, dbRecords(t, db), f) + + syncTree(t, db, dir, sub) + recordByPath(t, dbRecords(t, db), f) +} + +// TestScanDropsOperandReachedBehindSymlink checks that an operand under +// another one is still dropped when both lie behind the same symlinked +// directory: the walk of the other operand starts past the symlink and +// reaches it, so the file under it is walked once. +func TestScanDropsOperandReachedBehindSymlink(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + target := t.TempDir() + db := openTestDB(t) + writeFile(t, target, "sub/deeper/f", pattern(1, 10)) + + link := filepath.Join(dir, "link") + + err := os.Symlink(target, link) + if err != nil { + t.Fatal(err) + } + + sub := filepath.Join(link, "sub") + + st := syncTree(t, db, sub, filepath.Join(sub, "deeper")) + if st != (scanStats{walked: 1, added: 1}) { + t.Fatalf("stats = %+v, want 1 walked and added", st) + } +} + // TestScanHashesUniqueSize checks that a file whose size no other file // shares is hashed like every other file. func TestScanHashesUniqueSize(t *testing.T) { @@ -2085,6 +2160,16 @@ func TestPruneRoots(t *testing.T) { t.Fatalf("pruneRoots with %s not persistent = %q, want %q", pqRoot, got, want) } + + // Nor does it enter a directory named .zfs, so /p/.zfs/s is kept. The + // walk of /p/.zfs/s reaches /p/.zfs/s/t, which is dropped. + got = pruneRoots([]string{"/p", "/p/.zfs/s", "/p/.zfs/s/t"}, false, + walkRules{}, device) + + want = []string{"/p", "/p/.zfs/s"} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots with .zfs = %q, want %q", got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) {