diff --git a/README.md b/README.md index 40d47e1..2a1bb62 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,11 @@ results do not depend on the working directory. All operands belong to a single scan and are enumerated concurrently: every operand seeds the shared walk worker pool. Overlapping operands are harmless — an operand that duplicates another or lies under another is dropped before walking, so every file is reached exactly -once and produces one database record. +once and produces one database record. With `-x`, though, an operand under +another one is kept and walked on its own when any directory from it (from its +directory, for a regular file) up to but not including the operand covering it +is on a different filesystem than that operand, because that operand's walk does +not enter such a directory and so does not reach it. An operand that is a symlink (never followed, not even as an operand), socket, FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints @@ -407,9 +411,8 @@ drops it from the scanned operands before reading the database. Another operand beneath it is still scanned. The records stored beneath it are not deleted: they are treated like any other record outside the scanned operands, including the content-phase exception below. If it lies under another operand, they are under -that operand instead, and are deleted like any other record there that this scan -did not verify. This is not an error: a scan whose every operand is dropped -walks nothing and exits 0. +that operand instead, and are treated like any other record there. This is not +an error: a scan whose every operand is dropped walks nothing and exits 0. `scan` synchronizes the database with the filesystem state under the scanned operands: @@ -451,7 +454,8 @@ operands: directory is on another filesystem than its operand counts as outside the scanned operands: it is not deleted, and its size joins the size census. A filesystem mounted under an operand can therefore be scanned on its own - schedule into the same database. + schedule into the same database. Records below an operand at or inside such a + directory are not kept this way, since that operand's own walk verifies them. `scan` runs **four sequential phases over the whole scan**. Parallelism lives inside each phase; batched database writes begin during the hash phase: @@ -518,17 +522,19 @@ Rules for the walk: - Filesystem boundaries are crossed by default. With `-x` (long form `--one-file-system`, following the GNU `du`/`rsync` convention), never descend into a directory on a different filesystem than its `PATH` operand; each - operand is bounded by its own filesystem. The records below such a directory - count as outside the scanned operands, as described in "`scan` mode" above: - they are not deleted, and their sizes join the size census. + operand is bounded by its own filesystem. The records below such a directory, + unless they lie below an operand at or inside that directory, count as outside + the scanned operands, as described in "`scan` mode" above: they are not + deleted, and their sizes join the size census. - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were skipped. As specified above, a skipped path that has a database record from an earlier scan loses that record, unless it failed only in the content phase, or - is an operand dropped before the database was read that lies under no other - operand; an unreadable directory subtree likewise loses its records (accepted: - the database mirrors what the latest scan could actually verify). + is an operand dropped before the database was read whose record counts as + outside the scanned operands; an unreadable directory subtree likewise loses + its records (accepted: the database mirrors what the latest scan could + actually verify). Concurrency: the walk phase (which also stats files), the hash phase, and the content phase each use a worker pool of `--workers` workers (default diff --git a/TODO.md b/TODO.md index c588fef..185157f 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ # Completed Steps +- with `-x`, an operand under another operand is walked on its own instead of + being dropped when a directory from it (from its directory, for a regular + file) up to but not including that operand is on a different filesystem than + that operand, whose walk then cannot reach it, and the records of files gone + from below it are deleted (2026-10-14, + https://git.eeqj.de/sneak/sfdupes/issues/106) + - with `-x`, `scan` keeps the records below a directory on another filesystem under its operand, and counts their sizes in the size census, instead of deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99) diff --git a/scan.go b/scan.go index f4a031c..0f70378 100644 --- a/scan.go +++ b/scan.go @@ -203,10 +203,16 @@ func resolveRoots(roots []string) ([]string, error) { } // pruneRoots drops operands already covered by another operand: -// duplicates and any operand lying under another one. Every remaining -// file is then reachable through exactly one root, so walked paths are -// unique without keeping a scan-wide set of every path seen. -func pruneRoots(roots []string) []string { +// duplicates and any operand lying under another one. With -x, an +// operand under another one is kept when the walk of the deepest kept +// operand above it cannot reach it: when device puts any directory from +// it (from its directory, for a regular file) up to but not including +// that operand on a different filesystem than that operand. Every +// remaining file is then reachable through exactly one root, so walked +// paths are unique without keeping a scan-wide set of every path seen. +func pruneRoots(roots []string, oneFS bool, + device func(path string) (uint64, bool), +) []string { // Shorter-first ordering guarantees an ancestor is kept before any // operand under it is considered. sorted := slices.Clone(roots) @@ -221,14 +227,58 @@ func pruneRoots(roots []string) []string { kept := make([]string, 0, len(sorted)) for _, r := range sorted { - if !underAnyRoot(r, kept) { + above := deepestRoot(r, kept) + if above == "" { kept = append(kept, r) + + continue + } + + // Without -x, and always for a duplicate, r is dropped. + if !oneFS || above == r { + continue + } + + aboveDev, aboveOK := device(above) + if !aboveOK { + continue + } + + // The walk of above enters a directory only when it is on + // above's filesystem, and then reaches every regular file in it + // whatever device the file reports, so for a regular-file + // operand the directories checked start at its directory. + dir := r + + fi, err := os.Lstat(r) + if err == nil && fi.Mode().IsRegular() { + dir = filepath.Dir(r) + } + + for ; dir != above; dir = filepath.Dir(dir) { + dev, ok := device(dir) + if ok && dev != aboveDev { + kept = append(kept, r) + + break + } } } return kept } +// deviceOfPath returns the filesystem device ID of the file at path, +// when lstat succeeds and the platform exposes one. +func deviceOfPath(path string) (uint64, bool) { + fi, err := os.Lstat(path) + if err != nil { + return 0, false + } + + return deviceOfInfo(fi) +} + // scanStats summarizes one scan's database synchronization for the // final stderr summary, or for the line an interrupted scan prints. type scanStats struct { @@ -288,21 +338,22 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string, // size, head, and tail match another record's). Records outside the // roots are never touched, except that the content phase fills in // their content hash. Operands the walk cannot start from are dropped -// first, so the records beneath them count as outside the roots unless -// they lie under another root. +// first, so the records beneath them are treated as if those operands +// had not been given. func (s *scanState) runPhases(ctx context.Context, roots []string, workers int, oneFS bool, ) error { // Types are checked before pruning so that an operand under a // dropped one is still scanned, not dropped as lying under it. - roots = pruneRoots(s.walkableRoots(roots)) + roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath) err := s.loadIndex(ctx, roots) if err != nil { return err } - changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers)) + changed, unhashed := s.walkPhase( + startWalk(ctx, roots, oneFS, workers), roots) // A cancelled walk stops early, so its size census covers only part // of the roots, and every file it never reached would look vanished @@ -330,10 +381,10 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, // walkableRoots returns the operands the walk can start from: regular // files, and directories not named .zfs. Every other operand is warned // about, counted as skipped, and dropped. A dropped operand is no -// longer a root, so the records stored beneath it count as outside the -// roots and are not deleted as unverified, unless it lies under another -// root. An operand that fails lstat here is kept, and the walk warns -// about it. +// longer a root, so the records stored beneath it are treated as if it +// had not been given: those under no other root count as outside the +// roots and are not deleted as unverified. An operand that fails lstat +// here is kept, and the walk warns about it. func (s *scanState) walkableRoots(roots []string) []string { kept := make([]string, 0, len(roots)) @@ -393,9 +444,10 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // hashes; both remain candidates until the census decides whether // their sizes are shared. A record below a directory -x did not enter // counts as outside the roots: it is not deleted, and its size joins -// the census. +// the census. That does not hold below a root at or inside that +// directory, since that root's own walk verifies the records below it. func (s *scanState) walkPhase( - events <-chan walkEvent, + events <-chan walkEvent, roots []string, ) ([]fileRec, []fileRec) { prog := newProgress("walk", -1) @@ -441,7 +493,12 @@ func (s *scanState) walkPhase( prog.finish() for path, old := range s.existing { - if underAnyRoot(path, otherFSDirs) { + // Kept only when a directory -x did not enter lies below the + // deepest root above the record. A root at or below that + // directory was walked on its own and did not find the file. + // Both contain the record, so the longer path is the deeper. + otherFSDir := deepestRoot(path, otherFSDirs) + if len(otherFSDir) > len(deepestRoot(path, roots)) { delete(s.existing, path) s.sizes = append(s.sizes, old.size) @@ -875,6 +932,20 @@ func underRoot(path, root string) bool { return strings.HasPrefix(path, prefix) } +// deepestRoot returns the longest of the roots that path is or lies +// under, or "" when it lies under none. +func deepestRoot(path string, roots []string) string { + deepest := "" + + for _, root := range roots { + if underRoot(path, root) && len(root) > len(deepest) { + deepest = root + } + } + + return deepest +} + // dirJob is one directory awaiting traversal by the walk workers. It // carries its operand's filesystem device so -x can stop at // filesystem boundaries. @@ -898,10 +969,10 @@ type walkEvent struct { // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning // event per per-path error, and one event per directory -x did not -// enter. The channel is closed when the walk -// completes, and also when ctx is cancelled — every goroutine in the -// pool abandons its blocking send in that case, so the consumer sees a -// truncated but properly terminated stream instead of a stalled one. +// enter. The channel is closed when the walk completes, and also when +// ctx is cancelled — every goroutine in the pool abandons its blocking +// send in that case, so the consumer sees a truncated but properly +// terminated stream instead of a stalled one. func startWalk(ctx context.Context, roots []string, oneFS bool, workers int, ) <-chan walkEvent { @@ -968,7 +1039,8 @@ func operandWarning(root string, fi fs.FileInfo) string { // operand becomes an initial job. walkableRoots has already dropped // every other operand. One that has changed into something else since // is warned about and skipped here; it is still a root, so the records -// stored beneath it are deleted as unverified. +// stored beneath it, other than those under another root, are deleted +// as unverified. func seedRoot(ctx context.Context, root string, events chan<- walkEvent, ) []dirJob { @@ -1168,7 +1240,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // .zfs (ZFS snapshot pseudo-dirs would list every file once per // snapshot), and with -x never enter a directory on a different // filesystem than its operand, sending an event that names it so the -// scan keeps the records below it. +// scan keeps the records below it, except those below an operand at or +// inside it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, parent dirJob, oneFS bool, events chan<- walkEvent, ) (dirJob, bool) { diff --git a/scan_test.go b/scan_test.go index 56880ab..778fe64 100644 --- a/scan_test.go +++ b/scan_test.go @@ -1135,14 +1135,72 @@ func TestSubdirJobStatError(t *testing.T) { } } +// syncOneFilesystem scans roots into db with -x and returns the scan +// stats. Every device pruneRoots looks up, and the device each +// operand's walk compares its subdirectories with, comes from devices +// instead of from lstat, so a test can put a path on a filesystem that +// no directory here is on. Unlike runPhases, it skips walkableRoots, +// the check for a cancelled walk, and the content phase. +func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, + devices map[string]uint64, +) scanStats { + t.Helper() + + device := func(path string) (uint64, bool) { + dev, ok := devices[path] + + return dev, ok + } + + roots = pruneRoots(roots, true, device) + + ctx := t.Context() + s := &scanState{db: db} + + err := s.loadIndex(ctx, roots) + if err != nil { + t.Fatal(err) + } + + jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + + // seedRoot emits a regular-file operand itself and returns a job + // only for a directory operand. + var initial []dirJob + + for _, root := range roots { + for _, job := range seedRoot(ctx, root, events) { + job.rootDev = devices[root] + initial = append(initial, job) + } + } + + dispatchDirs(ctx, initial, jobs, subdirs) + + changed, unhashed := s.walkPhase(events, roots) + s.partition(changed, unhashed) + + err = s.hashPhase(ctx, 2) + if err != nil { + t.Fatal(err) + } + + err = s.updatePhase(ctx) + if err != nil { + t.Fatal(err) + } + + return s.st +} + // TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a // record below a directory the walk does not enter, because it is on // another filesystem than its operand, counts as outside the operands: // it is not deleted, and its size joins the size census. The record of // a gone file that is not below that directory is still deleted, even -// when its path begins with the directory's name. The walk starts from -// an operand device that no directory here is on, so no second real -// filesystem is needed; the phases after it run as runPhases runs them. +// when its path begins with the directory's name. The operand is given +// a device that no directory here is on, so no second real filesystem +// is needed. func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { t.Parallel() @@ -1174,30 +1232,7 @@ func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { t.Skip("platform exposes no device id") } - ctx := t.Context() - s := &scanState{db: db} - - err = s.loadIndex(ctx, []string{dir}) - if err != nil { - t.Fatal(err) - } - - jobs, subdirs, events := startWalkWorkers(ctx, 2, true) - root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true} - dispatchDirs(ctx, []dirJob{root}, jobs, subdirs) - - changed, unhashed := s.walkPhase(events) - s.partition(changed, unhashed) - - err = s.hashPhase(ctx, 2) - if err != nil { - t.Fatal(err) - } - - err = s.updatePhase(ctx) - if err != nil { - t.Fatal(err) - } + syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1}) recs := dbRecords(t, db) recordByPath(t, recs, mounted) @@ -1213,6 +1248,183 @@ func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { } } +// TestScanOneFilesystemWalksNestedOperand checks that with -x an +// operand under another one, on a different filesystem than it, is +// walked on its own: a file added below it is recorded, and the record +// of a file gone from below it is deleted. A record below a directory +// that its own walk does not enter is still kept. Each operand is given +// a device that no directory here is on, so neither walk enters a +// subdirectory. +func TestScanOneFilesystemWalksNestedOperand(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records both files below mnt. + gone := writeFile(t, dir, "mnt/gone", pattern(1, 10)) + kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "mnt/added", pattern(3, 30)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + mnt := filepath.Join(dir, "mnt") + syncOneFilesystem(t, db, []string{dir, mnt}, + map[string]uint64{dir: dev + 1, mnt: dev + 2}) + + paths := recordPaths(dbRecords(t, db)) + + if !slices.Contains(paths, added) { + t.Errorf("no record for %s: the walk did not reach %s", added, mnt) + } + + if slices.Contains(paths, gone) { + t.Errorf("record for %s kept: the file is gone", gone) + } + + if !slices.Contains(paths, kept) { + t.Errorf("record for %s deleted: the walk of %s did not enter %s", + kept, mnt, filepath.Dir(kept)) + } +} + +// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that +// with -x an operand on the same filesystem as the operand covering it +// is walked on its own when a directory between them is on another +// filesystem, since the walk of the covering operand does not enter +// that directory: a file added below the nested operand is recorded, +// and the record of a file gone from below it is deleted. A record +// below that directory but outside the nested operand is still kept. +// Both operands are given one device that no directory here is on, +// while mnt keeps its own. +func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records both files below mnt. + gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10)) + kept := writeFile(t, dir, "mnt/kept", pattern(2, 20)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + added := writeFile(t, dir, "mnt/back/added", pattern(3, 30)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + mnt := filepath.Join(dir, "mnt") + back := filepath.Join(mnt, "back") + syncOneFilesystem(t, db, []string{dir, back}, + map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1}) + + paths := recordPaths(dbRecords(t, db)) + + if !slices.Contains(paths, added) { + t.Errorf("no record for %s: the walk did not reach %s", added, back) + } + + if slices.Contains(paths, gone) { + t.Errorf("record for %s kept: the file is gone", gone) + } + + if !slices.Contains(paths, kept) { + t.Errorf("record for %s deleted: the walk of %s did not enter %s", + kept, dir, mnt) + } +} + +// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a +// regular-file operand under another operand is dropped when the file +// reports a different device than that operand but its directory does +// not, as a file bind-mounted into the tree does: the walk of the other +// operand enters the directory and reaches the file, so the file is +// recorded once. +func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + file := writeFile(t, dir, "sub/f", pattern(1, 10)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{ + dir: dev, filepath.Dir(file): dev, file: dev + 1, + }) + + if st.added != 1 { + t.Errorf("%s recorded %d times, want once", file, st.added) + } +} + +// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a +// regular-file operand under another operand is kept when its directory +// is on a different filesystem than that operand: the walk of the other +// operand does not enter the directory, so the file is recorded once, +// as its own operand. The other operand is given a device that no +// directory here is on. +func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + file := writeFile(t, dir, "sub/f", pattern(1, 10)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{ + dir: dev + 1, filepath.Dir(file): dev, + }) + + if st.added != 1 { + t.Errorf("%s recorded %d times, want once", file, st.added) + } +} + // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string { @@ -2123,14 +2335,43 @@ func TestHashRuns(t *testing.T) { func TestPruneRoots(t *testing.T) { t.Parallel() + const pqRoot = "/p/q" + + // /p/q is on another filesystem than /p; /p/q/r is on the same one as + // /p/q, /p/s as /p, and /p/t has no known device. + devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1} + device := func(path string) (uint64, bool) { + dev, ok := devices[path] + + return dev, ok + } + // Duplicates and operands under other operands are dropped; /cc is // not under /c (sibling with a shared prefix). - got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}) + got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, + false, device) want := []string{"/a", "/c", "/cc"} if !slices.Equal(got, want) { t.Fatalf("pruneRoots = %q, want %q", got, want) } + + nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"} + + got = pruneRoots(nested, false, device) + + want = []string{"/p"} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots without -x = %q, want %q", got, want) + } + + // With -x, /p/q, on another filesystem than /p, is kept as well. + got = pruneRoots(nested, true, device) + + want = []string{"/p", pqRoot} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots with -x = %q, want %q", got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) {