diff --git a/README.md b/README.md index 27f10b4..40d47e1 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,11 @@ operands: a `content` hash is read for one, wherever it lies, once its size, `head`, and `tail` match another record's. If that file is gone or has changed since its record was written, the record is left as it is. +- With `-x`, a record below a directory the walk did not enter because that + directory is on another filesystem than its operand counts as outside the + scanned operands: it is not deleted, and its size joins the size census. A + filesystem mounted under an operand can therefore be scanned on its own + schedule into the same database. `scan` runs **four sequential phases over the whole scan**. Parallelism lives inside each phase; batched database writes begin during the hash phase: @@ -513,7 +518,9 @@ Rules for the walk: - Filesystem boundaries are crossed by default. With `-x` (long form `--one-file-system`, following the GNU `du`/`rsync` convention), never descend into a directory on a different filesystem than its `PATH` operand; each - operand is bounded by its own filesystem. + operand is bounded by its own filesystem. The records below such a directory + count as outside the scanned operands, as described in "`scan` mode" above: + they are not deleted, and their sizes join the size census. - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were diff --git a/TODO.md b/TODO.md index 423e692..c588fef 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,10 @@ # Completed Steps +- with `-x`, `scan` keeps the records below a directory on another filesystem + under its operand, and counts their sizes in the size census, instead of + deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99) + - the hash and content phases open each file without blocking and check that it is still a regular file, so one replaced by a FIFO or device node after it was statted is warned about and skipped instead of hanging the scan (2026-10-14, diff --git a/scan.go b/scan.go index 1b8eb68..f4a031c 100644 --- a/scan.go +++ b/scan.go @@ -391,15 +391,26 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // (the content phase may still fill in its content hash). It returns // the new-or-changed files and the unchanged files whose records lack // hashes; both remain candidates until the census decides whether -// their sizes are shared. +// their sizes are shared. A record below a directory -x did not enter +// counts as outside the roots: it is not deleted, and its size joins +// the census. func (s *scanState) walkPhase( events <-chan walkEvent, ) ([]fileRec, []fileRec) { prog := newProgress("walk", -1) - var changed, unhashed []fileRec + var ( + changed, unhashed []fileRec + otherFSDirs []string + ) for ev := range events { + if ev.otherFSDir != "" { + otherFSDirs = append(otherFSDirs, ev.otherFSDir) + + continue + } + if ev.fail { s.st.skipped++ @@ -429,6 +440,14 @@ func (s *scanState) walkPhase( prog.finish() + for path, old := range s.existing { + if underAnyRoot(path, otherFSDirs) { + delete(s.existing, path) + + s.sizes = append(s.sizes, old.size) + } + } + return changed, unhashed } @@ -866,16 +885,20 @@ type dirJob struct { } // walkEvent is one walk result delivered to the walk phase: a regular -// file's statted record, or a warning when fail is set. +// file's statted record, a warning when fail is set, or, when +// otherFSDir is set, a directory -x did not enter because it is on +// another filesystem than its operand. type walkEvent struct { - rec fileRec - warn string - fail bool + rec fileRec + warn string + fail bool + otherFSDir string } // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning -// event per per-path error. The channel is closed when the walk +// event per per-path error, and one event per directory -x did not +// enter. The channel is closed when the walk // completes, and also when ctx is cancelled — every goroutine in the // pool abandons its blocking send in that case, so the consumer sees a // truncated but properly terminated stream instead of a stalled one. @@ -1144,7 +1167,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // subdirJob applies the descent rules to directory p: never enter // .zfs (ZFS snapshot pseudo-dirs would list every file once per // snapshot), and with -x never enter a directory on a different -// filesystem than its operand. +// filesystem than its operand, sending an event that names it so the +// scan keeps the records below it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, parent dirJob, oneFS bool, events chan<- walkEvent, ) (dirJob, bool) { @@ -1168,6 +1192,8 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry, } if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev { + sendEvent(ctx, events, walkEvent{otherFSDir: p}) + return dirJob{}, false } diff --git a/scan_test.go b/scan_test.go index 7d3c705..56880ab 100644 --- a/scan_test.go +++ b/scan_test.go @@ -997,7 +997,7 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry { } // callSubdirJob runs subdirJob against e under parent, collecting any -// warning events it emits (subdirJob emits at most one). +// events it emits (subdirJob emits at most one). func callSubdirJob(t *testing.T, p string, e fs.DirEntry, parent dirJob, oneFS bool, ) (dirJob, bool, []walkEvent) { @@ -1075,8 +1075,15 @@ func TestSubdirJobOneFilesystem(t *testing.T) { t.Fatalf("accepted = %v, want %v", ok, tc.wantOK) } - if len(evs) != 0 { - t.Fatalf("unexpected events: %+v", evs) + // A refused descent names the directory, so the scan keeps + // the records below it. + var wantEvs []walkEvent + if !ok { + wantEvs = []walkEvent{{otherFSDir: sub}} + } + + if !slices.Equal(evs, wantEvs) { + t.Fatalf("events = %+v, want %+v", evs, wantEvs) } // The accepted job must carry the operand's device down, or -x @@ -1128,6 +1135,84 @@ func TestSubdirJobStatError(t *testing.T) { } } +// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a +// record below a directory the walk does not enter, because it is on +// another filesystem than its operand, counts as outside the operands: +// it is not deleted, and its size joins the size census. The record of +// a gone file that is not below that directory is still deleted, even +// when its path begins with the directory's name. The walk starts from +// an operand device that no directory here is on, so no second real +// filesystem is needed; the phases after it run as runPhases runs them. +func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + + // A scan without -x records the file below mnt, and mnt2, which is + // beside mnt, not below it. + mounted := writeFile(t, dir, "mnt/f", pattern(1, 100)) + gone := writeFile(t, dir, "mnt2", pattern(3, 50)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + // The same size as mnt/f, so it is hashed only if mnt/f's size is + // in the census. + added := writeFile(t, dir, "added", pattern(2, 100)) + + info, err := os.Lstat(dir) + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfInfo(info) + if !ok { + t.Skip("platform exposes no device id") + } + + ctx := t.Context() + s := &scanState{db: db} + + err = s.loadIndex(ctx, []string{dir}) + if err != nil { + t.Fatal(err) + } + + jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true} + dispatchDirs(ctx, []dirJob{root}, jobs, subdirs) + + changed, unhashed := s.walkPhase(events) + s.partition(changed, unhashed) + + err = s.hashPhase(ctx, 2) + if err != nil { + t.Fatal(err) + } + + err = s.updatePhase(ctx) + if err != nil { + t.Fatal(err) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, mounted) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, mounted) + } + + if slices.Contains(recordPaths(recs), gone) { + t.Errorf("record for %s kept: the file is gone and not below mnt", + gone) + } +} + // buildSmokeTree recreates the README smoke-test filesystem layout // with deterministic content and returns the tree root. func buildSmokeTree(t *testing.T) string {