Keep records on other filesystems under an operand with -x (closes #99) #105

Merged
clawbot merged 1 commits from issue-99-xdev-records into next 2026-10-14 20:22:51 +02:00
4 changed files with 134 additions and 12 deletions
+8 -1
View File
@@ -447,6 +447,11 @@ operands:
a `content` hash is read for one, wherever it lies, once its size, `head`, and
`tail` match another record's. If that file is gone or has changed since its
record was written, the record is left as it is.
- With `-x`, a record below a directory the walk did not enter because that
directory is on another filesystem than its operand counts as outside the
scanned operands: it is not deleted, and its size joins the size census. A
filesystem mounted under an operand can therefore be scanned on its own
schedule into the same database.
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
inside each phase; batched database writes begin during the hash phase:
@@ -513,7 +518,9 @@ Rules for the walk:
- Filesystem boundaries are crossed by default. With `-x` (long form
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
into a directory on a different filesystem than its `PATH` operand; each
operand is bounded by its own filesystem.
operand is bounded by its own filesystem. The records below such a directory
count as outside the scanned operands, as described in "`scan` mode" above:
they are not deleted, and their sizes join the size census.
- On any per-path error (permission denied, file vanished between passes,
unreadable): print a one-line warning to stderr, skip the path, and continue.
Per-file errors never abort the run; the final summary reports how many were
+4
View File
@@ -29,6 +29,10 @@
# Completed Steps
- with `-x`, `scan` keeps the records below a directory on another filesystem
under its operand, and counts their sizes in the size census, instead of
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
- the hash and content phases open each file without blocking and check that it
is still a regular file, so one replaced by a FIFO or device node after it was
statted is warned about and skipped instead of hanging the scan (2026-10-14,
+34 -8
View File
@@ -391,15 +391,26 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
// (the content phase may still fill in its content hash). It returns
// the new-or-changed files and the unchanged files whose records lack
// hashes; both remain candidates until the census decides whether
// their sizes are shared.
// their sizes are shared. A record below a directory -x did not enter
// counts as outside the roots: it is not deleted, and its size joins
// the census.
func (s *scanState) walkPhase(
events <-chan walkEvent,
) ([]fileRec, []fileRec) {
prog := newProgress("walk", -1)
var changed, unhashed []fileRec
var (
changed, unhashed []fileRec
otherFSDirs []string
)
for ev := range events {
if ev.otherFSDir != "" {
otherFSDirs = append(otherFSDirs, ev.otherFSDir)
continue
}
if ev.fail {
s.st.skipped++
@@ -429,6 +440,14 @@ func (s *scanState) walkPhase(
prog.finish()
for path, old := range s.existing {
if underAnyRoot(path, otherFSDirs) {
delete(s.existing, path)
s.sizes = append(s.sizes, old.size)
}
}
return changed, unhashed
}
@@ -866,16 +885,20 @@ type dirJob struct {
}
// walkEvent is one walk result delivered to the walk phase: a regular
// file's statted record, or a warning when fail is set.
// file's statted record, a warning when fail is set, or, when
// otherFSDir is set, a directory -x did not enter because it is on
// another filesystem than its operand.
type walkEvent struct {
rec fileRec
warn string
fail bool
rec fileRec
warn string
fail bool
otherFSDir string
}
// startWalk seeds every root into the shared walk worker pool and
// returns the event stream: one record per regular file, one warning
// event per per-path error. The channel is closed when the walk
// event per per-path error, and one event per directory -x did not
// enter. The channel is closed when the walk
// completes, and also when ctx is cancelled — every goroutine in the
// pool abandons its blocking send in that case, so the consumer sees a
// truncated but properly terminated stream instead of a stalled one.
@@ -1144,7 +1167,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
// subdirJob applies the descent rules to directory p: never enter
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
// snapshot), and with -x never enter a directory on a different
// filesystem than its operand.
// filesystem than its operand, sending an event that names it so the
// scan keeps the records below it.
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
parent dirJob, oneFS bool, events chan<- walkEvent,
) (dirJob, bool) {
@@ -1168,6 +1192,8 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
}
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
sendEvent(ctx, events, walkEvent{otherFSDir: p})
return dirJob{}, false
}
+88 -3
View File
@@ -997,7 +997,7 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry {
}
// callSubdirJob runs subdirJob against e under parent, collecting any
// warning events it emits (subdirJob emits at most one).
// events it emits (subdirJob emits at most one).
func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
parent dirJob, oneFS bool,
) (dirJob, bool, []walkEvent) {
@@ -1075,8 +1075,15 @@ func TestSubdirJobOneFilesystem(t *testing.T) {
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
}
if len(evs) != 0 {
t.Fatalf("unexpected events: %+v", evs)
// A refused descent names the directory, so the scan keeps
// the records below it.
var wantEvs []walkEvent
if !ok {
wantEvs = []walkEvent{{otherFSDir: sub}}
}
if !slices.Equal(evs, wantEvs) {
t.Fatalf("events = %+v, want %+v", evs, wantEvs)
}
// The accepted job must carry the operand's device down, or -x
@@ -1128,6 +1135,84 @@ func TestSubdirJobStatError(t *testing.T) {
}
}
// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
// record below a directory the walk does not enter, because it is on
// another filesystem than its operand, counts as outside the operands:
// it is not deleted, and its size joins the size census. The record of
// a gone file that is not below that directory is still deleted, even
// when its path begins with the directory's name. The walk starts from
// an operand device that no directory here is on, so no second real
// filesystem is needed; the phases after it run as runPhases runs them.
func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
// A scan without -x records the file below mnt, and mnt2, which is
// beside mnt, not below it.
mounted := writeFile(t, dir, "mnt/f", pattern(1, 100))
gone := writeFile(t, dir, "mnt2", pattern(3, 50))
syncTree(t, db, dir)
err := os.Remove(gone)
if err != nil {
t.Fatal(err)
}
// The same size as mnt/f, so it is hashed only if mnt/f's size is
// in the census.
added := writeFile(t, dir, "added", pattern(2, 100))
info, err := os.Lstat(dir)
if err != nil {
t.Fatal(err)
}
dev, ok := deviceOfInfo(info)
if !ok {
t.Skip("platform exposes no device id")
}
ctx := t.Context()
s := &scanState{db: db}
err = s.loadIndex(ctx, []string{dir})
if err != nil {
t.Fatal(err)
}
jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true}
dispatchDirs(ctx, []dirJob{root}, jobs, subdirs)
changed, unhashed := s.walkPhase(events)
s.partition(changed, unhashed)
err = s.hashPhase(ctx, 2)
if err != nil {
t.Fatal(err)
}
err = s.updatePhase(ctx)
if err != nil {
t.Fatal(err)
}
recs := dbRecords(t, db)
recordByPath(t, recs, mounted)
if recordByPath(t, recs, added).head == "" {
t.Errorf("%s not hashed: the census lacks the size of %s",
added, mounted)
}
if slices.Contains(recordPaths(recs), gone) {
t.Errorf("record for %s kept: the file is gone and not below mnt",
gone)
}
}
// buildSmokeTree recreates the README smoke-test filesystem layout
// with deterministic content and returns the tree root.
func buildSmokeTree(t *testing.T) string {