Keep records on other filesystems under an operand with -x (closes #99) #105
@@ -447,6 +447,11 @@ operands:
|
||||
a `content` hash is read for one, wherever it lies, once its size, `head`, and
|
||||
`tail` match another record's. If that file is gone or has changed since its
|
||||
record was written, the record is left as it is.
|
||||
- With `-x`, a record below a directory the walk did not enter because that
|
||||
directory is on another filesystem than its operand counts as outside the
|
||||
scanned operands: it is not deleted, and its size joins the size census. A
|
||||
filesystem mounted under an operand can therefore be scanned on its own
|
||||
schedule into the same database.
|
||||
|
||||
`scan` runs **four sequential phases over the whole scan**. Parallelism lives
|
||||
inside each phase; batched database writes begin during the hash phase:
|
||||
@@ -513,7 +518,9 @@ Rules for the walk:
|
||||
- Filesystem boundaries are crossed by default. With `-x` (long form
|
||||
`--one-file-system`, following the GNU `du`/`rsync` convention), never descend
|
||||
into a directory on a different filesystem than its `PATH` operand; each
|
||||
operand is bounded by its own filesystem.
|
||||
operand is bounded by its own filesystem. The records below such a directory
|
||||
count as outside the scanned operands, as described in "`scan` mode" above:
|
||||
they are not deleted, and their sizes join the size census.
|
||||
- On any per-path error (permission denied, file vanished between passes,
|
||||
unreadable): print a one-line warning to stderr, skip the path, and continue.
|
||||
Per-file errors never abort the run; the final summary reports how many were
|
||||
|
||||
@@ -29,6 +29,10 @@
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
||||
under its operand, and counts their sizes in the size census, instead of
|
||||
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
||||
|
||||
- the hash and content phases open each file without blocking and check that it
|
||||
is still a regular file, so one replaced by a FIFO or device node after it was
|
||||
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
||||
|
||||
@@ -391,15 +391,26 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||
// (the content phase may still fill in its content hash). It returns
|
||||
// the new-or-changed files and the unchanged files whose records lack
|
||||
// hashes; both remain candidates until the census decides whether
|
||||
// their sizes are shared.
|
||||
// their sizes are shared. A record below a directory -x did not enter
|
||||
// counts as outside the roots: it is not deleted, and its size joins
|
||||
// the census.
|
||||
func (s *scanState) walkPhase(
|
||||
events <-chan walkEvent,
|
||||
) ([]fileRec, []fileRec) {
|
||||
prog := newProgress("walk", -1)
|
||||
|
||||
var changed, unhashed []fileRec
|
||||
var (
|
||||
changed, unhashed []fileRec
|
||||
otherFSDirs []string
|
||||
)
|
||||
|
||||
for ev := range events {
|
||||
if ev.otherFSDir != "" {
|
||||
otherFSDirs = append(otherFSDirs, ev.otherFSDir)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if ev.fail {
|
||||
s.st.skipped++
|
||||
|
||||
@@ -429,6 +440,14 @@ func (s *scanState) walkPhase(
|
||||
|
||||
prog.finish()
|
||||
|
||||
for path, old := range s.existing {
|
||||
if underAnyRoot(path, otherFSDirs) {
|
||||
delete(s.existing, path)
|
||||
|
||||
s.sizes = append(s.sizes, old.size)
|
||||
}
|
||||
}
|
||||
|
||||
return changed, unhashed
|
||||
}
|
||||
|
||||
@@ -866,16 +885,20 @@ type dirJob struct {
|
||||
}
|
||||
|
||||
// walkEvent is one walk result delivered to the walk phase: a regular
|
||||
// file's statted record, or a warning when fail is set.
|
||||
// file's statted record, a warning when fail is set, or, when
|
||||
// otherFSDir is set, a directory -x did not enter because it is on
|
||||
// another filesystem than its operand.
|
||||
type walkEvent struct {
|
||||
rec fileRec
|
||||
warn string
|
||||
fail bool
|
||||
rec fileRec
|
||||
warn string
|
||||
fail bool
|
||||
otherFSDir string
|
||||
}
|
||||
|
||||
// startWalk seeds every root into the shared walk worker pool and
|
||||
// returns the event stream: one record per regular file, one warning
|
||||
// event per per-path error. The channel is closed when the walk
|
||||
// event per per-path error, and one event per directory -x did not
|
||||
// enter. The channel is closed when the walk
|
||||
// completes, and also when ctx is cancelled — every goroutine in the
|
||||
// pool abandons its blocking send in that case, so the consumer sees a
|
||||
// truncated but properly terminated stream instead of a stalled one.
|
||||
@@ -1144,7 +1167,8 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
||||
// subdirJob applies the descent rules to directory p: never enter
|
||||
// .zfs (ZFS snapshot pseudo-dirs would list every file once per
|
||||
// snapshot), and with -x never enter a directory on a different
|
||||
// filesystem than its operand.
|
||||
// filesystem than its operand, sending an event that names it so the
|
||||
// scan keeps the records below it.
|
||||
func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||
parent dirJob, oneFS bool, events chan<- walkEvent,
|
||||
) (dirJob, bool) {
|
||||
@@ -1168,6 +1192,8 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
||||
}
|
||||
|
||||
if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
|
||||
sendEvent(ctx, events, walkEvent{otherFSDir: p})
|
||||
|
||||
return dirJob{}, false
|
||||
}
|
||||
|
||||
|
||||
+88
-3
@@ -997,7 +997,7 @@ func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry {
|
||||
}
|
||||
|
||||
// callSubdirJob runs subdirJob against e under parent, collecting any
|
||||
// warning events it emits (subdirJob emits at most one).
|
||||
// events it emits (subdirJob emits at most one).
|
||||
func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
|
||||
parent dirJob, oneFS bool,
|
||||
) (dirJob, bool, []walkEvent) {
|
||||
@@ -1075,8 +1075,15 @@ func TestSubdirJobOneFilesystem(t *testing.T) {
|
||||
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
|
||||
}
|
||||
|
||||
if len(evs) != 0 {
|
||||
t.Fatalf("unexpected events: %+v", evs)
|
||||
// A refused descent names the directory, so the scan keeps
|
||||
// the records below it.
|
||||
var wantEvs []walkEvent
|
||||
if !ok {
|
||||
wantEvs = []walkEvent{{otherFSDir: sub}}
|
||||
}
|
||||
|
||||
if !slices.Equal(evs, wantEvs) {
|
||||
t.Fatalf("events = %+v, want %+v", evs, wantEvs)
|
||||
}
|
||||
|
||||
// The accepted job must carry the operand's device down, or -x
|
||||
@@ -1128,6 +1135,84 @@ func TestSubdirJobStatError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
|
||||
// record below a directory the walk does not enter, because it is on
|
||||
// another filesystem than its operand, counts as outside the operands:
|
||||
// it is not deleted, and its size joins the size census. The record of
|
||||
// a gone file that is not below that directory is still deleted, even
|
||||
// when its path begins with the directory's name. The walk starts from
|
||||
// an operand device that no directory here is on, so no second real
|
||||
// filesystem is needed; the phases after it run as runPhases runs them.
|
||||
func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
|
||||
// A scan without -x records the file below mnt, and mnt2, which is
|
||||
// beside mnt, not below it.
|
||||
mounted := writeFile(t, dir, "mnt/f", pattern(1, 100))
|
||||
gone := writeFile(t, dir, "mnt2", pattern(3, 50))
|
||||
syncTree(t, db, dir)
|
||||
|
||||
err := os.Remove(gone)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The same size as mnt/f, so it is hashed only if mnt/f's size is
|
||||
// in the census.
|
||||
added := writeFile(t, dir, "added", pattern(2, 100))
|
||||
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
ctx := t.Context()
|
||||
s := &scanState{db: db}
|
||||
|
||||
err = s.loadIndex(ctx, []string{dir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
|
||||
root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true}
|
||||
dispatchDirs(ctx, []dirJob{root}, jobs, subdirs)
|
||||
|
||||
changed, unhashed := s.walkPhase(events)
|
||||
s.partition(changed, unhashed)
|
||||
|
||||
err = s.hashPhase(ctx, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.updatePhase(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
recs := dbRecords(t, db)
|
||||
recordByPath(t, recs, mounted)
|
||||
|
||||
if recordByPath(t, recs, added).head == "" {
|
||||
t.Errorf("%s not hashed: the census lacks the size of %s",
|
||||
added, mounted)
|
||||
}
|
||||
|
||||
if slices.Contains(recordPaths(recs), gone) {
|
||||
t.Errorf("record for %s kept: the file is gone and not below mnt",
|
||||
gone)
|
||||
}
|
||||
}
|
||||
|
||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||
// with deterministic content and returns the tree root.
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
|
||||
Reference in New Issue
Block a user