Walk an operand nested on another filesystem with -x (closes #106)

With -x, an operand under another operand is kept and walked on its
own when the walk of the deepest kept operand above it cannot reach
it: some directory from it (from its directory, for a regular file) up
to but not including that operand is on a different filesystem than
that operand. Below a kept operand, the rule keeping records under a
directory -x did not enter covers only directories its own walk did
not enter, so records of files gone from it are deleted.

The new tests stub the devices and share their scan steps with the
test for #99. Comments and
README sentences on operands dropped before the walk now also hold
with -x.

Model: opus-5-5

scan_test.go

@@ -1135,14 +1135,72 @@ func TestSubdirJobStatError(t *testing.T) {
 	}
 }
 
+// syncOneFilesystem scans roots into db with -x and returns the scan
+// stats. Every device pruneRoots looks up, and the device each
+// operand's walk compares its subdirectories with, comes from devices
+// instead of from lstat, so a test can put a path on a filesystem that
+// no directory here is on. Unlike runPhases, it skips walkableRoots,
+// the check for a cancelled walk, and the content phase.
+func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string,
+	devices map[string]uint64,
+) scanStats {
+	t.Helper()
+
+	device := func(path string) (uint64, bool) {
+		dev, ok := devices[path]
+
+		return dev, ok
+	}
+
+	roots = pruneRoots(roots, true, device)
+
+	ctx := t.Context()
+	s := &scanState{db: db}
+
+	err := s.loadIndex(ctx, roots)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
+
+	// seedRoot emits a regular-file operand itself and returns a job
+	// only for a directory operand.
+	var initial []dirJob
+
+	for _, root := range roots {
+		for _, job := range seedRoot(ctx, root, events) {
+			job.rootDev = devices[root]
+			initial = append(initial, job)
+		}
+	}
+
+	dispatchDirs(ctx, initial, jobs, subdirs)
+
+	changed, unhashed := s.walkPhase(events, roots)
+	s.partition(changed, unhashed)
+
+	err = s.hashPhase(ctx, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = s.updatePhase(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return s.st
+}
+
 // TestScanOneFilesystemKeepsOtherFilesystemRecords checks that with -x a
 // record below a directory the walk does not enter, because it is on
 // another filesystem than its operand, counts as outside the operands:
 // it is not deleted, and its size joins the size census. The record of
 // a gone file that is not below that directory is still deleted, even
-// when its path begins with the directory's name. The walk starts from
-// an operand device that no directory here is on, so no second real
-// filesystem is needed; the phases after it run as runPhases runs them.
+// when its path begins with the directory's name. The operand is given
+// a device that no directory here is on, so no second real filesystem
+// is needed.
 func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
 	t.Parallel()
 
@@ -1174,30 +1232,7 @@ func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
 		t.Skip("platform exposes no device id")
 	}
 
-	ctx := t.Context()
-	s := &scanState{db: db}
-
-	err = s.loadIndex(ctx, []string{dir})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	jobs, subdirs, events := startWalkWorkers(ctx, 2, true)
-	root := dirJob{path: dir, rootDev: dev + 1, rootDevOK: true}
-	dispatchDirs(ctx, []dirJob{root}, jobs, subdirs)
-
-	changed, unhashed := s.walkPhase(events)
-	s.partition(changed, unhashed)
-
-	err = s.hashPhase(ctx, 2)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = s.updatePhase(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
+	syncOneFilesystem(t, db, []string{dir}, map[string]uint64{dir: dev + 1})
 
 	recs := dbRecords(t, db)
 	recordByPath(t, recs, mounted)
@@ -1213,6 +1248,183 @@ func TestScanOneFilesystemKeepsOtherFilesystemRecords(t *testing.T) {
 	}
 }
 
+// TestScanOneFilesystemWalksNestedOperand checks that with -x an
+// operand under another one, on a different filesystem than it, is
+// walked on its own: a file added below it is recorded, and the record
+// of a file gone from below it is deleted. A record below a directory
+// that its own walk does not enter is still kept. Each operand is given
+// a device that no directory here is on, so neither walk enters a
+// subdirectory.
+func TestScanOneFilesystemWalksNestedOperand(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	// A scan without -x records both files below mnt.
+	gone := writeFile(t, dir, "mnt/gone", pattern(1, 10))
+	kept := writeFile(t, dir, "mnt/sub/kept", pattern(2, 20))
+	syncTree(t, db, dir)
+
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	added := writeFile(t, dir, "mnt/added", pattern(3, 30))
+
+	info, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev, ok := deviceOfInfo(info)
+	if !ok {
+		t.Skip("platform exposes no device id")
+	}
+
+	mnt := filepath.Join(dir, "mnt")
+	syncOneFilesystem(t, db, []string{dir, mnt},
+		map[string]uint64{dir: dev + 1, mnt: dev + 2})
+
+	paths := recordPaths(dbRecords(t, db))
+
+	if !slices.Contains(paths, added) {
+		t.Errorf("no record for %s: the walk did not reach %s", added, mnt)
+	}
+
+	if slices.Contains(paths, gone) {
+		t.Errorf("record for %s kept: the file is gone", gone)
+	}
+
+	if !slices.Contains(paths, kept) {
+		t.Errorf("record for %s deleted: the walk of %s did not enter %s",
+			kept, mnt, filepath.Dir(kept))
+	}
+}
+
+// TestScanOneFilesystemWalksOperandBelowOtherFilesystem checks that
+// with -x an operand on the same filesystem as the operand covering it
+// is walked on its own when a directory between them is on another
+// filesystem, since the walk of the covering operand does not enter
+// that directory: a file added below the nested operand is recorded,
+// and the record of a file gone from below it is deleted. A record
+// below that directory but outside the nested operand is still kept.
+// Both operands are given one device that no directory here is on,
+// while mnt keeps its own.
+func TestScanOneFilesystemWalksOperandBelowOtherFilesystem(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+
+	// A scan without -x records both files below mnt.
+	gone := writeFile(t, dir, "mnt/back/gone", pattern(1, 10))
+	kept := writeFile(t, dir, "mnt/kept", pattern(2, 20))
+	syncTree(t, db, dir)
+
+	err := os.Remove(gone)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	added := writeFile(t, dir, "mnt/back/added", pattern(3, 30))
+
+	info, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev, ok := deviceOfInfo(info)
+	if !ok {
+		t.Skip("platform exposes no device id")
+	}
+
+	mnt := filepath.Join(dir, "mnt")
+	back := filepath.Join(mnt, "back")
+	syncOneFilesystem(t, db, []string{dir, back},
+		map[string]uint64{dir: dev + 1, mnt: dev, back: dev + 1})
+
+	paths := recordPaths(dbRecords(t, db))
+
+	if !slices.Contains(paths, added) {
+		t.Errorf("no record for %s: the walk did not reach %s", added, back)
+	}
+
+	if slices.Contains(paths, gone) {
+		t.Errorf("record for %s kept: the file is gone", gone)
+	}
+
+	if !slices.Contains(paths, kept) {
+		t.Errorf("record for %s deleted: the walk of %s did not enter %s",
+			kept, dir, mnt)
+	}
+}
+
+// TestScanOneFilesystemRecordsNestedFileOnce checks that with -x a
+// regular-file operand under another operand is dropped when the file
+// reports a different device than that operand but its directory does
+// not, as a file bind-mounted into the tree does: the walk of the other
+// operand enters the directory and reaches the file, so the file is
+// recorded once.
+func TestScanOneFilesystemRecordsNestedFileOnce(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	file := writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	info, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev, ok := deviceOfInfo(info)
+	if !ok {
+		t.Skip("platform exposes no device id")
+	}
+
+	st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
+		dir: dev, filepath.Dir(file): dev, file: dev + 1,
+	})
+
+	if st.added != 1 {
+		t.Errorf("%s recorded %d times, want once", file, st.added)
+	}
+}
+
+// TestScanOneFilesystemWalksNestedFileOperand checks that with -x a
+// regular-file operand under another operand is kept when its directory
+// is on a different filesystem than that operand: the walk of the other
+// operand does not enter the directory, so the file is recorded once,
+// as its own operand. The other operand is given a device that no
+// directory here is on.
+func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	file := writeFile(t, dir, "sub/f", pattern(1, 10))
+
+	info, err := os.Lstat(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dev, ok := deviceOfInfo(info)
+	if !ok {
+		t.Skip("platform exposes no device id")
+	}
+
+	st := syncOneFilesystem(t, db, []string{dir, file}, map[string]uint64{
+		dir: dev + 1, filepath.Dir(file): dev,
+	})
+
+	if st.added != 1 {
+		t.Errorf("%s recorded %d times, want once", file, st.added)
+	}
+}
+
 // buildSmokeTree recreates the README smoke-test filesystem layout
 // with deterministic content and returns the tree root.
 func buildSmokeTree(t *testing.T) string {
@@ -2123,14 +2335,43 @@ func TestHashRuns(t *testing.T) {
 func TestPruneRoots(t *testing.T) {
 	t.Parallel()
 
+	const pqRoot = "/p/q"
+
+	// /p/q is on another filesystem than /p; /p/q/r is on the same one as
+	// /p/q, /p/s as /p, and /p/t has no known device.
+	devices := map[string]uint64{"/p": 1, pqRoot: 2, "/p/q/r": 2, "/p/s": 1}
+	device := func(path string) (uint64, bool) {
+		dev, ok := devices[path]
+
+		return dev, ok
+	}
+
 	// Duplicates and operands under other operands are dropped; /cc is
 	// not under /c (sibling with a shared prefix).
-	got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"})
+	got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"},
+		false, device)
 
 	want := []string{"/a", "/c", "/cc"}
 	if !slices.Equal(got, want) {
 		t.Fatalf("pruneRoots = %q, want %q", got, want)
 	}
+
+	nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"}
+
+	got = pruneRoots(nested, false, device)
+
+	want = []string{"/p"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("pruneRoots without -x = %q, want %q", got, want)
+	}
+
+	// With -x, /p/q, on another filesystem than /p, is kept as well.
+	got = pruneRoots(nested, true, device)
+
+	want = []string{"/p", pqRoot}
+	if !slices.Equal(got, want) {
+		t.Fatalf("pruneRoots with -x = %q, want %q", got, want)
+	}
 }
 
 func TestReportsNeverTouchFilesystem(t *testing.T) {