Walk an operand nested on another filesystem with -x (closes #106)
check / check (push) Waiting to run
check / check (push) Waiting to run
With -x, an operand under another operand is kept and walked on its own when it is a directory on a different filesystem than the deepest kept operand above it, or a regular file in such a directory, since that operand's walk does not reach it. Below a kept operand, the rule keeping records under a directory -x did not enter covers only directories its own walk did not enter, so records of files gone from it are deleted. The new tests stub the devices and share their scan steps with the test for #99, whose comment no longer claims it runs every phase. The startWalk comment is rewrapped. Model: opus-5-5
This commit is contained in:
@@ -203,10 +203,16 @@ func resolveRoots(roots []string) ([]string, error) {
|
||||
}
|
||||
|
||||
// pruneRoots drops operands already covered by another operand:
|
||||
// duplicates and any operand lying under another one. Every remaining
|
||||
// file is then reachable through exactly one root, so walked paths are
|
||||
// unique without keeping a scan-wide set of every path seen.
|
||||
func pruneRoots(roots []string) []string {
|
||||
// duplicates and any operand lying under another one. With -x, an
|
||||
// operand under another one is kept when device puts it, or for a
|
||||
// regular file its directory, on a different filesystem than the
|
||||
// deepest kept operand above it, because that operand's walk then does
|
||||
// not reach it. Every remaining file is then reachable through exactly
|
||||
// one root, so walked paths are unique without keeping a scan-wide set
|
||||
// of every path seen.
|
||||
func pruneRoots(roots []string, oneFS bool,
|
||||
device func(path string) (uint64, bool),
|
||||
) []string {
|
||||
// Shorter-first ordering guarantees an ancestor is kept before any
|
||||
// operand under it is considered.
|
||||
sorted := slices.Clone(roots)
|
||||
@@ -221,7 +227,33 @@ func pruneRoots(roots []string) []string {
|
||||
kept := make([]string, 0, len(sorted))
|
||||
|
||||
for _, r := range sorted {
|
||||
if !underAnyRoot(r, kept) {
|
||||
above := deepestRoot(r, kept)
|
||||
if above == "" {
|
||||
kept = append(kept, r)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Without -x, and always for a duplicate, r is dropped.
|
||||
if !oneFS || above == r {
|
||||
continue
|
||||
}
|
||||
|
||||
// The walk of above enters a directory only when it is on
|
||||
// above's filesystem, and then reaches every regular file in it
|
||||
// whatever device the file reports, so for a regular-file
|
||||
// operand its directory decides.
|
||||
dir := r
|
||||
|
||||
fi, err := os.Lstat(r)
|
||||
if err == nil && fi.Mode().IsRegular() {
|
||||
dir = filepath.Dir(r)
|
||||
}
|
||||
|
||||
dev, ok := device(dir)
|
||||
aboveDev, aboveOK := device(above)
|
||||
|
||||
if ok && aboveOK && dev != aboveDev {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
@@ -229,6 +261,17 @@ func pruneRoots(roots []string) []string {
|
||||
return kept
|
||||
}
|
||||
|
||||
// deviceOfPath returns the filesystem device ID of the file at path,
|
||||
// when lstat succeeds and the platform exposes one.
|
||||
func deviceOfPath(path string) (uint64, bool) {
|
||||
fi, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return deviceOfInfo(fi)
|
||||
}
|
||||
|
||||
// scanStats summarizes one scan's database synchronization for the
|
||||
// final stderr summary, or for the line an interrupted scan prints.
|
||||
type scanStats struct {
|
||||
@@ -295,14 +338,15 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||
) error {
|
||||
// Types are checked before pruning so that an operand under a
|
||||
// dropped one is still scanned, not dropped as lying under it.
|
||||
roots = pruneRoots(s.walkableRoots(roots))
|
||||
roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath)
|
||||
|
||||
err := s.loadIndex(ctx, roots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
changed, unhashed := s.walkPhase(startWalk(ctx, roots, oneFS, workers))
|
||||
changed, unhashed := s.walkPhase(
|
||||
startWalk(ctx, roots, oneFS, workers), roots)
|
||||
|
||||
// A cancelled walk stops early, so its size census covers only part
|
||||
// of the roots, and every file it never reached would look vanished
|
||||
@@ -393,9 +437,10 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||
// hashes; both remain candidates until the census decides whether
|
||||
// their sizes are shared. A record below a directory -x did not enter
|
||||
// counts as outside the roots: it is not deleted, and its size joins
|
||||
// the census.
|
||||
// the census. That does not hold below a root at or inside that
|
||||
// directory, since that root's own walk verifies the records below it.
|
||||
func (s *scanState) walkPhase(
|
||||
events <-chan walkEvent,
|
||||
events <-chan walkEvent, roots []string,
|
||||
) ([]fileRec, []fileRec) {
|
||||
prog := newProgress("walk", -1)
|
||||
|
||||
@@ -441,7 +486,12 @@ func (s *scanState) walkPhase(
|
||||
prog.finish()
|
||||
|
||||
for path, old := range s.existing {
|
||||
if underAnyRoot(path, otherFSDirs) {
|
||||
// Kept only when a directory -x did not enter lies below the
|
||||
// deepest root above the record. A root at or below that
|
||||
// directory was walked on its own and did not find the file.
|
||||
// Both contain the record, so the longer path is the deeper.
|
||||
otherFSDir := deepestRoot(path, otherFSDirs)
|
||||
if len(otherFSDir) > len(deepestRoot(path, roots)) {
|
||||
delete(s.existing, path)
|
||||
|
||||
s.sizes = append(s.sizes, old.size)
|
||||
@@ -875,6 +925,20 @@ func underRoot(path, root string) bool {
|
||||
return strings.HasPrefix(path, prefix)
|
||||
}
|
||||
|
||||
// deepestRoot returns the longest of the roots that path is or lies
|
||||
// under, or "" when it lies under none.
|
||||
func deepestRoot(path string, roots []string) string {
|
||||
deepest := ""
|
||||
|
||||
for _, root := range roots {
|
||||
if underRoot(path, root) && len(root) > len(deepest) {
|
||||
deepest = root
|
||||
}
|
||||
}
|
||||
|
||||
return deepest
|
||||
}
|
||||
|
||||
// dirJob is one directory awaiting traversal by the walk workers. It
|
||||
// carries its operand's filesystem device so -x can stop at
|
||||
// filesystem boundaries.
|
||||
@@ -898,10 +962,10 @@ type walkEvent struct {
|
||||
// startWalk seeds every root into the shared walk worker pool and
|
||||
// returns the event stream: one record per regular file, one warning
|
||||
// event per per-path error, and one event per directory -x did not
|
||||
// enter. The channel is closed when the walk
|
||||
// completes, and also when ctx is cancelled — every goroutine in the
|
||||
// pool abandons its blocking send in that case, so the consumer sees a
|
||||
// truncated but properly terminated stream instead of a stalled one.
|
||||
// enter. The channel is closed when the walk completes, and also when
|
||||
// ctx is cancelled — every goroutine in the pool abandons its blocking
|
||||
// send in that case, so the consumer sees a truncated but properly
|
||||
// terminated stream instead of a stalled one.
|
||||
func startWalk(ctx context.Context, roots []string, oneFS bool,
|
||||
workers int,
|
||||
) <-chan walkEvent {
|
||||
|
||||
Reference in New Issue
Block a user