scan: skip OS default excludes, and record nothing on filesystems that are not persistent (closes #124)

scan never records anything on a filesystem that is not persistent
(tmpfs, proc, sysfs and the other kernel or memory-backed types). As it
starts it notes the device ID of each such mount point, from
/proc/self/mountinfo on Linux and the mounted-filesystem list on macOS,
and checks every operand, directory and file the walk meets against
them; no flag turns this off. Below an operand the walk also skips the
default excludes, a path list per system (Linux, macOS, and Windows,
which is not used yet), and --no-default-excludes turns off only those.
A default exclude that is itself an operand gets no skip line. The tests
stop at once, saying why, when TMPDIR is not on a persistent filesystem.

Judgement call: an unreadable mount list fails the scan, and no flag gets round it.
Judgement call: a file on such a filesystem below an operand gets the same line as a directory.
Judgement call: an operand on such a filesystem that another operand's walk also reaches is named twice.
Unverified: the macOS mount list is built for macOS but never run; bind mounts need root to test.

Model: opus-5-5

scan.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -114,9 +115,12 @@ func scanPhase(name string) string {
 // SIGTERM that interruptContext catches, the scan keeps what it has
 // hashed (see syncScan), prints how many files its walk reached, and
 // returns errInterrupted. workers must be at least 1; the scan command
-// rejects anything less.
+// rejects anything less. Before the lock is taken, the scan lists the
+// filesystems that are not persistent, on which the walk records
+// nothing; unless noDefaultExcludes is set, the walk does not enter the
+// default excludes either.
 func runScan(ctx context.Context, roots []string, workers int,
-	oneFS bool,
+	oneFS, noDefaultExcludes bool,
 ) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -139,6 +143,16 @@ func runScan(ctx context.Context, roots []string, workers int,
 		return err
 	}
 
+	nonPersistent, err := nonPersistentFilesystems()
+	if err != nil {
+		return fmt.Errorf("list the mounted filesystems: %w", err)
+	}
+
+	rules := walkRules{nonPersistent: nonPersistent}
+	if !noDefaultExcludes {
+		rules.excludes = defaultExcludes(runtime.GOOS)
+	}
+
 	lock, err := lockScanDatabase(dbPath)
 	if err != nil {
 		return err
@@ -159,7 +173,7 @@ func runScan(ctx context.Context, roots []string, workers int,
 
 	defer closeScanDatabase(ctx, db, dbPath)
 
-	st, err := syncScan(ctx, db, roots, workers, oneFS)
+	st, err := syncScan(ctx, db, roots, workers, oneFS, rules)
 	if errors.Is(err, context.Canceled) {
 		return interrupted(st.walked)
 	}
@@ -231,14 +245,12 @@ func resolveRoots(roots []string) ([]string, error) {
 }
 
 // pruneRoots drops operands already covered by another operand:
-// duplicates and any operand lying under another one. With -x, an
-// operand under another one is kept when the walk of the deepest kept
-// operand above it cannot reach it: when device puts any directory from
-// it (from its directory, for a regular file) up to but not including
-// that operand on a different filesystem than that operand. Every
-// remaining file is then reachable through exactly one root, so walked
-// paths are unique without keeping a scan-wide set of every path seen.
-func pruneRoots(roots []string, oneFS bool,
+// duplicates and any operand lying under another one. An operand under
+// another one is kept, though, when the walk of the deepest kept operand
+// above it does not reach it (see walkReaches). Every remaining file is
+// then reachable through exactly one root, so walked paths are unique
+// without keeping a scan-wide set of every path seen.
+func pruneRoots(roots []string, oneFS bool, rules walkRules,
 	device func(path string) (uint64, bool),
 ) []string {
 	// Shorter-first ordering guarantees an ancestor is kept before any
@@ -262,40 +274,57 @@ func pruneRoots(roots []string, oneFS bool,
 			continue
 		}
 
-		// Without -x, and always for a duplicate, r is dropped.
-		if !oneFS || above == r {
-			continue
-		}
-
-		aboveDev, aboveOK := device(above)
-		if !aboveOK {
-			continue
-		}
-
-		// The walk of above enters a directory only when it is on
-		// above's filesystem, and then reaches every regular file in it
-		// whatever device the file reports, so for a regular-file
-		// operand the directories checked start at its directory.
-		dir := r
-
-		fi, err := os.Lstat(r)
-		if err == nil && fi.Mode().IsRegular() {
-			dir = filepath.Dir(r)
-		}
-
-		for ; dir != above; dir = filepath.Dir(dir) {
-			dev, ok := device(dir)
-			if ok && dev != aboveDev {
-				kept = append(kept, r)
-
-				break
-			}
+		// A duplicate is dropped, and so is an operand the walk of above
+		// reaches.
+		if above != r && !walkReaches(above, r, oneFS, rules, device) {
+			kept = append(kept, r)
 		}
 	}
 
 	return kept
 }
 
+// walkReaches reports whether the walk of operand above reaches r, an
+// operand under it. That walk does not enter a default exclude in
+// rules, nor a directory that device puts on a filesystem in rules that
+// is not persistent, nor, with -x, one that device puts on a different
+// filesystem than above, when device knows above's. It reaches every
+// regular file in a directory it enters, so for a regular-file operand
+// the directories checked start at its directory; they end below above.
+func walkReaches(above, r string, oneFS bool, rules walkRules,
+	device func(path string) (uint64, bool),
+) bool {
+	aboveDev, aboveDevOK := device(above)
+
+	dir := r
+
+	fi, err := os.Lstat(r)
+	if err == nil && fi.Mode().IsRegular() {
+		dir = filepath.Dir(r)
+	}
+
+	for ; dir != above; dir = filepath.Dir(dir) {
+		if isDefaultExclude(dir, rules.excludes) {
+			return false
+		}
+
+		dev, ok := device(dir)
+		if !ok {
+			continue
+		}
+
+		if _, found := rules.nonPersistent[dev]; found {
+			return false
+		}
+
+		if oneFS && aboveDevOK && dev != aboveDev {
+			return false
+		}
+	}
+
+	return true
+}
+
 // deviceOfPath returns the filesystem device ID of the file at path,
 // when lstat succeeds and the platform exposes one.
 func deviceOfPath(path string) (uint64, bool) {
@@ -337,11 +366,11 @@ type scanState struct {
 // the hashed records still waiting in the batch, starts no other write
 // or deletion, and returns the cancellation.
 func syncScan(ctx context.Context, db *sql.DB, roots []string,
-	workers int, oneFS bool,
+	workers int, oneFS bool, rules walkRules,
 ) (scanStats, error) {
 	s := &scanState{db: db}
 
-	err := s.runPhases(ctx, roots, workers, oneFS)
+	err := s.runPhases(ctx, roots, workers, oneFS, rules)
 	if err == nil || ctx.Err() == nil {
 		return s.st, err
 	}
@@ -365,15 +394,18 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
 // and content (fill in the content hash of every record of headTailMin
 // or more whose size, head, and tail match another record's). Records
 // outside the roots are never touched, except that the content phase
-// fills in their content hash. Operands the walk cannot start from are
-// dropped first, so the records beneath them are treated as if those
-// operands had not been given.
+// fills in their content hash. Operands the walk cannot start from, or
+// that are not on a persistent filesystem, are dropped first, so the
+// records beneath them are treated as if those operands had not been
+// given. Below an operand, the walk follows rules.
 func (s *scanState) runPhases(ctx context.Context, roots []string,
-	workers int, oneFS bool,
+	workers int, oneFS bool, rules walkRules,
 ) error {
-	// Types are checked before pruning so that an operand under a
-	// dropped one is still scanned, not dropped as lying under it.
-	roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath)
+	// Types and filesystems are checked before pruning so that an
+	// operand under a dropped one is still scanned, not dropped as lying
+	// under it.
+	roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
+		deviceOfPath)
 
 	err := s.loadIndex(ctx, roots)
 	if err != nil {
@@ -381,7 +413,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 	}
 
 	changed, unhashed := s.walkPhase(
-		startWalk(ctx, roots, oneFS, workers), roots)
+		startWalk(ctx, roots, oneFS, rules, workers), roots)
 
 	// A cancelled walk stops early, so its size census covers only part
 	// of the roots, and every file it never reached would look vanished
@@ -411,9 +443,11 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 // about, counted as skipped, and dropped. A dropped operand is no
 // longer a root, so the records stored beneath it are treated as if it
 // had not been given: those under no other root count as outside the
-// roots and are not deleted as unverified. An operand that fails lstat
-// here is kept, and the walk warns about it.
-func (s *scanState) walkableRoots(roots []string) []string {
+// roots and are not deleted as unverified. An operand on a filesystem
+// in rules that is not persistent is dropped the same way, with a line
+// saying so that is not a warning, and is not counted as skipped. An
+// operand that fails lstat here is kept, and the walk warns about it.
+func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
 	kept := make([]string, 0, len(roots))
 
 	for _, root := range roots {
@@ -427,6 +461,13 @@ func (s *scanState) walkableRoots(roots []string) []string {
 
 				continue
 			}
+
+			line := rules.nonPersistentLine(root, fi)
+			if line != "" {
+				fmt.Fprintln(os.Stderr, escapePath(line))
+
+				continue
+			}
 		}
 
 		kept = append(kept, root)
@@ -481,10 +522,12 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 // (the content phase may still fill in its content hash). It returns
 // the new-or-changed files and the unchanged files whose records lack
 // hashes; both remain candidates until the census decides whether
-// their sizes are shared. A record below a directory -x did not enter
-// counts as outside the roots: it is not deleted, and its size joins
-// the census. That does not hold below a root at or inside that
-// directory, since that root's own walk verifies the records below it.
+// their sizes are shared. A record at or below a path the walk left
+// out, because -x stopped at it, it is a default exclude, or it is not
+// on a persistent filesystem, counts as outside the roots: it is not
+// deleted, and its size joins the census. That does not hold below a
+// root at or inside that path, since that root's own walk verifies the
+// records below it; nor is a line printed for such a root.
 func (s *scanState) walkPhase(
 	events <-chan walkEvent, roots []string,
 ) ([]fileRec, []fileRec) {
@@ -502,12 +545,25 @@ func (s *scanState) walkPhase(
 
 	var (
 		changed, unhashed []fileRec
-		otherFSDirs       []string
+		leftOut           []string
 	)
 
 	for ev := range events {
 		if ev.otherFSDir != "" {
-			otherFSDirs = append(otherFSDirs, ev.otherFSDir)
+			leftOut = append(leftOut, ev.otherFSDir)
+
+			continue
+		}
+
+		// The line saying so is printed as a warning is, but what the
+		// walk leaves out is not counted as skipped. A default exclude
+		// that is itself a root is walked as one, so it gets no line.
+		if ev.leftOut != "" {
+			leftOut = append(leftOut, ev.leftOut)
+
+			if !slices.Contains(roots, ev.leftOut) {
+				prog.warnf("%s", ev.warn)
+			}
 
 			continue
 		}
@@ -542,12 +598,13 @@ func (s *scanState) walkPhase(
 	prog.finish()
 
 	for path, old := range s.existing {
-		// Kept only when a directory -x did not enter lies below the
-		// deepest root above the record. A root at or below that
-		// directory was walked on its own and did not find the file.
-		// Both contain the record, so the longer path is the deeper.
-		otherFSDir := deepestRoot(path, otherFSDirs)
-		if len(otherFSDir) > len(deepestRoot(path, roots)) {
+		// Kept only when a path the walk left out lies below the deepest
+		// root above the record, or is the record's own. A root at or
+		// below that path was walked on its own and did not find the
+		// file. Both contain the record, so the longer path is the
+		// deeper.
+		left := deepestRoot(path, leftOut)
+		if len(left) > len(deepestRoot(path, roots)) {
 			delete(s.existing, path)
 
 			s.sizes = append(s.sizes, old.size)
@@ -1046,27 +1103,31 @@ type dirJob struct {
 }
 
 // walkEvent is one walk result delivered to the walk phase: a regular
-// file's statted record, a warning when fail is set, or, when
-// otherFSDir is set, a directory -x did not enter because it is on
-// another filesystem than its operand.
+// file's statted record, a warning when fail is set, or a path the walk
+// left out: when otherFSDir is set, a directory -x did not enter
+// because it is on another filesystem than its operand, and when
+// leftOut is set, a default exclude or a directory or file on a
+// filesystem that is not persistent, with warn holding the line that
+// says so.
 type walkEvent struct {
 	rec        fileRec
 	warn       string
 	fail       bool
 	otherFSDir string
+	leftOut    string
 }
 
 // startWalk seeds every root into the shared walk worker pool and
 // returns the event stream: one record per regular file, one warning
-// event per per-path error, and one event per directory -x did not
-// enter. The channel is closed when the walk completes, and also when
-// ctx is cancelled — every goroutine in the pool abandons its blocking
-// send in that case, so the consumer sees a truncated but properly
-// terminated stream instead of a stalled one.
+// event per per-path error, and one event per path -x or rules kept the
+// walk from entering or recording. The channel is closed when the walk
+// completes, and also when ctx is cancelled — every goroutine in the
+// pool abandons its blocking send in that case, so the consumer sees a
+// truncated but properly terminated stream instead of a stalled one.
 func startWalk(ctx context.Context, roots []string, oneFS bool,
-	workers int,
+	rules walkRules, workers int,
 ) <-chan walkEvent {
-	jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS)
+	jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, rules)
 
 	go func() {
 		initial := make([]dirJob, 0, len(roots))
@@ -1177,7 +1238,7 @@ func seedRoot(ctx context.Context, root string,
 // workers drop the directories still queued rather than stop reading
 // jobs, so the range always runs out and the pool always tears down.
 func startWalkWorkers(ctx context.Context, workers int,
-	oneFS bool,
+	oneFS bool, rules walkRules,
 ) (chan dirJob, chan []dirJob, chan walkEvent) {
 	jobs := make(chan dirJob, workQueueDepth)
 	subdirs := make(chan []dirJob, workers)
@@ -1193,7 +1254,7 @@ func startWalkWorkers(ctx context.Context, workers int,
 				}
 
 				select {
-				case subdirs <- walkOneDir(ctx, job, oneFS, events):
+				case subdirs <- walkOneDir(ctx, job, oneFS, rules, events):
 				case <-ctx.Done():
 				}
 			}
@@ -1251,7 +1312,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
 // entry and a warning event per unreadable one, and returns the
 // subdirectories to descend into.
 func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
-	events chan<- walkEvent,
+	rules walkRules, events chan<- walkEvent,
 ) []dirJob {
 	entries, err := os.ReadDir(job.path)
 	if err != nil {
@@ -1275,7 +1336,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 		p := filepath.Join(job.path, e.Name())
 
 		if e.IsDir() {
-			if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok {
+			sub, ok := subdirJob(ctx, p, e, job, oneFS, rules, events)
+			if ok {
 				subs = append(subs, sub)
 			}
 
@@ -1287,7 +1349,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 			continue
 		}
 
-		emitFile(ctx, p, e, events)
+		emitFile(ctx, p, e, rules, events)
 	}
 
 	return subs
@@ -1297,9 +1359,11 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 // The lstat happens here in the walk worker, while the directory's
 // metadata is still hot; a path that fails to stat (or stops being a
 // regular file) between the directory read and the lstat is warned
-// about and skipped.
+// about and skipped. A file on a filesystem in rules that is not
+// persistent, as through a bind mount of one file, is not recorded: it
+// gets an event naming it instead.
 func emitFile(ctx context.Context, p string, e fs.DirEntry,
-	events chan<- walkEvent,
+	rules walkRules, events chan<- walkEvent,
 ) {
 	info, err := e.Info()
 	if err != nil {
@@ -1315,6 +1379,13 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 		return
 	}
 
+	line := rules.nonPersistentLine(p, info)
+	if line != "" {
+		sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
+
+		return
+	}
+
 	dev, ino := inodeOfInfo(info)
 
 	sendEvent(ctx, events, walkEvent{rec: fileRec{
@@ -1328,22 +1399,20 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 
 // subdirJob applies the descent rules to directory p: never enter
 // .zfs (ZFS snapshot pseudo-dirs would list every file once per
-// snapshot), and with -x never enter a directory on a different
-// filesystem than its operand, sending an event that names it so the
-// scan keeps the records below it, except those below an operand at or
+// snapshot), never enter a directory on a filesystem in rules that is
+// not persistent, nor a default exclude in rules, and with -x never
+// enter a directory on a different filesystem than its operand. For the
+// last three it sends an event that names the directory, so the scan
+// keeps the records below it, except those below an operand at or
 // inside it.
 func subdirJob(ctx context.Context, p string, e fs.DirEntry,
-	parent dirJob, oneFS bool, events chan<- walkEvent,
+	parent dirJob, oneFS bool, rules walkRules,
+	events chan<- walkEvent,
 ) (dirJob, bool) {
 	if e.Name() == ".zfs" {
 		return dirJob{}, false
 	}
 
-	job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
-	if !oneFS || !parent.rootDevOK {
-		return job, true
-	}
-
 	info, err := e.Info()
 	if err != nil {
 		sendEvent(ctx, events, walkEvent{
@@ -1354,6 +1423,27 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
 		return dirJob{}, false
 	}
 
+	line := rules.nonPersistentLine(p, info)
+	if line != "" {
+		sendEvent(ctx, events, walkEvent{leftOut: p, warn: line})
+
+		return dirJob{}, false
+	}
+
+	if isDefaultExclude(p, rules.excludes) {
+		sendEvent(ctx, events, walkEvent{
+			leftOut: p,
+			warn:    fmt.Sprintf("scan: skipped %s (default exclude)", p),
+		})
+
+		return dirJob{}, false
+	}
+
+	job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
+	if !oneFS || !parent.rootDevOK {
+		return job, true
+	}
+
 	if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev {
 		sendEvent(ctx, events, walkEvent{otherFSDir: p})