scan: skip OS-appropriate default excludes (/proc, /sys, /tmp, /run, /var/tmp on Linux) (closes #124)

Below an operand, the walk no longer enters the default excludes: on
Linux /proc, /sys, /dev, /run, /tmp, /var/tmp and every mount point of a
kernel or memory-backed filesystem, read once from /proc/self/mountinfo;
on macOS /dev, /private/tmp, /private/var/tmp, /private/var/vm,
/System/Volumes/VM and /System/Volumes/Data. The lists live in the new
exclude.go. An operand at or inside one is still walked, also under
another operand, as with -x. Each skipped directory gets one stderr line
through the warning path, is not counted as skipped, and its records
count as outside the scanned operands. --no-default-excludes turns them
off; scan --help lists them.

Judgement call: an unreadable /proc/self/mountinfo fails the scan.
Judgement call: where filesystems are mounted over one another, the last decides.

Model: opus-5-5

scan.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -114,9 +115,10 @@ func scanPhase(name string) string {
 // SIGTERM that interruptContext catches, the scan keeps what it has
 // hashed (see syncScan), prints how many files its walk reached, and
 // returns errInterrupted. workers must be at least 1; the scan command
-// rejects anything less.
+// rejects anything less. Unless noDefaultExcludes is set, the walk does
+// not enter the default excludes, found before the lock is taken.
 func runScan(ctx context.Context, roots []string, workers int,
-	oneFS bool,
+	oneFS, noDefaultExcludes bool,
 ) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -139,6 +141,16 @@ func runScan(ctx context.Context, roots []string, workers int,
 		return err
 	}
 
+	// With no excludes the walk enters every directory.
+	var excludes map[string]string
+
+	if !noDefaultExcludes {
+		excludes, err = findDefaultExcludes(runtime.GOOS)
+		if err != nil {
+			return fmt.Errorf("find the default excludes: %w", err)
+		}
+	}
+
 	lock, err := lockScanDatabase(dbPath)
 	if err != nil {
 		return err
@@ -159,7 +171,7 @@ func runScan(ctx context.Context, roots []string, workers int,
 
 	defer closeScanDatabase(ctx, db, dbPath)
 
-	st, err := syncScan(ctx, db, roots, workers, oneFS)
+	st, err := syncScan(ctx, db, roots, workers, oneFS, excludes)
 	if errors.Is(err, context.Canceled) {
 		return interrupted(st.walked)
 	}
@@ -231,14 +243,12 @@ func resolveRoots(roots []string) ([]string, error) {
 }
 
 // pruneRoots drops operands already covered by another operand:
-// duplicates and any operand lying under another one. With -x, an
-// operand under another one is kept when the walk of the deepest kept
-// operand above it cannot reach it: when device puts any directory from
-// it (from its directory, for a regular file) up to but not including
-// that operand on a different filesystem than that operand. Every
-// remaining file is then reachable through exactly one root, so walked
-// paths are unique without keeping a scan-wide set of every path seen.
-func pruneRoots(roots []string, oneFS bool,
+// duplicates and any operand lying under another one. An operand under
+// another one is kept, though, when the walk of the deepest kept operand
+// above it does not reach it (see walkReaches). Every remaining file is
+// then reachable through exactly one root, so walked paths are unique
+// without keeping a scan-wide set of every path seen.
+func pruneRoots(roots []string, oneFS bool, excludes map[string]string,
 	device func(path string) (uint64, bool),
 ) []string {
 	// Shorter-first ordering guarantees an ancestor is kept before any
@@ -262,40 +272,59 @@ func pruneRoots(roots []string, oneFS bool,
 			continue
 		}
 
-		// Without -x, and always for a duplicate, r is dropped.
-		if !oneFS || above == r {
-			continue
-		}
-
-		aboveDev, aboveOK := device(above)
-		if !aboveOK {
-			continue
-		}
-
-		// The walk of above enters a directory only when it is on
-		// above's filesystem, and then reaches every regular file in it
-		// whatever device the file reports, so for a regular-file
-		// operand the directories checked start at its directory.
-		dir := r
-
-		fi, err := os.Lstat(r)
-		if err == nil && fi.Mode().IsRegular() {
-			dir = filepath.Dir(r)
-		}
-
-		for ; dir != above; dir = filepath.Dir(dir) {
-			dev, ok := device(dir)
-			if ok && dev != aboveDev {
-				kept = append(kept, r)
-
-				break
-			}
+		// A duplicate is dropped, and so is an operand the walk of above
+		// reaches.
+		if above != r && !walkReaches(above, r, oneFS, excludes, device) {
+			kept = append(kept, r)
 		}
 	}
 
 	return kept
 }
 
+// walkReaches reports whether the walk of operand above reaches r, an
+// operand under it. That walk does not enter a directory in excludes,
+// nor, with -x, one that device puts on a different filesystem than
+// above, when device knows above's. It reaches every regular file in a
+// directory it enters, whatever device the file reports, so for a
+// regular-file operand the directories checked start at its directory;
+// they end below above.
+func walkReaches(above, r string, oneFS bool, excludes map[string]string,
+	device func(path string) (uint64, bool),
+) bool {
+	var (
+		aboveDev    uint64
+		checkDevice bool
+	)
+
+	if oneFS {
+		aboveDev, checkDevice = device(above)
+	}
+
+	dir := r
+
+	fi, err := os.Lstat(r)
+	if err == nil && fi.Mode().IsRegular() {
+		dir = filepath.Dir(r)
+	}
+
+	for ; dir != above; dir = filepath.Dir(dir) {
+		if _, ok := excludes[dir]; ok {
+			return false
+		}
+
+		if !checkDevice {
+			continue
+		}
+
+		if dev, ok := device(dir); ok && dev != aboveDev {
+			return false
+		}
+	}
+
+	return true
+}
+
 // deviceOfPath returns the filesystem device ID of the file at path,
 // when lstat succeeds and the platform exposes one.
 func deviceOfPath(path string) (uint64, bool) {
@@ -337,11 +366,11 @@ type scanState struct {
 // the hashed records still waiting in the batch, starts no other write
 // or deletion, and returns the cancellation.
 func syncScan(ctx context.Context, db *sql.DB, roots []string,
-	workers int, oneFS bool,
+	workers int, oneFS bool, excludes map[string]string,
 ) (scanStats, error) {
 	s := &scanState{db: db}
 
-	err := s.runPhases(ctx, roots, workers, oneFS)
+	err := s.runPhases(ctx, roots, workers, oneFS, excludes)
 	if err == nil || ctx.Err() == nil {
 		return s.st, err
 	}
@@ -367,13 +396,15 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
 // outside the roots are never touched, except that the content phase
 // fills in their content hash. Operands the walk cannot start from are
 // dropped first, so the records beneath them are treated as if those
-// operands had not been given.
+// operands had not been given. Below an operand, the walk does not
+// enter the directories in excludes.
 func (s *scanState) runPhases(ctx context.Context, roots []string,
-	workers int, oneFS bool,
+	workers int, oneFS bool, excludes map[string]string,
 ) error {
 	// Types are checked before pruning so that an operand under a
 	// dropped one is still scanned, not dropped as lying under it.
-	roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath)
+	roots = pruneRoots(s.walkableRoots(roots), oneFS, excludes,
+		deviceOfPath)
 
 	err := s.loadIndex(ctx, roots)
 	if err != nil {
@@ -381,7 +412,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 	}
 
 	changed, unhashed := s.walkPhase(
-		startWalk(ctx, roots, oneFS, workers), roots)
+		startWalk(ctx, roots, oneFS, excludes, workers), roots)
 
 	// A cancelled walk stops early, so its size census covers only part
 	// of the roots, and every file it never reached would look vanished
@@ -481,10 +512,11 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 // (the content phase may still fill in its content hash). It returns
 // the new-or-changed files and the unchanged files whose records lack
 // hashes; both remain candidates until the census decides whether
-// their sizes are shared. A record below a directory -x did not enter
-// counts as outside the roots: it is not deleted, and its size joins
-// the census. That does not hold below a root at or inside that
-// directory, since that root's own walk verifies the records below it.
+// their sizes are shared. A record below a directory the walk did not
+// enter, because -x stopped at it or it is a default exclude, counts as
+// outside the roots: it is not deleted, and its size joins the census.
+// That does not hold below a root at or inside that directory, since
+// that root's own walk verifies the records below it.
 func (s *scanState) walkPhase(
 	events <-chan walkEvent, roots []string,
 ) ([]fileRec, []fileRec) {
@@ -502,12 +534,22 @@ func (s *scanState) walkPhase(
 
 	var (
 		changed, unhashed []fileRec
-		otherFSDirs       []string
+		notEntered        []string
 	)
 
 	for ev := range events {
 		if ev.otherFSDir != "" {
-			otherFSDirs = append(otherFSDirs, ev.otherFSDir)
+			notEntered = append(notEntered, ev.otherFSDir)
+
+			continue
+		}
+
+		// The line saying so is printed as a warning is, but a default
+		// exclude is not counted as skipped.
+		if ev.excludedDir != "" {
+			notEntered = append(notEntered, ev.excludedDir)
+
+			prog.warnf("%s", ev.warn)
 
 			continue
 		}
@@ -542,12 +584,12 @@ func (s *scanState) walkPhase(
 	prog.finish()
 
 	for path, old := range s.existing {
-		// Kept only when a directory -x did not enter lies below the
-		// deepest root above the record. A root at or below that
+		// Kept only when a directory the walk did not enter lies below
+		// the deepest root above the record. A root at or below that
 		// directory was walked on its own and did not find the file.
 		// Both contain the record, so the longer path is the deeper.
-		otherFSDir := deepestRoot(path, otherFSDirs)
-		if len(otherFSDir) > len(deepestRoot(path, roots)) {
+		dir := deepestRoot(path, notEntered)
+		if len(dir) > len(deepestRoot(path, roots)) {
 			delete(s.existing, path)
 
 			s.sizes = append(s.sizes, old.size)
@@ -1046,27 +1088,30 @@ type dirJob struct {
 }
 
 // walkEvent is one walk result delivered to the walk phase: a regular
-// file's statted record, a warning when fail is set, or, when
-// otherFSDir is set, a directory -x did not enter because it is on
-// another filesystem than its operand.
+// file's statted record, a warning when fail is set, or a directory the
+// walk did not enter: when otherFSDir is set, one -x did not enter
+// because it is on another filesystem than its operand, and when
+// excludedDir is set, a default exclude, with warn holding the line
+// that says so.
 type walkEvent struct {
-	rec        fileRec
-	warn       string
-	fail       bool
-	otherFSDir string
+	rec         fileRec
+	warn        string
+	fail        bool
+	otherFSDir  string
+	excludedDir string
 }
 
 // startWalk seeds every root into the shared walk worker pool and
 // returns the event stream: one record per regular file, one warning
-// event per per-path error, and one event per directory -x did not
-// enter. The channel is closed when the walk completes, and also when
-// ctx is cancelled — every goroutine in the pool abandons its blocking
-// send in that case, so the consumer sees a truncated but properly
-// terminated stream instead of a stalled one.
+// event per per-path error, and one event per directory -x or excludes
+// kept the walk from entering. The channel is closed when the walk
+// completes, and also when ctx is cancelled — every goroutine in the
+// pool abandons its blocking send in that case, so the consumer sees a
+// truncated but properly terminated stream instead of a stalled one.
 func startWalk(ctx context.Context, roots []string, oneFS bool,
-	workers int,
+	excludes map[string]string, workers int,
 ) <-chan walkEvent {
-	jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS)
+	jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, excludes)
 
 	go func() {
 		initial := make([]dirJob, 0, len(roots))
@@ -1177,7 +1222,7 @@ func seedRoot(ctx context.Context, root string,
 // workers drop the directories still queued rather than stop reading
 // jobs, so the range always runs out and the pool always tears down.
 func startWalkWorkers(ctx context.Context, workers int,
-	oneFS bool,
+	oneFS bool, excludes map[string]string,
 ) (chan dirJob, chan []dirJob, chan walkEvent) {
 	jobs := make(chan dirJob, workQueueDepth)
 	subdirs := make(chan []dirJob, workers)
@@ -1193,7 +1238,7 @@ func startWalkWorkers(ctx context.Context, workers int,
 				}
 
 				select {
-				case subdirs <- walkOneDir(ctx, job, oneFS, events):
+				case subdirs <- walkOneDir(ctx, job, oneFS, excludes, events):
 				case <-ctx.Done():
 				}
 			}
@@ -1251,7 +1296,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
 // entry and a warning event per unreadable one, and returns the
 // subdirectories to descend into.
 func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
-	events chan<- walkEvent,
+	excludes map[string]string, events chan<- walkEvent,
 ) []dirJob {
 	entries, err := os.ReadDir(job.path)
 	if err != nil {
@@ -1275,7 +1320,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 		p := filepath.Join(job.path, e.Name())
 
 		if e.IsDir() {
-			if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok {
+			sub, ok := subdirJob(ctx, p, e, job, oneFS, excludes, events)
+			if ok {
 				subs = append(subs, sub)
 			}
 
@@ -1328,17 +1374,28 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 
 // subdirJob applies the descent rules to directory p: never enter
 // .zfs (ZFS snapshot pseudo-dirs would list every file once per
-// snapshot), and with -x never enter a directory on a different
-// filesystem than its operand, sending an event that names it so the
+// snapshot), never enter a default exclude in excludes, and with -x
+// never enter a directory on a different filesystem than its operand.
+// For the last two it sends an event that names the directory, so the
 // scan keeps the records below it, except those below an operand at or
 // inside it.
 func subdirJob(ctx context.Context, p string, e fs.DirEntry,
-	parent dirJob, oneFS bool, events chan<- walkEvent,
+	parent dirJob, oneFS bool, excludes map[string]string,
+	events chan<- walkEvent,
 ) (dirJob, bool) {
 	if e.Name() == ".zfs" {
 		return dirJob{}, false
 	}
 
+	if why, ok := excludes[p]; ok {
+		sendEvent(ctx, events, walkEvent{
+			excludedDir: p,
+			warn:        fmt.Sprintf("scan: skipped %s (%s)", p, why),
+		})
+
+		return dirJob{}, false
+	}
+
 	job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK}
 	if !oneFS || !parent.rootDevOK {
 		return job, true