From 08323879ec5968ba175b4848da2b7519e293708d Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 16 Oct 2026 23:28:07 +0000 Subject: [PATCH] scan: skip OS default excludes, and record nothing on filesystems that are not persistent (closes #124) scan never records or reads anything on a filesystem that is not persistent (tmpfs, proc, sysfs and the like), however it reaches it; no flag turns this off. It notes each such filesystem's device ID as it starts: on Linux from /proc/self/mountinfo, on macOS by lstat of each listed mount point. Below an operand the walk also skips the default excludes, a path list per system (Linux, macOS, and Windows, not used yet); --no-default-excludes turns off only those. A default exclude that is itself an operand gets no skip line. The tests stop at once, saying why, when TMPDIR is not on a persistent filesystem. Judgement call: an unreadable mount list fails the scan, and no flag gets round it. Judgement call: a file on such a filesystem below an operand gets the same line as a directory. Judgement call: such an operand that another operand's walk also reaches is named twice. Unverified: whether macOS's mount list gives the device lstat reports, so macOS keeps lstat, and a hidden mount there can still add another filesystem's device. Unverified: the macOS code is never built or run here; bind mounts need root to test. Model: opus-5-5 --- README.md | 144 +++++++++--- TODO.md | 17 ++ cancel_test.go | 25 +- exclude.go | 192 +++++++++++++++ exclude_darwin.go | 52 +++++ exclude_other.go | 36 +++ exclude_test.go | 583 ++++++++++++++++++++++++++++++++++++++++++++++ main.go | 25 +- main_test.go | 33 ++- scan.go | 319 ++++++++++++++++--------- scan_test.go | 37 ++- 11 files changed, 1294 insertions(+), 169 deletions(-) create mode 100644 exclude.go create mode 100644 exclude_darwin.go create mode 100644 exclude_other.go create mode 100644 exclude_test.go diff --git a/README.md b/README.md index 4273341..148bb72 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,11 @@ As a file `/etc/cron.d/sfdupes`: `/var/log/sfdupes.log`, holds a first line naming the database (see "Database"), plain progress lines naming each phase and its place among the five, as `[2/5] walk` (one as the phase starts, one every 5 seconds while it - runs, and one as it ends), a warning for each path it skips, and the summary - line (see "Progress" and "`scan` mode"). The log grows with every scan; rotate - it like any other. + runs, and one as it ends), a warning for each path it skips, a line for each + default exclude its walk does not enter and for each path it leaves out + because it is not on a persistent filesystem (see "Rules for the walk"), and + the summary line (see "Progress" and "`scan` mode"). The log grows with every + scan; rotate it like any other. - Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the reason among the last lines of the log; `tail` prints them, and cron mails @@ -248,7 +250,7 @@ Four subcommands, all implemented: schema version and the counts of its records. ``` -sfdupes scan [--workers N] [-x] PATH... +sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv sfdupes info @@ -257,8 +259,9 @@ sfdupes [command] --help ``` `--workers N` sets the size of each `scan` worker pool (default: the number of -CPUs), and `-x` (`--one-file-system`) keeps the walk of each operand on that -operand's filesystem; both are described under "`scan` mode". +CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that +operand's filesystem, and `--no-default-excludes` lets the walk enter the +default excludes, a list of paths; all three are described under "`scan` mode". `sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a subcommand, prints the help text to stderr and exits 0, writing nothing to @@ -439,11 +442,12 @@ results do not depend on the working directory. All operands belong to a single scan and are enumerated concurrently: every operand seeds the shared walk worker pool. Overlapping operands are harmless — an operand that duplicates another or lies under another is dropped before walking, so every file is reached exactly -once and produces one database record. With `-x`, though, an operand under -another one is kept and walked on its own when any directory from it (from its -directory, for a regular file) up to but not including the operand covering it -is on a different filesystem than that operand, because that operand's walk does -not enter such a directory and so does not reach it. +once and produces one database record. An operand under another one is kept, +though, and walked on its own when any directory from it (from its directory, +for a regular file) up to but not including the operand covering it is a default +exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with +`-x`, is on a different filesystem than that operand, because that operand's +walk does not enter such a directory and so does not reach it. An operand that is a symlink (never followed, not even as an operand), socket, FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints @@ -453,7 +457,10 @@ beneath it is still scanned. The records stored beneath it are not deleted: they are treated like any other record outside the scanned operands, including the content-phase exception below. If it lies under another operand, they are under that operand instead, and are treated like any other record there. This is not -an error: a scan whose every operand is dropped walks nothing and exits 0. +an error: a scan whose every operand is dropped walks nothing and exits 0. An +operand that is not on a persistent filesystem (see "Rules for the walk") is +dropped the same way, except that the line `scan` prints for it is not a +warning, and it is not counted as skipped. `scan` synchronizes the database with the filesystem state under the scanned operands: @@ -489,14 +496,16 @@ operands: can be scanned on different schedules into the same database. The one exception is the content phase below: a stored file of 10 MiB or more without a `content` hash is read for one, wherever it lies, once its size, `head`, and - `tail` match another record's. If that file is gone or has changed since its - record was written, the record is left as it is. -- With `-x`, a record below a directory the walk did not enter because that - directory is on another filesystem than its operand counts as outside the - scanned operands: it is not deleted, and its size joins the size census. A - filesystem mounted under an operand can therefore be scanned on its own - schedule into the same database. Records below an operand at or inside such a - directory are not kept this way, since that operand's own walk verifies them. + `tail` match another record's. If that file is gone, has changed since its + record was written, or is not on a persistent filesystem, it is not read and + the record is left as it is. +- A record at or below a path the walk left out, because that path is a default + exclude, is not on a persistent filesystem, or, with `-x`, is a directory on + another filesystem than its operand, counts as outside the scanned operands: + it is not deleted, and its size joins the size census. A filesystem mounted + under an operand can therefore be scanned on its own schedule into the same + database. Records below an operand at or inside such a path are not kept this + way, since that operand's own walk verifies them. `scan` runs **five sequential phases over the whole scan**. Parallelism lives inside the walk, hash and content phases; batched database writes begin during @@ -542,19 +551,20 @@ the hash phase: be read are kept in memory, never every file's hashes. Every record sharing their size, `head`, and `tail`, including one that already has a `content` hash, has its file checked with `lstat` first. A file that is gone, is no - longer a regular file, or has changed (a different size, or an mtime newer - than recorded) keeps its record as it is and does not count as a match for - the others. Any other `lstat` error is warned about and counted as skipped, - with the same result. If such a record has no `content` hash, it stays out - of duplicate groups; if it has one, it is still reported until a scan - covering its own tree updates or removes it. The files that pass and have no - `content` hash are read only if at least two of those records pass, so a - file whose only matches are stale costs no read; a file that already has a - `content` hash is never read again. They are read by a worker pool as in the - hash phase, in inode order and once per inode, and their content hashes are - committed in batches. A failed read is warned about and counted as skipped; - its record keeps an empty `content`, so it is not a duplicate, and a later - scan tries again. + longer a regular file, has changed (a different size, or an mtime newer than + recorded), or is not on a persistent filesystem (see "Rules for the walk") + keeps its record as it is and does not count as a match for the others. Any + other `lstat` error is warned about and counted as skipped, with the same + result. If such a record has no `content` hash, it stays out of duplicate + groups; if it has one, it is still reported until a scan covering its own + tree updates or removes it. The files that pass and have no `content` hash + are read only if at least two of those records pass, so a file whose only + matches are stale costs no read; a file that already has a `content` hash is + never read again. They are read by a worker pool as in the hash phase, in + inode order and once per inode, and their content hashes are committed in + batches. A failed read is warned about and counted as skipped; its record + keeps an empty `content`, so it is not a duplicate, and a later scan tries + again. Rules for the walk: @@ -572,6 +582,74 @@ Rules for the walk: unless they lie below an operand at or inside that directory, count as outside the scanned operands, as described in "`scan` mode" above: they are not deleted, and their sizes join the size census. +- Never record anything that is not on a persistent filesystem. A filesystem is + not persistent when the kernel makes it up or keeps it in memory: one of type + `proc`, `sysfs`, `devtmpfs`, `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`, + `debugfs`, `tracefs`, `securityfs`, `pstore`, `bpf`, `configfs`, `fusectl`, + `mqueue`, `hugetlbfs`, `efivarfs`, `binfmt_misc`, `autofs`, `nsfs`, + `rpc_pipefs`, `selinuxfs` or `devfs` (the macOS filesystem of `/dev`). After + it names the database and before it takes the lock, `scan` lists the mounted + filesystems, from `/proc/self/mountinfo` on Linux and from the system's list + of mounted filesystems on macOS (on any other system it lists none), and notes + the device ID of each one of those types. On Linux that is the device ID the + list gives, so a filesystem that a later mount hides still gives its own. On + macOS it is read with `lstat` on the mount point, and where one filesystem is + mounted over another on the same mount point, the one mounted last counts. If + `scan` cannot list them, it fails (exit 1). The walk then never enters a + directory, and never records a file, on one of those devices, however it + reaches it: as an operand, below one, or through a bind mount, which keeps the + device ID of what it binds. The content phase never reads a file on one of + them either. No flag turns this off. For each operand, directory or file it + leaves out this way, `scan` prints a line on stderr naming it and the type of + its filesystem, as it prints a warning, so it never lands inside a progress + display: + + ``` + scan: skipped /dev/shm (tmpfs, not a persistent filesystem) + ``` + + That line is not a warning, and the path is not counted as skipped. Records + that an earlier scan left at or below it, unless they lie below an operand + at or inside it, count as outside the scanned operands, as described in + "`scan` mode" above: they are not deleted, and their sizes join the size + census. + +- Below an operand, never descend into a default exclude: a path that holds the + kernel's view of the running system, or temporary files, so that + `sfdupes scan /` walks only the files worth comparing. The default excludes + depend on the system, and `sfdupes scan --help` lists those of the system it + runs on: + - On Linux: `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`. + - On macOS: `/dev`, `/private/tmp`, `/private/var/tmp`, `/private/var/vm`, + `/System/Volumes/VM` and `/System/Volumes/Data`. A walk of `/` reaches + everything in `/System/Volumes/Data` through the firmlinks, so walking it + as well would record each of those files twice. + - On Windows, relative to the root of the drive an operand lies on: the + page, swap and hibernation files `pagefile.sys`, `swapfile.sys` and + `hiberfil.sys`, and the directories `System Volume Information`, + `$Recycle.Bin`, `Windows\Temp` and `Users\*\AppData\Local\Temp`, where `*` + matches one path component, as it does in any default exclude. sfdupes + does not build for Windows yet, so this list is not used. + - On any other system there are none. + + `--no-default-excludes` turns them all off; what is not on a persistent + filesystem stays left out. An operand at or inside a default exclude is + still walked, so `sfdupes scan /tmp/x` works, unless it is not on a + persistent filesystem. For each default exclude its walk does not enter, + `scan` prints a line on stderr naming it, as it prints a warning: + + ``` + scan: skipped /var/tmp (default exclude) + ``` + + That line is not a warning, and the directory is not counted as skipped. A + default exclude that is not on a persistent filesystem, as `/proc` is, gets + the line for its filesystem instead, and one that is itself an operand gets + no line, since it is walked as that operand. The records below a default + exclude the walk does not enter, unless they lie below an operand at or + inside it, count as outside the scanned operands, as described in "`scan` + mode" above: they are not deleted, and their sizes join the size census. + - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were diff --git a/TODO.md b/TODO.md index b814411..37fc9a6 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,23 @@ # Completed Steps +- `scan` never records or reads anything that is not on a persistent filesystem + (`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types), + matched by the device ID of each such filesystem, listed as the scan starts + from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS, + however the walk or the content phase reaches it; no flag turns that off. + Below an operand, its walk does not enter the default excludes either, so + `sfdupes scan /` skips the kernel's view of the running system and the + temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and + `/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`, + `/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a + Windows list, not used yet. An operand at or inside a default exclude is still + walked. The scan prints a line for each path it leaves out, without counting + it as skipped, and the records there count as outside the scanned operands, as + for `-x`. `--no-default-excludes` turns off the default excludes only, and + `scan --help` lists them (2026-10-17, + https://git.eeqj.de/sneak/sfdupes/issues/124) + - the rate a progress display shows covers the items completed over the last 5 seconds, or since the line before when stderr is not a terminal, so it falls to 0 while nothing completes, and a burst after a wait no longer reads as a diff --git a/cancel_test.go b/cancel_test.go index da543fd..20e72e2 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -196,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { base := baselineGoroutines(t) st, err := syncScan(newWalkClock(walkCancelAtDone), db, - []string{dir}, walkCancelWorkers, false) + []string{dir}, walkCancelWorkers, false, walkRules{}) assertWalkGuardAborted(t, st, err) assertRecordsIntact(t, db, before) @@ -268,7 +268,8 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false) + st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false, + walkRules{}) if !errors.Is(err, context.Canceled) { t.Fatalf("syncScan on a cancelled context = %v, want %v", err, context.Canceled) @@ -301,7 +302,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) { db := openTestDB(t) st, err := syncScan(newWalkClock(hashCancelAtDone), db, - []string{dir}, walkCancelWorkers, false) + []string{dir}, walkCancelWorkers, false, walkRules{}) if !errors.Is(err, context.Canceled) { t.Fatalf("syncScan cancelled mid-hash = %v, want %v", err, context.Canceled) @@ -356,7 +357,8 @@ func TestRunScanInterrupted(t *testing.T) { stderr := captureStderr(t) dir := buildSmokeTree(t) - err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false, + false) if err != nil { t.Fatal(err) } @@ -380,7 +382,7 @@ func TestRunScanInterrupted(t *testing.T) { defer close(done) err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers, - false) + false, false) }() awaitReturn(t, done, "runScan") @@ -407,7 +409,8 @@ func TestRunScanInterrupted(t *testing.T) { got, before) } - err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false, + false) if err != nil { t.Fatal(err) } @@ -433,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) { dir := buildWalkCancelTree(t) err := runScan(newWalkClock(hashCancelAtDone), []string{dir}, - walkCancelWorkers, false) + walkCancelWorkers, false, false) if !errors.Is(err, errInterrupted) { t.Fatalf("runScan interrupted mid-hash = %v, want %v", err, errInterrupted) @@ -617,7 +620,8 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) { // Unbuffered and unread: on a cancelled scan every send gives up. events := make(chan walkEvent) - subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events) + subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, + walkRules{}, events) if len(subs) != 0 { t.Errorf("cancelled walkOneDir returned %+v to descend into, "+ "want none", subs) @@ -637,7 +641,8 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) { missing := filepath.Join(t.TempDir(), "missing") - jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false) + jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false, + walkRules{}) for range 64 { jobs <- dirJob{path: missing} @@ -668,7 +673,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - jobs, subdirs, events := startWalkWorkers(ctx, 1, false) + jobs, subdirs, events := startWalkWorkers(ctx, 1, false, walkRules{}) // Fill the hand-back channel to its capacity — one slot per worker // — so the worker's own hand-back is certain to block. diff --git a/exclude.go b/exclude.go new file mode 100644 index 0000000..9700258 --- /dev/null +++ b/exclude.go @@ -0,0 +1,192 @@ +package main + +// This file holds two rules for the walk (README "Rules for the walk"): +// the default excludes, the paths a scan's walk does not enter below an +// operand unless scan is given --no-default-excludes, and the +// filesystems that are not persistent, on which the scan never records +// or reads anything. + +import ( + "bufio" + "errors" + "fmt" + "io" + "io/fs" + "path/filepath" + "slices" + "strings" + + "golang.org/x/sys/unix" +) + +// The systems that have default excludes, as runtime.GOOS names them. +const ( + goosLinux = "linux" + goosDarwin = "darwin" + goosWindows = "windows" +) + +// The fields of a line of mountInfoPath that readMountInfo reads, +// counted from 0 (see proc(5)): the device ID, as major:minor, is the +// third field, and after the sixth come optional fields, then a field +// holding "-", then the filesystem type. +const ( + deviceField = 2 + optionalFieldsStart = 6 +) + +// typesPerHelpLine is how many filesystem types the scan command's help +// lists on each line. +const typesPerHelpLine = 6 + +// errMountInfoLine reports a line of mountInfoPath without the fields +// readMountInfo reads. +var errMountInfoLine = errors.New("unexpected line") + +// walkRules holds the two rules for the walk that a scan sets up as it +// starts. excludes are the default excludes, or none with +// --no-default-excludes. nonPersistent maps the device ID of every +// mounted filesystem that is not persistent to its type; the scan +// records and reads nothing on one of those, whatever the flags. +type walkRules struct { + excludes []string + nonPersistent map[uint64]string +} + +// defaultExcludes returns the default excludes of the system goos +// names, as runtime.GOOS does: a fixed list on Linux, macOS and +// Windows, and none on any other system. +func defaultExcludes(goos string) []string { + switch goos { + case goosLinux: + return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"} + case goosDarwin: + // A walk of / reaches everything in /System/Volumes/Data through + // the firmlinks, so walking it as well would record each of + // those files twice. + return []string{ + "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", + "/System/Volumes/VM", "/System/Volumes/Data", + } + case goosWindows: + // Relative to the root of the drive an operand lies on. The + // first three are the page, swap and hibernation files. + return []string{ + "pagefile.sys", "swapfile.sys", "hiberfil.sys", + "System Volume Information", "$Recycle.Bin", `Windows\Temp`, + `Users\*\AppData\Local\Temp`, + } + default: + return nil + } +} + +// isDefaultExclude reports whether path is one of excludes, in which a +// * matches one path component. +func isDefaultExclude(path string, excludes []string) bool { + for _, pattern := range excludes { + // The patterns are fixed, so none is malformed. + if ok, _ := filepath.Match(pattern, path); ok { + return true + } + } + + return false +} + +// nonPersistentFilesystemTypes returns the filesystem types that are +// not persistent: filesystems that the kernel makes up or that are kept +// in memory. devfs is the macOS filesystem of /dev; the others are +// Linux names. +func nonPersistentFilesystemTypes() []string { + return []string{ + "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs", + "cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore", + "bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs", + "binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", "devfs", + } +} + +// nonPersistentLine returns the line the scan prints for path when fi, +// its lstat, puts it on a filesystem in rules.nonPersistent, and "" +// when it is on any other. +func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string { + dev, ok := deviceOfInfo(fi) + if !ok { + return "" + } + + fsType, found := rules.nonPersistent[dev] + if !found { + return "" + } + + return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)", + path, fsType) +} + +// readMountInfo reads mountinfo, laid out as mountInfoPath is, and +// returns the device ID of every mounted filesystem that is not +// persistent, with its type. The device ID is the one each line gives, +// not that of the file now at its mount point, so a mount that a later +// mount hides never adds the device of another filesystem. +func readMountInfo(mountinfo io.Reader) (map[uint64]string, error) { + types := nonPersistentFilesystemTypes() + devices := make(map[uint64]string) + lines := bufio.NewScanner(mountinfo) + + for lines.Scan() { + fields := strings.Fields(lines.Text()) + + sep := slices.Index(fields, "-") + if sep < optionalFieldsStart || sep+1 >= len(fields) { + return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text()) + } + + var major, minor uint32 + + _, err := fmt.Sscanf(fields[deviceField], "%d:%d", &major, &minor) + if err != nil { + return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text()) + } + + fsType := fields[sep+1] + if slices.Contains(types, fsType) { + devices[unix.Mkdev(major, minor)] = fsType + } + } + + return devices, lines.Err() +} + +// defaultExcludesHelp describes, for the scan command's help, the +// default excludes of the system goos names and, where the scan lists +// the mounted filesystems, the types it never records anything on. +func defaultExcludesHelp(goos string) string { + var help strings.Builder + + dirs := defaultExcludes(goos) + if len(dirs) == 0 { + help.WriteString("This system has no default excludes.\n") + } else { + help.WriteString("Below an operand, the walk does not enter these " + + "default excludes, unless\n--no-default-excludes is given:\n\n") + + for _, dir := range dirs { + fmt.Fprintf(&help, " %s\n", dir) + } + } + + if goos == goosLinux || goos == goosDarwin { + help.WriteString("\nWith or without --no-default-excludes, the scan " + + "records nothing on a\nfilesystem that is not persistent, " + + "one of these types:\n\n") + + types := nonPersistentFilesystemTypes() + for line := range slices.Chunk(types, typesPerHelpLine) { + fmt.Fprintf(&help, " %s\n", strings.Join(line, ", ")) + } + } + + return help.String() +} diff --git a/exclude_darwin.go b/exclude_darwin.go new file mode 100644 index 0000000..a669ab5 --- /dev/null +++ b/exclude_darwin.go @@ -0,0 +1,52 @@ +//go:build darwin + +package main + +import ( + "slices" + + "golang.org/x/sys/unix" +) + +// nonPersistentFilesystems returns the device ID of every mounted +// filesystem that is not persistent, with its type, from the system's +// list of mounted filesystems. The device ID is read with lstat on the +// mount point. Where one filesystem is mounted over another on the same +// mount point, the later one in the list counts. A mount point that +// lstat cannot reach is left out. +func nonPersistentFilesystems() (map[uint64]string, error) { + // The first call counts the filesystems, the second lists them. + n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + + list := make([]unix.Statfs_t, n) + + n, err = unix.Getfsstat(list, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + + mounts := make(map[string]string, n) + + for _, st := range list[:n] { + mounts[unix.ByteSliceToString(st.Mntonname[:])] = + unix.ByteSliceToString(st.Fstypename[:]) + } + + types := nonPersistentFilesystemTypes() + devices := make(map[uint64]string) + + for mountPoint, fsType := range mounts { + if !slices.Contains(types, fsType) { + continue + } + + if dev, ok := deviceOfPath(mountPoint); ok { + devices[dev] = fsType + } + } + + return devices, nil +} diff --git a/exclude_other.go b/exclude_other.go new file mode 100644 index 0000000..80b26a6 --- /dev/null +++ b/exclude_other.go @@ -0,0 +1,36 @@ +//go:build !darwin + +package main + +import ( + "fmt" + "os" + "runtime" +) + +// mountInfoPath is the file in which Linux lists the mounts the reading +// process sees. +const mountInfoPath = "/proc/self/mountinfo" + +// nonPersistentFilesystems returns the device ID of every mounted +// filesystem that is not persistent, with its type: on Linux read from +// mountInfoPath (see readMountInfo), and on any other system none. +func nonPersistentFilesystems() (map[uint64]string, error) { + if runtime.GOOS != goosLinux { + return map[uint64]string{}, nil + } + + f, err := os.Open(mountInfoPath) + if err != nil { + return nil, err + } + + defer func() { _ = f.Close() }() + + devices, err := readMountInfo(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", mountInfoPath, err) + } + + return devices, nil +} diff --git a/exclude_test.go b/exclude_test.go new file mode 100644 index 0000000..20f195d --- /dev/null +++ b/exclude_test.go @@ -0,0 +1,583 @@ +package main + +import ( + "bytes" + "errors" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +func TestDefaultExcludes(t *testing.T) { + t.Parallel() + + // README "Rules for the walk": each system's list. + cases := map[string][]string{ + goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, + goosDarwin: { + "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", + "/System/Volumes/VM", "/System/Volumes/Data", + }, + goosWindows: { + "pagefile.sys", "swapfile.sys", "hiberfil.sys", + "System Volume Information", "$Recycle.Bin", `Windows\Temp`, + `Users\*\AppData\Local\Temp`, + }, + "freebsd": nil, + } + + for goos, want := range cases { + if got := defaultExcludes(goos); !slices.Equal(got, want) { + t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want) + } + } +} + +func TestIsDefaultExclude(t *testing.T) { + t.Parallel() + + // A * matches one path component, no more and no fewer. + excludes := []string{"/scratch", "/home/*/cache"} + + cases := map[string]bool{ + "/scratch": true, + "/scratch/x": false, + "/home/u/cache": true, + "/home/u/v/cache": false, + "/home/cache": false, + } + + for path, want := range cases { + if got := isDefaultExclude(path, excludes); got != want { + t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want) + } + } +} + +func TestReadMountInfo(t *testing.T) { + t.Parallel() + + // The second-to-last line has no optional fields. The tmpfs on + // /data/a/b is hidden by the root filesystem's /srv, mounted over + // /data/a after it: the tmpfs still gives its own device, and the + // root filesystem's device is not listed. + mountinfo := strings.Join([]string{ + "22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw", + "23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw", + "24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw", + "25 22 0:23 / /data/a/b rw shared:2 - tmpfs none rw", + "26 22 8:1 /srv /data/a rw - ext4 /dev/sda1 rw", + "27 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw", + }, "\n") + + got, err := readMountInfo(strings.NewReader(mountinfo)) + if err != nil { + t.Fatal(err) + } + + want := map[uint64]string{ + unix.Mkdev(0, 21): "proc", unix.Mkdev(0, 22): "sysfs", + unix.Mkdev(0, 23): "tmpfs", unix.Mkdev(0, 25): "autofs", + } + if !maps.Equal(got, want) { + t.Errorf("readMountInfo = %v, want %v", got, want) + } + + for _, line := range []string{ + "22 1 8:1 / / rw,relatime\n", + "22 1 sda / / rw,relatime shared:1 - ext4 /dev/sda1 rw\n", + } { + _, err = readMountInfo(strings.NewReader(line)) + if !errors.Is(err, errMountInfoLine) { + t.Errorf("readMountInfo(%q) = %v, want %v", line, err, + errMountInfoLine) + } + } +} + +func TestNonPersistentFilesystems(t *testing.T) { + t.Parallel() + + if runtime.GOOS != goosLinux { + t.Skip("the test knows a mount only on Linux: /proc") + } + + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfPath("/proc") + if !ok { + t.Fatal("no device for /proc") + } + + if got := nonPersistent[dev]; got != "proc" { + t.Errorf("filesystem of /proc = %q, want %q", got, "proc") + } +} + +func TestDefaultExcludesHelp(t *testing.T) { + t.Parallel() + + for _, goos := range []string{goosLinux, goosDarwin, goosWindows} { + help := defaultExcludesHelp(goos) + + for _, dir := range defaultExcludes(goos) { + if !strings.Contains(help, "\n "+dir+"\n") { + t.Errorf("help for %s does not list %s:\n%s", goos, dir, help) + } + } + + // Only Linux and macOS list their mounted filesystems. + for _, fsType := range nonPersistentFilesystemTypes() { + listed := strings.Contains(help, " "+fsType+",") || + strings.Contains(help, " "+fsType+"\n") + if listed != (goos != goosWindows) { + t.Errorf("help for %s lists %s: %v", goos, fsType, listed) + } + } + } + + want := "This system has no default excludes.\n" + if got := defaultExcludesHelp("freebsd"); got != want { + t.Errorf("help for freebsd = %q, want %q", got, want) + } + + // scan's help shows those of the system it runs on. + var stdout, stderr bytes.Buffer + + run([]string{cmdScan, "-h"}, &stdout, &stderr) + + help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS)) + if !strings.Contains(stderr.String(), help) { + t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help) + } +} + +// TestScanDoesNotEnterDefaultExclude checks that the walk does not +// enter a default exclude below an operand. It says so on a line of its +// own and does not count it as skipped. The records below it count as +// outside the operands: the record of a file gone from it is kept, and +// its size joins the size census, while a file added to it is not +// recorded. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanDoesNotEnterDefaultExclude(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + excluded := filepath.Join(dir, "tmp") + + // A scan with no default excludes records the file in excluded. + gone := writeFile(t, excluded, "gone", pattern(1, 100)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + unseen := writeFile(t, excluded, "new", pattern(2, 10)) + + // The same size as gone, so it is hashed only if the census holds the + // size of gone's record. + added := writeFile(t, dir, "added", pattern(3, 100)) + + st, err := syncScan(t.Context(), db, []string{dir}, 1, false, + walkRules{excludes: []string{excluded}}) + if err != nil { + t.Fatal(err) + } + + if st.skipped != 0 { + t.Errorf("skipped = %d, want 0", st.skipped) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, gone) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, gone) + } + + if slices.Contains(recordPaths(recs), unseen) { + t.Errorf("%s recorded: the walk entered %s", unseen, excluded) + } + + line := "\nscan: skipped " + excluded + " (default exclude)\n" + if !strings.Contains(stderr(), line) { + t.Errorf("stderr = %q, want %q", stderr(), line) + } +} + +// TestScanWalksOperandInDefaultExclude checks that an operand at or +// inside a default exclude is walked, also when it lies under another +// operand, whose walk does not enter the default exclude. +func TestScanWalksOperandInDefaultExclude(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + excluded := filepath.Join(dir, "tmp") + f := writeFile(t, excluded, "sub/f", pattern(1, 10)) + sub := filepath.Dir(f) + + cases := map[string][]string{ + "at it": {excluded}, + "inside it": {sub}, + "inside it under an operand": {dir, sub}, + } + + for name, roots := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + st, err := syncScan(t.Context(), db, roots, 1, false, + walkRules{excludes: []string{excluded}}) + if err != nil { + t.Fatal(err) + } + + if st.added != 1 { + t.Errorf("added = %d, want %s recorded once", st.added, f) + } + + recordByPath(t, dbRecords(t, db), f) + }) + } +} + +// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default +// exclude given as an operand under another operand, which the walk of +// that other operand does not enter, gets no line saying it was +// skipped: it is walked as an operand of its own. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + excluded := filepath.Join(dir, "tmp") + f := writeFile(t, excluded, "f", pattern(1, 10)) + + st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false, + walkRules{excludes: []string{excluded}}) + if err != nil { + t.Fatal(err) + } + + if st.added != 1 { + t.Errorf("added = %d, want %s recorded once", st.added, f) + } + + recordByPath(t, dbRecords(t, db), f) + + if strings.Contains(stderr(), "scan: skipped") { + t.Errorf("stderr = %q, want no line for %s", stderr(), excluded) + } +} + +// TestRunScanTempDirOperand scans a tree in the tests' temporary +// directory, which on Linux lies inside the default exclude /tmp unless +// TMPDIR names another place, with the system's own default excludes: +// the scan walks the whole tree and reports no default exclude. +func TestRunScanTempDirOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) { + t.Skip(dir + " is not inside a default exclude") + } + + f := writeFile(t, dir, "sub/f", pattern(1, 10)) + + stderr := scanOK(t, dir) + if strings.Contains(stderr, "scan: skipped") { + t.Errorf("stderr = %q, want no line saying a path was skipped", + stderr) + } + + assertRecorded(t, path, f, true) +} + +// TestRunScanNoDefaultExcludes scans /var through the command. Its +// /var/tmp, on a persistent filesystem, is a default exclude on Linux: +// without --no-default-excludes the walk does not enter it and says so, +// and with that flag the walk records a file the test puts there. +func TestRunScanNoDefaultExcludes(t *testing.T) { + if runtime.GOOS != goosLinux { + t.Skip("only Linux has /var/tmp among its default excludes") + } + + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + t.Fatal(err) + } + + rules := walkRules{nonPersistent: nonPersistent} + + for _, dir := range []string{"/var", "/var/tmp"} { + fi, lstatErr := os.Lstat(dir) + if lstatErr != nil || !fi.IsDir() || + rules.nonPersistentLine(dir, fi) != "" { + t.Skip(dir + " is not a directory on a persistent filesystem here") + } + } + + //nolint:usetesting // the directory must be inside /var/tmp + tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-") + if err != nil { + t.Skip("cannot write to /var/tmp here: " + err.Error()) + } + + t.Cleanup(func() { _ = os.RemoveAll(tmp) }) + + f := writeFile(t, tmp, "f", pattern(1, 10)) + + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + line := "scan: skipped /var/tmp (default exclude)\n" + + if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) { + t.Errorf("scan /var: stderr = %q, want %q", stderr, line) + } + + assertRecorded(t, path, f, false) + + stderr := scanOK(t, "--no-default-excludes", "/var") + if strings.Contains(stderr, line) { + t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q", + stderr, line) + } + + assertRecorded(t, path, f, true) +} + +// TestRunScanSkipsNonPersistentOperand scans, through the command, a +// directory on a filesystem that is not persistent, with and without +// --no-default-excludes: either way it is skipped, with a line saying +// so, and nothing in it is recorded. +func TestRunScanSkipsNonPersistentOperand(t *testing.T) { + shm, fsType, _ := shmTempDir(t) + writeFile(t, shm, "f", pattern(1, 10)) + + t.Setenv(databaseEnv, testDBPath(t)) + + line := "scan: skipped " + shm + " (" + fsType + + ", not a persistent filesystem)\n" + summary := "scan: 0 files seen" + + for _, flags := range [][]string{nil, {"--no-default-excludes"}} { + stderr := scanOK(t, append(flags, shm)...) + if !strings.Contains(stderr, line) || + !strings.Contains(stderr, summary) { + t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr, + line, summary) + } + } +} + +// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem +// that is not persistent beside one on the tests' temporary directory. +// The records an earlier scan left in it count as outside the operands: +// the record of a file gone from it is kept, and its size joins the size +// census, while a file added to it is not recorded, and its size does +// not join the census. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanKeepsRecordsOnNonPersistent(t *testing.T) { + shm, fsType, rules := shmTempDir(t) + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + + // A scan that, without the list of filesystems, records gone. + gone := writeFile(t, shm, "gone", pattern(1, 100)) + syncTree(t, db, shm) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + unseen := writeFile(t, shm, "new", pattern(2, 10)) + + // added has the size of gone, and lonely that of unseen: each is + // hashed only if the census holds that size. + added := writeFile(t, dir, "added", pattern(3, 100)) + lonely := writeFile(t, dir, "lonely", pattern(4, 10)) + + st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules) + if err != nil { + t.Fatal(err) + } + + if st.skipped != 0 { + t.Errorf("skipped = %d, want 0", st.skipped) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, gone) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, gone) + } + + if recordByPath(t, recs, lonely).head != "" { + t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen) + } + + if slices.Contains(recordPaths(recs), unseen) { + t.Errorf("%s recorded", unseen) + } + + line := "scan: skipped " + shm + " (" + fsType + + ", not a persistent filesystem)\n" + if !strings.Contains(stderr(), line) { + t.Errorf("stderr = %q, want %q", stderr(), line) + } +} + +// TestScanContentSkipsNonPersistent checks that the content phase does +// not read a file on a filesystem that is not persistent, though an +// earlier scan left a record for it: it is treated as a changed file, +// so its record is left as it is and its identical copy in another +// tree, having no other match, is not read either. +func TestScanContentSkipsNonPersistent(t *testing.T) { + t.Parallel() + + shm, _, rules := shmTempDir(t) + db := openTestDB(t) + + // A scan that, without the list of filesystems, records a with a + // head and tail but no content hash. + a := sparseFileWithoutMatch(t, shm, "a", headTailMin) + + syncTree(t, db, shm) + + before := recordByPath(t, dbRecords(t, db), a) + if before.head == "" || before.content != "" { + t.Fatalf("after scanning %s: %+v, want head and tail only", shm, before) + } + + b := sparseFile(t, t.TempDir(), "b", headTailMin) + + st, err := syncScan(t.Context(), db, []string{filepath.Dir(b)}, 1, false, + rules) + if err != nil { + t.Fatal(err) + } + + if st.skipped != 0 { + t.Errorf("skipped = %d, want 0", st.skipped) + } + + recs := dbRecords(t, db) + if r := recordByPath(t, recs, a); r != before { + t.Errorf("record = %+v, want it left as %+v", r, before) + } + + if r := recordByPath(t, recs, b); r.content != "" { + t.Errorf("%s: content = %q, want none: its only match is on %s", + b, r.content, shm) + } +} + +// TestWalkLeavesOutNonPersistent starts the walk itself in a directory +// on a filesystem that is not persistent, which a scan never does: it +// drops such an operand first. Below a directory on a persistent +// filesystem, one on another takes a mount, which a test cannot make. +// The walk enters no directory and records no file there, and names +// each. +func TestWalkLeavesOutNonPersistent(t *testing.T) { + t.Parallel() + + shm, fsType, rules := shmTempDir(t) + f := writeFile(t, shm, "f", pattern(1, 10)) + sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10))) + + var leftOut []string + + for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) { + if ev.rec.path != "" { + t.Errorf("walk recorded %s", ev.rec.path) + } + + if ev.leftOut == "" { + continue + } + + leftOut = append(leftOut, ev.leftOut) + + want := "scan: skipped " + ev.leftOut + " (" + fsType + + ", not a persistent filesystem)" + if ev.warn != want { + t.Errorf("line = %q, want %q", ev.warn, want) + } + } + + slices.Sort(leftOut) + + if want := []string{f, sub}; !slices.Equal(leftOut, want) { + t.Errorf("walk left out %q, want %q", leftOut, want) + } +} + +// shmTempDir returns a new directory in /dev/shm, removed when the test +// ends, the type of its filesystem, and the rules holding the real +// filesystems that are not persistent. It skips the test where /dev/shm +// is on a persistent filesystem, or cannot be written to. +func shmTempDir(t *testing.T) (string, string, walkRules) { + t.Helper() + + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfPath("/dev/shm") + + fsType, found := nonPersistent[dev] + if !ok || !found { + t.Skip("/dev/shm is not a memory-backed filesystem here") + } + + //nolint:usetesting // the directory must be inside /dev/shm + dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-") + if err != nil { + t.Skip("cannot write to /dev/shm here: " + err.Error()) + } + + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + return dir, fsType, walkRules{nonPersistent: nonPersistent} +} + +// assertRecorded checks that the database at dbPath holds a record for +// path when want is set, and none when it is not. +func assertRecorded(t *testing.T, dbPath, path string, want bool) { + t.Helper() + + db, err := openDB(dbPath, reportParams) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want { + t.Errorf("%s recorded: %v, want %v", path, got, want) + } +} diff --git a/main.go b/main.go index 8f33973..1c3843c 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ // // Usage: // -// sfdupes scan [--workers N] [-x] PATH... +// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv // sfdupes info @@ -182,17 +182,23 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { return root } -// newScanCommand builds the scan subcommand, with its --workers and -x -// flags. +// newScanCommand builds the scan subcommand, with its --workers, -x and +// --no-default-excludes flags. Its help lists the default excludes of +// the system it runs on, and the filesystem types it never records +// anything on. func newScanCommand() *cobra.Command { var ( - scanWorkers int - scanOneFS bool + scanWorkers int + scanOneFS bool + scanNoDefaultExcludes bool ) + short := "Walk trees and synchronize the scan database" + scanCmd := &cobra.Command{ - Use: cmdScan + " [--workers N] [-x] PATH...", - Short: "Walk trees and synchronize the scan database", + Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...", + Short: short, + Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS), Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, _ []string) error { return checkScanWorkers(cmd, scanWorkers) @@ -201,13 +207,16 @@ func newScanCommand() *cobra.Command { ctx, stop := interruptContext(ctx) defer stop() - return runScan(ctx, args, scanWorkers, scanOneFS) + return runScan(ctx, args, scanWorkers, scanOneFS, + scanNoDefaultExcludes) }), } scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(), "concurrent workers for the walk, hash, and content phases") scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false, "do not cross filesystem boundaries") + scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes", + false, "enter the default excludes, the paths listed above") return scanCmd } diff --git a/main_test.go b/main_test.go index f93fca2..8cd0b6f 100644 --- a/main_test.go +++ b/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "io/fs" "os" @@ -18,6 +19,36 @@ import ( // error prints and a runtime failure must not. const usageMarker = "Usage:" +// TestMain stops the tests at once, saying why, when their temporary +// directory is not on a persistent filesystem: scan records nothing +// there (README "Rules for the walk"), so most scan tests would fail +// without saying why. +func TestMain(m *testing.M) { + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + fmt.Fprintf(os.Stderr, "list the mounted filesystems: %v\n", err) + os.Exit(1) + } + + dir := os.TempDir() + + fi, err := os.Stat(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + dev, ok := deviceOfInfo(fi) + if fsType, found := nonPersistent[dev]; ok && found { + fmt.Fprintf(os.Stderr, "%s is on %s, not a persistent filesystem, "+ + "and scan records nothing there: set TMPDIR to a directory on a "+ + "persistent filesystem\n", dir, fsType) + os.Exit(1) + } + + os.Exit(m.Run()) +} + // walSuffixes are the SQLite sidecar files a WAL-mode database keeps // while it is open. A clean close checkpoints the WAL and removes // both; finding either afterwards means the database was never closed. @@ -535,7 +566,7 @@ func TestRunNamesDefaultDatabase(t *testing.T) { }{ cmdScan: { run: func() error { - return runScan(t.Context(), []string{missing}, 1, false) + return runScan(t.Context(), []string{missing}, 1, false, false) }, want: scanWant, }, diff --git a/scan.go b/scan.go index 642055b..a10984f 100644 --- a/scan.go +++ b/scan.go @@ -13,6 +13,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -114,9 +115,12 @@ func scanPhase(name string) string { // SIGTERM that interruptContext catches, the scan keeps what it has // hashed (see syncScan), prints how many files its walk reached, and // returns errInterrupted. workers must be at least 1; the scan command -// rejects anything less. +// rejects anything less. Before the lock is taken, the scan lists the +// filesystems that are not persistent, on which the walk records +// nothing; unless noDefaultExcludes is set, the walk does not enter the +// default excludes either. func runScan(ctx context.Context, roots []string, workers int, - oneFS bool, + oneFS, noDefaultExcludes bool, ) error { dbPath, source, err := databasePath() if err != nil { @@ -139,6 +143,16 @@ func runScan(ctx context.Context, roots []string, workers int, return err } + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + return fmt.Errorf("list the mounted filesystems: %w", err) + } + + rules := walkRules{nonPersistent: nonPersistent} + if !noDefaultExcludes { + rules.excludes = defaultExcludes(runtime.GOOS) + } + lock, err := lockScanDatabase(dbPath) if err != nil { return err @@ -159,7 +173,7 @@ func runScan(ctx context.Context, roots []string, workers int, defer closeScanDatabase(ctx, db, dbPath) - st, err := syncScan(ctx, db, roots, workers, oneFS) + st, err := syncScan(ctx, db, roots, workers, oneFS, rules) if errors.Is(err, context.Canceled) { return interrupted(st.walked) } @@ -231,14 +245,12 @@ func resolveRoots(roots []string) ([]string, error) { } // pruneRoots drops operands already covered by another operand: -// duplicates and any operand lying under another one. With -x, an -// operand under another one is kept when the walk of the deepest kept -// operand above it cannot reach it: when device puts any directory from -// it (from its directory, for a regular file) up to but not including -// that operand on a different filesystem than that operand. Every -// remaining file is then reachable through exactly one root, so walked -// paths are unique without keeping a scan-wide set of every path seen. -func pruneRoots(roots []string, oneFS bool, +// duplicates and any operand lying under another one. An operand under +// another one is kept, though, when the walk of the deepest kept operand +// above it does not reach it (see walkReaches). Every remaining file is +// then reachable through exactly one root, so walked paths are unique +// without keeping a scan-wide set of every path seen. +func pruneRoots(roots []string, oneFS bool, rules walkRules, device func(path string) (uint64, bool), ) []string { // Shorter-first ordering guarantees an ancestor is kept before any @@ -262,40 +274,57 @@ func pruneRoots(roots []string, oneFS bool, continue } - // Without -x, and always for a duplicate, r is dropped. - if !oneFS || above == r { - continue - } - - aboveDev, aboveOK := device(above) - if !aboveOK { - continue - } - - // The walk of above enters a directory only when it is on - // above's filesystem, and then reaches every regular file in it - // whatever device the file reports, so for a regular-file - // operand the directories checked start at its directory. - dir := r - - fi, err := os.Lstat(r) - if err == nil && fi.Mode().IsRegular() { - dir = filepath.Dir(r) - } - - for ; dir != above; dir = filepath.Dir(dir) { - dev, ok := device(dir) - if ok && dev != aboveDev { - kept = append(kept, r) - - break - } + // A duplicate is dropped, and so is an operand the walk of above + // reaches. + if above != r && !walkReaches(above, r, oneFS, rules, device) { + kept = append(kept, r) } } return kept } +// walkReaches reports whether the walk of operand above reaches r, an +// operand under it. That walk does not enter a default exclude in +// rules, nor a directory that device puts on a filesystem in rules that +// is not persistent, nor, with -x, one that device puts on a different +// filesystem than above, when device knows above's. It reaches every +// regular file in a directory it enters, so for a regular-file operand +// the directories checked start at its directory; they end below above. +func walkReaches(above, r string, oneFS bool, rules walkRules, + device func(path string) (uint64, bool), +) bool { + aboveDev, aboveDevOK := device(above) + + dir := r + + fi, err := os.Lstat(r) + if err == nil && fi.Mode().IsRegular() { + dir = filepath.Dir(r) + } + + for ; dir != above; dir = filepath.Dir(dir) { + if isDefaultExclude(dir, rules.excludes) { + return false + } + + dev, ok := device(dir) + if !ok { + continue + } + + if _, found := rules.nonPersistent[dev]; found { + return false + } + + if oneFS && aboveDevOK && dev != aboveDev { + return false + } + } + + return true +} + // deviceOfPath returns the filesystem device ID of the file at path, // when lstat succeeds and the platform exposes one. func deviceOfPath(path string) (uint64, bool) { @@ -337,11 +366,11 @@ type scanState struct { // the hashed records still waiting in the batch, starts no other write // or deletion, and returns the cancellation. func syncScan(ctx context.Context, db *sql.DB, roots []string, - workers int, oneFS bool, + workers int, oneFS bool, rules walkRules, ) (scanStats, error) { s := &scanState{db: db} - err := s.runPhases(ctx, roots, workers, oneFS) + err := s.runPhases(ctx, roots, workers, oneFS, rules) if err == nil || ctx.Err() == nil { return s.st, err } @@ -365,15 +394,18 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string, // and content (fill in the content hash of every record of headTailMin // or more whose size, head, and tail match another record's). Records // outside the roots are never touched, except that the content phase -// fills in their content hash. Operands the walk cannot start from are -// dropped first, so the records beneath them are treated as if those -// operands had not been given. +// fills in their content hash. Operands the walk cannot start from, or +// that are not on a persistent filesystem, are dropped first, so the +// records beneath them are treated as if those operands had not been +// given. Below an operand, the walk follows rules. func (s *scanState) runPhases(ctx context.Context, roots []string, - workers int, oneFS bool, + workers int, oneFS bool, rules walkRules, ) error { - // Types are checked before pruning so that an operand under a - // dropped one is still scanned, not dropped as lying under it. - roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath) + // Types and filesystems are checked before pruning so that an + // operand under a dropped one is still scanned, not dropped as lying + // under it. + roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules, + deviceOfPath) err := s.loadIndex(ctx, roots) if err != nil { @@ -381,7 +413,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, } changed, unhashed := s.walkPhase( - startWalk(ctx, roots, oneFS, workers), roots) + startWalk(ctx, roots, oneFS, rules, workers), roots) // A cancelled walk stops early, so its size census covers only part // of the roots, and every file it never reached would look vanished @@ -403,7 +435,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, return err } - return s.contentPhase(ctx, workers) + return s.contentPhase(ctx, workers, rules) } // walkableRoots returns the operands the walk can start from: regular @@ -411,9 +443,11 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, // about, counted as skipped, and dropped. A dropped operand is no // longer a root, so the records stored beneath it are treated as if it // had not been given: those under no other root count as outside the -// roots and are not deleted as unverified. An operand that fails lstat -// here is kept, and the walk warns about it. -func (s *scanState) walkableRoots(roots []string) []string { +// roots and are not deleted as unverified. An operand on a filesystem +// in rules that is not persistent is dropped the same way, with a line +// saying so that is not a warning, and is not counted as skipped. An +// operand that fails lstat here is kept, and the walk warns about it. +func (s *scanState) walkableRoots(roots []string, rules walkRules) []string { kept := make([]string, 0, len(roots)) for _, root := range roots { @@ -427,6 +461,13 @@ func (s *scanState) walkableRoots(roots []string) []string { continue } + + line := rules.nonPersistentLine(root, fi) + if line != "" { + fmt.Fprintln(os.Stderr, escapePath(line)) + + continue + } } kept = append(kept, root) @@ -481,10 +522,12 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // (the content phase may still fill in its content hash). It returns // the new-or-changed files and the unchanged files whose records lack // hashes; both remain candidates until the census decides whether -// their sizes are shared. A record below a directory -x did not enter -// counts as outside the roots: it is not deleted, and its size joins -// the census. That does not hold below a root at or inside that -// directory, since that root's own walk verifies the records below it. +// their sizes are shared. A record at or below a path the walk left +// out, because -x stopped at it, it is a default exclude, or it is not +// on a persistent filesystem, counts as outside the roots: it is not +// deleted, and its size joins the census. That does not hold below a +// root at or inside that path, since that root's own walk verifies the +// records below it; nor is a line printed for such a root. func (s *scanState) walkPhase( events <-chan walkEvent, roots []string, ) ([]fileRec, []fileRec) { @@ -502,12 +545,25 @@ func (s *scanState) walkPhase( var ( changed, unhashed []fileRec - otherFSDirs []string + leftOut []string ) for ev := range events { if ev.otherFSDir != "" { - otherFSDirs = append(otherFSDirs, ev.otherFSDir) + leftOut = append(leftOut, ev.otherFSDir) + + continue + } + + // The line saying so is printed as a warning is, but what the + // walk leaves out is not counted as skipped. A default exclude + // that is itself a root is walked as one, so it gets no line. + if ev.leftOut != "" { + leftOut = append(leftOut, ev.leftOut) + + if !slices.Contains(roots, ev.leftOut) { + prog.warnf("%s", ev.warn) + } continue } @@ -542,12 +598,13 @@ func (s *scanState) walkPhase( prog.finish() for path, old := range s.existing { - // Kept only when a directory -x did not enter lies below the - // deepest root above the record. A root at or below that - // directory was walked on its own and did not find the file. - // Both contain the record, so the longer path is the deeper. - otherFSDir := deepestRoot(path, otherFSDirs) - if len(otherFSDir) > len(deepestRoot(path, roots)) { + // Kept only when a path the walk left out lies below the deepest + // root above the record, or is the record's own. A root at or + // below that path was walked on its own and did not find the + // file. Both contain the record, so the longer path is the + // deeper. + left := deepestRoot(path, leftOut) + if len(left) > len(deepestRoot(path, roots)) { delete(s.existing, path) s.sizes = append(s.sizes, old.size) @@ -821,9 +878,12 @@ func (s *scanState) updatePhase(ctx context.Context) error { // hash phase's worker pool and their records written back in batches. A // failed read is warned about and counted as skipped; the record keeps // its empty content, so it is never grouped, and a later scan tries -// again. -func (s *scanState) contentPhase(ctx context.Context, workers int) error { - toRead, recs, err := s.contentCandidates(ctx) +// again. No file on a filesystem in rules that is not persistent is +// read. +func (s *scanState) contentPhase(ctx context.Context, workers int, + rules walkRules, +) error { + toRead, recs, err := s.contentCandidates(ctx, rules) if err != nil { return err } @@ -858,18 +918,19 @@ func (s *scanState) contentPhase(ctx context.Context, workers int) error { // contentCandidates returns the files the content phase reads, and // their records by path. Every record contentCandidatesSQL returns has // its file checked with lstat, whether or not it already has a content -// hash: a file that is gone, is no longer a regular file, or has -// changed by the walk's rule keeps its record as it is and does not -// count as a match for the others, and any other lstat error is warned -// about and counted as skipped, with the same result. If such a record -// has no content hash, it stays out of duplicate groups; if it has one, -// it is still reported until a scan covering its own tree updates or -// removes it. The files of a group that pass and have no content hash -// are read only if at least minGroupSize of the group's files pass, so -// a group whose other members are all stale costs no reads. Only the -// records to be read are kept. +// hash: a file that is gone, is no longer a regular file, has changed +// by the walk's rule, or is on a filesystem in rules that is not +// persistent keeps its record as it is and does not count as a match +// for the others, and any other lstat error is warned about and counted +// as skipped, with the same result. If such a record has no content +// hash, it stays out of duplicate groups; if it has one, it is still +// reported until a scan covering its own tree updates or removes it. +// The files of a group that pass and have no content hash are read only +// if at least minGroupSize of the group's files pass, so a group whose +// other members are all stale costs no reads. Only the records to be +// read are kept. func (s *scanState) contentCandidates( - ctx context.Context, + ctx context.Context, rules walkRules, ) ([]fileRec, map[string]scanRec, error) { // The search and the checks take real time on a large database; // without a display the scan looks hung before the reads begin. @@ -927,7 +988,7 @@ func (s *scanState) contentCandidates( first = r } - f, ok, err := unchangedFile(r) + f, ok, err := unchangedFile(r, rules) if err != nil { s.st.skipped++ @@ -961,9 +1022,10 @@ func (s *scanState) contentCandidates( // unchangedFile lstats the file r names and returns it for reading if // it is still the regular file r records: the same size, and an mtime -// no newer than recorded (the walk's change rule). A file that is gone -// or has changed reports false; any other lstat error is returned. -func unchangedFile(r scanRec) (fileRec, bool, error) { +// no newer than recorded (the walk's change rule). A file that is gone, +// has changed, or is on a filesystem in rules that is not persistent +// reports false; any other lstat error is returned. +func unchangedFile(r scanRec, rules walkRules) (fileRec, bool, error) { fi, err := os.Lstat(r.path) if errors.Is(err, fs.ErrNotExist) { return fileRec{}, false, nil @@ -980,6 +1042,12 @@ func unchangedFile(r scanRec) (fileRec, bool, error) { dev, ino := inodeOfInfo(fi) + // The scan reads nothing on a filesystem that is not persistent, + // whatever record an earlier scan left for the file. + if _, found := rules.nonPersistent[dev]; found { + return fileRec{}, false, nil + } + return fileRec{ path: r.path, size: r.size, mtime: r.mtime, dev: dev, ino: ino, }, true, nil @@ -1046,27 +1114,31 @@ type dirJob struct { } // walkEvent is one walk result delivered to the walk phase: a regular -// file's statted record, a warning when fail is set, or, when -// otherFSDir is set, a directory -x did not enter because it is on -// another filesystem than its operand. +// file's statted record, a warning when fail is set, or a path the walk +// left out: when otherFSDir is set, a directory -x did not enter +// because it is on another filesystem than its operand, and when +// leftOut is set, a default exclude or a directory or file on a +// filesystem that is not persistent, with warn holding the line that +// says so. type walkEvent struct { rec fileRec warn string fail bool otherFSDir string + leftOut string } // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning -// event per per-path error, and one event per directory -x did not -// enter. The channel is closed when the walk completes, and also when -// ctx is cancelled — every goroutine in the pool abandons its blocking -// send in that case, so the consumer sees a truncated but properly -// terminated stream instead of a stalled one. +// event per per-path error, and one event per path -x or rules kept the +// walk from entering or recording. The channel is closed when the walk +// completes, and also when ctx is cancelled — every goroutine in the +// pool abandons its blocking send in that case, so the consumer sees a +// truncated but properly terminated stream instead of a stalled one. func startWalk(ctx context.Context, roots []string, oneFS bool, - workers int, + rules walkRules, workers int, ) <-chan walkEvent { - jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS) + jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, rules) go func() { initial := make([]dirJob, 0, len(roots)) @@ -1177,7 +1249,7 @@ func seedRoot(ctx context.Context, root string, // workers drop the directories still queued rather than stop reading // jobs, so the range always runs out and the pool always tears down. func startWalkWorkers(ctx context.Context, workers int, - oneFS bool, + oneFS bool, rules walkRules, ) (chan dirJob, chan []dirJob, chan walkEvent) { jobs := make(chan dirJob, workQueueDepth) subdirs := make(chan []dirJob, workers) @@ -1193,7 +1265,7 @@ func startWalkWorkers(ctx context.Context, workers int, } select { - case subdirs <- walkOneDir(ctx, job, oneFS, events): + case subdirs <- walkOneDir(ctx, job, oneFS, rules, events): case <-ctx.Done(): } } @@ -1251,7 +1323,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob, // entry and a warning event per unreadable one, and returns the // subdirectories to descend into. func walkOneDir(ctx context.Context, job dirJob, oneFS bool, - events chan<- walkEvent, + rules walkRules, events chan<- walkEvent, ) []dirJob { entries, err := os.ReadDir(job.path) if err != nil { @@ -1275,7 +1347,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, p := filepath.Join(job.path, e.Name()) if e.IsDir() { - if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok { + sub, ok := subdirJob(ctx, p, e, job, oneFS, rules, events) + if ok { subs = append(subs, sub) } @@ -1287,7 +1360,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, continue } - emitFile(ctx, p, e, events) + emitFile(ctx, p, e, rules, events) } return subs @@ -1297,9 +1370,11 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, // The lstat happens here in the walk worker, while the directory's // metadata is still hot; a path that fails to stat (or stops being a // regular file) between the directory read and the lstat is warned -// about and skipped. +// about and skipped. A file on a filesystem in rules that is not +// persistent, as through a bind mount of one file, is not recorded: it +// gets an event naming it instead. func emitFile(ctx context.Context, p string, e fs.DirEntry, - events chan<- walkEvent, + rules walkRules, events chan<- walkEvent, ) { info, err := e.Info() if err != nil { @@ -1315,6 +1390,13 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, return } + line := rules.nonPersistentLine(p, info) + if line != "" { + sendEvent(ctx, events, walkEvent{leftOut: p, warn: line}) + + return + } + dev, ino := inodeOfInfo(info) sendEvent(ctx, events, walkEvent{rec: fileRec{ @@ -1328,22 +1410,20 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // subdirJob applies the descent rules to directory p: never enter // .zfs (ZFS snapshot pseudo-dirs would list every file once per -// snapshot), and with -x never enter a directory on a different -// filesystem than its operand, sending an event that names it so the -// scan keeps the records below it, except those below an operand at or +// snapshot), never enter a directory on a filesystem in rules that is +// not persistent, nor a default exclude in rules, and with -x never +// enter a directory on a different filesystem than its operand. For the +// last three it sends an event that names the directory, so the scan +// keeps the records below it, except those below an operand at or // inside it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, - parent dirJob, oneFS bool, events chan<- walkEvent, + parent dirJob, oneFS bool, rules walkRules, + events chan<- walkEvent, ) (dirJob, bool) { if e.Name() == ".zfs" { return dirJob{}, false } - job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK} - if !oneFS || !parent.rootDevOK { - return job, true - } - info, err := e.Info() if err != nil { sendEvent(ctx, events, walkEvent{ @@ -1354,6 +1434,27 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry, return dirJob{}, false } + line := rules.nonPersistentLine(p, info) + if line != "" { + sendEvent(ctx, events, walkEvent{leftOut: p, warn: line}) + + return dirJob{}, false + } + + if isDefaultExclude(p, rules.excludes) { + sendEvent(ctx, events, walkEvent{ + leftOut: p, + warn: fmt.Sprintf("scan: skipped %s (default exclude)", p), + }) + + return dirJob{}, false + } + + job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK} + if !oneFS || !parent.rootDevOK { + return job, true + } + if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev { sendEvent(ctx, events, walkEvent{otherFSDir: p}) diff --git a/scan_test.go b/scan_test.go index 1cfe8f2..3f45bae 100644 --- a/scan_test.go +++ b/scan_test.go @@ -782,7 +782,8 @@ func collectWalk(t *testing.T, roots []string, oneFS bool, errs int ) - for ev := range startWalk(t.Context(), roots, oneFS, workers) { + for ev := range startWalk(t.Context(), roots, oneFS, walkRules{}, + workers) { if ev.fail { errs++ @@ -1004,7 +1005,8 @@ func callSubdirJob(t *testing.T, p string, e fs.DirEntry, t.Helper() events := make(chan walkEvent, 1) - job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events) + job, ok := subdirJob(t.Context(), p, e, parent, oneFS, walkRules{}, + events) close(events) var evs []walkEvent @@ -1152,7 +1154,7 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, return dev, ok } - roots = pruneRoots(roots, true, device) + roots = pruneRoots(roots, true, walkRules{}, device) ctx := t.Context() s := &scanState{db: db} @@ -1162,7 +1164,7 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, t.Fatal(err) } - jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + jobs, subdirs, events := startWalkWorkers(ctx, 2, true, walkRules{}) // seedRoot emits a regular-file operand itself and returns a job // only for a directory operand. @@ -1463,7 +1465,7 @@ const smokeTreeFiles = 15 func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { t.Helper() - st, err := syncScan(t.Context(), db, roots, 4, false) + st, err := syncScan(t.Context(), db, roots, 4, false, walkRules{}) if err != nil { t.Fatalf("syncScan: %v", err) } @@ -2412,7 +2414,7 @@ func TestPruneRoots(t *testing.T) { // Duplicates and operands under other operands are dropped; /cc is // not under /c (sibling with a shared prefix). got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, - false, device) + false, walkRules{}, device) want := []string{"/a", "/c", "/cc"} if !slices.Equal(got, want) { @@ -2421,7 +2423,7 @@ func TestPruneRoots(t *testing.T) { nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"} - got = pruneRoots(nested, false, device) + got = pruneRoots(nested, false, walkRules{}, device) want = []string{"/p"} if !slices.Equal(got, want) { @@ -2429,12 +2431,31 @@ func TestPruneRoots(t *testing.T) { } // With -x, /p/q, on another filesystem than /p, is kept as well. - got = pruneRoots(nested, true, device) + got = pruneRoots(nested, true, walkRules{}, device) want = []string{"/p", pqRoot} if !slices.Equal(got, want) { t.Fatalf("pruneRoots with -x = %q, want %q", got, want) } + + // The walk of /p does not enter /p/q when it is a default exclude, so + // /p/q is kept, without -x too; /p/q/r lies under it and is dropped. + got = pruneRoots(nested, false, walkRules{excludes: []string{pqRoot}}, + device) + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want) + } + + // Nor does it enter /p/q when the filesystem of /p/q is not + // persistent, so /p/q/r/m, of no known device, is kept. + got = pruneRoots([]string{"/p", "/p/q/r/m"}, false, + walkRules{nonPersistent: map[uint64]string{2: "tmpfs"}}, device) + + want = []string{"/p", "/p/q/r/m"} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots with %s not persistent = %q, want %q", + pqRoot, got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) { -- 2.54.0