From 18aa3a383de12bc3adf213e317a25429877d3a51 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 16 Oct 2026 23:28:07 +0000 Subject: [PATCH] scan: skip OS default excludes, and record nothing on filesystems that are not persistent (closes #124) scan never records anything on a filesystem that is not persistent (tmpfs, proc, sysfs and the other kernel or memory-backed types). As it starts it notes the device ID of each such mount point, from /proc/self/mountinfo on Linux and the mounted-filesystem list on macOS, and checks every operand, directory and file the walk meets against them; no flag turns this off. Below an operand the walk also skips the default excludes, a path list per system (Linux, macOS, and Windows, which is not used yet), and --no-default-excludes turns off only those. A default exclude that is itself an operand gets no skip line. The tests stop at once, saying why, when TMPDIR is not on a persistent filesystem. Judgement call: an unreadable mount list fails the scan, and no flag gets round it. Judgement call: a file on such a filesystem below an operand gets the same line as a directory. Judgement call: an operand on such a filesystem that another operand's walk also reaches is named twice. Unverified: the macOS mount list is built for macOS but never run; bind mounts need root to test. Model: opus-5-5 --- README.md | 109 ++++++++-- TODO.md | 16 ++ cancel_test.go | 25 ++- exclude.go | 209 ++++++++++++++++++ exclude_darwin.go | 32 +++ exclude_other.go | 36 ++++ exclude_test.go | 530 ++++++++++++++++++++++++++++++++++++++++++++++ main.go | 25 ++- main_test.go | 33 ++- scan.go | 270 +++++++++++++++-------- scan_test.go | 37 +++- 11 files changed, 1187 insertions(+), 135 deletions(-) create mode 100644 exclude.go create mode 100644 exclude_darwin.go create mode 100644 exclude_other.go create mode 100644 exclude_test.go diff --git a/README.md b/README.md index 4273341..1e0960e 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,11 @@ As a file `/etc/cron.d/sfdupes`: `/var/log/sfdupes.log`, holds a first line naming the database (see "Database"), plain progress lines naming each phase and its place among the five, as `[2/5] walk` (one as the phase starts, one every 5 seconds while it - runs, and one as it ends), a warning for each path it skips, and the summary - line (see "Progress" and "`scan` mode"). The log grows with every scan; rotate - it like any other. + runs, and one as it ends), a warning for each path it skips, a line for each + default exclude its walk does not enter and for each path it leaves out + because it is not on a persistent filesystem (see "Rules for the walk"), and + the summary line (see "Progress" and "`scan` mode"). The log grows with every + scan; rotate it like any other. - Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the reason among the last lines of the log; `tail` prints them, and cron mails @@ -248,7 +250,7 @@ Four subcommands, all implemented: schema version and the counts of its records. ``` -sfdupes scan [--workers N] [-x] PATH... +sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv sfdupes info @@ -257,8 +259,9 @@ sfdupes [command] --help ``` `--workers N` sets the size of each `scan` worker pool (default: the number of -CPUs), and `-x` (`--one-file-system`) keeps the walk of each operand on that -operand's filesystem; both are described under "`scan` mode". +CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that +operand's filesystem, and `--no-default-excludes` lets the walk enter the +default excludes, a list of paths; all three are described under "`scan` mode". `sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a subcommand, prints the help text to stderr and exits 0, writing nothing to @@ -439,11 +442,12 @@ results do not depend on the working directory. All operands belong to a single scan and are enumerated concurrently: every operand seeds the shared walk worker pool. Overlapping operands are harmless — an operand that duplicates another or lies under another is dropped before walking, so every file is reached exactly -once and produces one database record. With `-x`, though, an operand under -another one is kept and walked on its own when any directory from it (from its -directory, for a regular file) up to but not including the operand covering it -is on a different filesystem than that operand, because that operand's walk does -not enter such a directory and so does not reach it. +once and produces one database record. An operand under another one is kept, +though, and walked on its own when any directory from it (from its directory, +for a regular file) up to but not including the operand covering it is a default +exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with +`-x`, is on a different filesystem than that operand, because that operand's +walk does not enter such a directory and so does not reach it. An operand that is a symlink (never followed, not even as an operand), socket, FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints @@ -453,7 +457,10 @@ beneath it is still scanned. The records stored beneath it are not deleted: they are treated like any other record outside the scanned operands, including the content-phase exception below. If it lies under another operand, they are under that operand instead, and are treated like any other record there. This is not -an error: a scan whose every operand is dropped walks nothing and exits 0. +an error: a scan whose every operand is dropped walks nothing and exits 0. An +operand that is not on a persistent filesystem (see "Rules for the walk") is +dropped the same way, except that the line `scan` prints for it is not a +warning, and it is not counted as skipped. `scan` synchronizes the database with the filesystem state under the scanned operands: @@ -491,12 +498,13 @@ operands: a `content` hash is read for one, wherever it lies, once its size, `head`, and `tail` match another record's. If that file is gone or has changed since its record was written, the record is left as it is. -- With `-x`, a record below a directory the walk did not enter because that - directory is on another filesystem than its operand counts as outside the - scanned operands: it is not deleted, and its size joins the size census. A - filesystem mounted under an operand can therefore be scanned on its own - schedule into the same database. Records below an operand at or inside such a - directory are not kept this way, since that operand's own walk verifies them. +- A record at or below a path the walk left out, because that path is a default + exclude, is not on a persistent filesystem, or, with `-x`, is a directory on + another filesystem than its operand, counts as outside the scanned operands: + it is not deleted, and its size joins the size census. A filesystem mounted + under an operand can therefore be scanned on its own schedule into the same + database. Records below an operand at or inside such a path are not kept this + way, since that operand's own walk verifies them. `scan` runs **five sequential phases over the whole scan**. Parallelism lives inside the walk, hash and content phases; batched database writes begin during @@ -572,6 +580,71 @@ Rules for the walk: unless they lie below an operand at or inside that directory, count as outside the scanned operands, as described in "`scan` mode" above: they are not deleted, and their sizes join the size census. +- Never record anything that is not on a persistent filesystem. A filesystem is + not persistent when the kernel makes it up or keeps it in memory: one of type + `proc`, `sysfs`, `devtmpfs`, `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`, + `debugfs`, `tracefs`, `securityfs`, `pstore`, `bpf`, `configfs`, `fusectl`, + `mqueue`, `hugetlbfs`, `efivarfs`, `binfmt_misc`, `autofs`, `nsfs`, + `rpc_pipefs`, `selinuxfs` or `devfs` (the macOS filesystem of `/dev`). After + it names the database and before it takes the lock, `scan` lists the mounted + filesystems, from `/proc/self/mountinfo` on Linux and from the system's list + of mounted filesystems on macOS (on any other system it lists none), and notes + the device ID of the mount point of each one of those types, read with + `lstat`. Where one filesystem is mounted over another, the one mounted last + counts. If `scan` cannot list them, it fails (exit 1). The walk then never + enters a directory, and never records a file, on one of those devices, however + it reaches it: as an operand, below one, or through a bind mount, which keeps + the device ID of what it binds. No flag turns this off. For each operand, + directory or file it leaves out this way, `scan` prints a line on stderr + naming it and the type of its filesystem, as it prints a warning, so it never + lands inside a progress display: + + ``` + scan: skipped /dev/shm (tmpfs, not a persistent filesystem) + ``` + + That line is not a warning, and the path is not counted as skipped. Records + that an earlier scan left at or below it, unless they lie below an operand + at or inside it, count as outside the scanned operands, as described in + "`scan` mode" above: they are not deleted, and their sizes join the size + census. + +- Below an operand, never descend into a default exclude: a path that holds the + kernel's view of the running system, or temporary files, so that + `sfdupes scan /` walks only the files worth comparing. The default excludes + depend on the system, and `sfdupes scan --help` lists those of the system it + runs on: + - On Linux: `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`. + - On macOS: `/dev`, `/private/tmp`, `/private/var/tmp`, `/private/var/vm`, + `/System/Volumes/VM` and `/System/Volumes/Data`. A walk of `/` reaches + everything in `/System/Volumes/Data` through the firmlinks, so walking it + as well would record each of those files twice. + - On Windows, relative to the root of the drive an operand lies on: the + page, swap and hibernation files `pagefile.sys`, `swapfile.sys` and + `hiberfil.sys`, and the directories `System Volume Information`, + `$Recycle.Bin`, `Windows\Temp` and `Users\*\AppData\Local\Temp`, where `*` + matches one path component, as it does in any default exclude. sfdupes + does not build for Windows yet, so this list is not used. + - On any other system there are none. + + `--no-default-excludes` turns them all off; what is not on a persistent + filesystem stays left out. An operand at or inside a default exclude is + still walked, so `sfdupes scan /tmp/x` works, unless it is not on a + persistent filesystem. For each default exclude its walk does not enter, + `scan` prints a line on stderr naming it, as it prints a warning: + + ``` + scan: skipped /var/tmp (default exclude) + ``` + + That line is not a warning, and the directory is not counted as skipped. A + default exclude that is not on a persistent filesystem, as `/proc` is, gets + the line for its filesystem instead, and one that is itself an operand gets + no line, since it is walked as that operand. The records below a default + exclude the walk does not enter, unless they lie below an operand at or + inside it, count as outside the scanned operands, as described in "`scan` + mode" above: they are not deleted, and their sizes join the size census. + - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were diff --git a/TODO.md b/TODO.md index b814411..f2dc756 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,22 @@ # Completed Steps +- `scan` never records anything that is not on a persistent filesystem (`tmpfs`, + `proc`, `sysfs` and the other kernel or memory-backed types), matched by the + device ID of each such mount point, listed as the scan starts from + `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS, + however the walk reaches it; no flag turns that off. Below an operand, its + walk does not enter the default excludes either, so `sfdupes scan /` skips the + kernel's view of the running system and the temporary directories: on Linux + `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`, on macOS `/dev`, + `/private/tmp`, `/private/var/tmp`, `/private/var/vm`, `/System/Volumes/VM` + and `/System/Volumes/Data`, and a Windows list, not used yet. An operand at or + inside a default exclude is still walked. The scan prints a line for each path + it leaves out, without counting it as skipped, and the records there count as + outside the scanned operands, as for `-x`. `--no-default-excludes` turns off + the default excludes only, and `scan --help` lists them (2026-10-17, + https://git.eeqj.de/sneak/sfdupes/issues/124) + - the rate a progress display shows covers the items completed over the last 5 seconds, or since the line before when stderr is not a terminal, so it falls to 0 while nothing completes, and a burst after a wait no longer reads as a diff --git a/cancel_test.go b/cancel_test.go index da543fd..20e72e2 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -196,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { base := baselineGoroutines(t) st, err := syncScan(newWalkClock(walkCancelAtDone), db, - []string{dir}, walkCancelWorkers, false) + []string{dir}, walkCancelWorkers, false, walkRules{}) assertWalkGuardAborted(t, st, err) assertRecordsIntact(t, db, before) @@ -268,7 +268,8 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false) + st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false, + walkRules{}) if !errors.Is(err, context.Canceled) { t.Fatalf("syncScan on a cancelled context = %v, want %v", err, context.Canceled) @@ -301,7 +302,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) { db := openTestDB(t) st, err := syncScan(newWalkClock(hashCancelAtDone), db, - []string{dir}, walkCancelWorkers, false) + []string{dir}, walkCancelWorkers, false, walkRules{}) if !errors.Is(err, context.Canceled) { t.Fatalf("syncScan cancelled mid-hash = %v, want %v", err, context.Canceled) @@ -356,7 +357,8 @@ func TestRunScanInterrupted(t *testing.T) { stderr := captureStderr(t) dir := buildSmokeTree(t) - err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false, + false) if err != nil { t.Fatal(err) } @@ -380,7 +382,7 @@ func TestRunScanInterrupted(t *testing.T) { defer close(done) err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers, - false) + false, false) }() awaitReturn(t, done, "runScan") @@ -407,7 +409,8 @@ func TestRunScanInterrupted(t *testing.T) { got, before) } - err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false, + false) if err != nil { t.Fatal(err) } @@ -433,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) { dir := buildWalkCancelTree(t) err := runScan(newWalkClock(hashCancelAtDone), []string{dir}, - walkCancelWorkers, false) + walkCancelWorkers, false, false) if !errors.Is(err, errInterrupted) { t.Fatalf("runScan interrupted mid-hash = %v, want %v", err, errInterrupted) @@ -617,7 +620,8 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) { // Unbuffered and unread: on a cancelled scan every send gives up. events := make(chan walkEvent) - subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events) + subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, + walkRules{}, events) if len(subs) != 0 { t.Errorf("cancelled walkOneDir returned %+v to descend into, "+ "want none", subs) @@ -637,7 +641,8 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) { missing := filepath.Join(t.TempDir(), "missing") - jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false) + jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false, + walkRules{}) for range 64 { jobs <- dirJob{path: missing} @@ -668,7 +673,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - jobs, subdirs, events := startWalkWorkers(ctx, 1, false) + jobs, subdirs, events := startWalkWorkers(ctx, 1, false, walkRules{}) // Fill the hand-back channel to its capacity — one slot per worker // — so the worker's own hand-back is certain to block. diff --git a/exclude.go b/exclude.go new file mode 100644 index 0000000..ff590de --- /dev/null +++ b/exclude.go @@ -0,0 +1,209 @@ +package main + +// This file holds two rules for the walk (README "Rules for the walk"): +// the default excludes, the paths a scan's walk does not enter below an +// operand unless scan is given --no-default-excludes, and the +// filesystems that are not persistent, on which the walk never records +// anything. + +import ( + "bufio" + "errors" + "fmt" + "io" + "io/fs" + "path/filepath" + "slices" + "strings" +) + +// The systems that have default excludes, as runtime.GOOS names them. +const ( + goosLinux = "linux" + goosDarwin = "darwin" + goosWindows = "windows" +) + +// The fields of a line of mountInfoPath that readMountInfo reads, +// counted from 0 (see proc(5)): the mount point is the fifth field, and +// after the sixth come optional fields, then a field holding "-", then +// the filesystem type. +const ( + mountPointField = 4 + optionalFieldsStart = 6 +) + +// typesPerHelpLine is how many filesystem types the scan command's help +// lists on each line. +const typesPerHelpLine = 6 + +// errMountInfoLine reports a line of mountInfoPath without the fields +// readMountInfo reads. +var errMountInfoLine = errors.New("unexpected line") + +// walkRules holds the two rules for the walk that a scan sets up as it +// starts. excludes are the default excludes, or none with +// --no-default-excludes. nonPersistent maps the device ID of every +// mounted filesystem that is not persistent to its type; the walk +// records nothing on one of those, whatever the flags. +type walkRules struct { + excludes []string + nonPersistent map[uint64]string +} + +// defaultExcludes returns the default excludes of the system goos +// names, as runtime.GOOS does: a fixed list on Linux, macOS and +// Windows, and none on any other system. +func defaultExcludes(goos string) []string { + switch goos { + case goosLinux: + return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"} + case goosDarwin: + // A walk of / reaches everything in /System/Volumes/Data through + // the firmlinks, so walking it as well would record each of + // those files twice. + return []string{ + "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", + "/System/Volumes/VM", "/System/Volumes/Data", + } + case goosWindows: + // Relative to the root of the drive an operand lies on. The + // first three are the page, swap and hibernation files. + return []string{ + "pagefile.sys", "swapfile.sys", "hiberfil.sys", + "System Volume Information", "$Recycle.Bin", `Windows\Temp`, + `Users\*\AppData\Local\Temp`, + } + default: + return nil + } +} + +// isDefaultExclude reports whether path is one of excludes, in which a +// * matches one path component. +func isDefaultExclude(path string, excludes []string) bool { + for _, pattern := range excludes { + // The patterns are fixed, so none is malformed. + if ok, _ := filepath.Match(pattern, path); ok { + return true + } + } + + return false +} + +// nonPersistentFilesystemTypes returns the filesystem types that are +// not persistent: filesystems that the kernel makes up or that are kept +// in memory. devfs is the macOS filesystem of /dev; the others are +// Linux names. +func nonPersistentFilesystemTypes() []string { + return []string{ + "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs", + "cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore", + "bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs", + "binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", "devfs", + } +} + +// nonPersistentFilesystems returns the device ID of every mounted +// filesystem that is not persistent, with its type, read with lstat on +// its mount point. mountedFilesystems lists the mounts. A mount point +// that lstat cannot reach is left out. +func nonPersistentFilesystems() (map[uint64]string, error) { + mounts, err := mountedFilesystems() + if err != nil { + return nil, err + } + + types := nonPersistentFilesystemTypes() + devices := make(map[uint64]string) + + for mountPoint, fsType := range mounts { + if !slices.Contains(types, fsType) { + continue + } + + if dev, ok := deviceOfPath(mountPoint); ok { + devices[dev] = fsType + } + } + + return devices, nil +} + +// nonPersistentLine returns the line the scan prints for path when fi, +// its lstat, puts it on a filesystem in rules.nonPersistent, and "" +// when it is on any other. +func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string { + dev, ok := deviceOfInfo(fi) + if !ok { + return "" + } + + fsType, found := rules.nonPersistent[dev] + if !found { + return "" + } + + return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)", + path, fsType) +} + +// readMountInfo reads mountinfo, laid out as mountInfoPath is, and +// returns every mount point with the type of its filesystem. Where one +// filesystem is mounted over another, a later line names the one on +// top, so the last line for a mount point decides. +func readMountInfo(mountinfo io.Reader) (map[string]string, error) { + // The kernel writes a space, tab, newline or backslash in a mount + // point as an octal escape. + unescape := strings.NewReplacer( + `\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + + mounts := make(map[string]string) + lines := bufio.NewScanner(mountinfo) + + for lines.Scan() { + fields := strings.Fields(lines.Text()) + + sep := slices.Index(fields, "-") + if sep < optionalFieldsStart || sep+1 >= len(fields) { + return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text()) + } + + mounts[unescape.Replace(fields[mountPointField])] = fields[sep+1] + } + + return mounts, lines.Err() +} + +// defaultExcludesHelp describes, for the scan command's help, the +// default excludes of the system goos names and, where the scan lists +// the mounted filesystems, the types it never records anything on. +func defaultExcludesHelp(goos string) string { + var help strings.Builder + + dirs := defaultExcludes(goos) + if len(dirs) == 0 { + help.WriteString("This system has no default excludes.\n") + } else { + help.WriteString("Below an operand, the walk does not enter these " + + "default excludes, unless\n--no-default-excludes is given:\n\n") + + for _, dir := range dirs { + fmt.Fprintf(&help, " %s\n", dir) + } + } + + if goos == goosLinux || goos == goosDarwin { + help.WriteString("\nWith or without --no-default-excludes, the scan " + + "records nothing on a\nfilesystem that is not persistent, " + + "one of these types:\n\n") + + types := nonPersistentFilesystemTypes() + for line := range slices.Chunk(types, typesPerHelpLine) { + fmt.Fprintf(&help, " %s\n", strings.Join(line, ", ")) + } + } + + return help.String() +} diff --git a/exclude_darwin.go b/exclude_darwin.go new file mode 100644 index 0000000..7094de9 --- /dev/null +++ b/exclude_darwin.go @@ -0,0 +1,32 @@ +//go:build darwin + +package main + +import "golang.org/x/sys/unix" + +// mountedFilesystems returns every mount point with the type of its +// filesystem, from the system's list of mounted filesystems. Where one +// filesystem is mounted over another, the later one in the list counts. +func mountedFilesystems() (map[string]string, error) { + // The first call counts the filesystems, the second lists them. + n, err := unix.Getfsstat(nil, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + + list := make([]unix.Statfs_t, n) + + n, err = unix.Getfsstat(list, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + + mounts := make(map[string]string, n) + + for _, st := range list[:n] { + mounts[unix.ByteSliceToString(st.Mntonname[:])] = + unix.ByteSliceToString(st.Fstypename[:]) + } + + return mounts, nil +} diff --git a/exclude_other.go b/exclude_other.go new file mode 100644 index 0000000..ae2d98c --- /dev/null +++ b/exclude_other.go @@ -0,0 +1,36 @@ +//go:build !darwin + +package main + +import ( + "fmt" + "os" + "runtime" +) + +// mountInfoPath is the file in which Linux lists the mounts the reading +// process sees. +const mountInfoPath = "/proc/self/mountinfo" + +// mountedFilesystems returns every mount point with the type of its +// filesystem: on Linux read from mountInfoPath (see readMountInfo), and +// on any other system none. +func mountedFilesystems() (map[string]string, error) { + if runtime.GOOS != goosLinux { + return map[string]string{}, nil + } + + f, err := os.Open(mountInfoPath) + if err != nil { + return nil, err + } + + defer func() { _ = f.Close() }() + + mounts, err := readMountInfo(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", mountInfoPath, err) + } + + return mounts, nil +} diff --git a/exclude_test.go b/exclude_test.go new file mode 100644 index 0000000..ce61c4a --- /dev/null +++ b/exclude_test.go @@ -0,0 +1,530 @@ +package main + +import ( + "bytes" + "errors" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" +) + +func TestDefaultExcludes(t *testing.T) { + t.Parallel() + + // README "Rules for the walk": each system's list. + cases := map[string][]string{ + goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, + goosDarwin: { + "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", + "/System/Volumes/VM", "/System/Volumes/Data", + }, + goosWindows: { + "pagefile.sys", "swapfile.sys", "hiberfil.sys", + "System Volume Information", "$Recycle.Bin", `Windows\Temp`, + `Users\*\AppData\Local\Temp`, + }, + "freebsd": nil, + } + + for goos, want := range cases { + if got := defaultExcludes(goos); !slices.Equal(got, want) { + t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want) + } + } +} + +func TestIsDefaultExclude(t *testing.T) { + t.Parallel() + + // A * matches one path component, no more and no fewer. + excludes := []string{"/scratch", "/home/*/cache"} + + cases := map[string]bool{ + "/scratch": true, + "/scratch/x": false, + "/home/u/cache": true, + "/home/u/v/cache": false, + "/home/cache": false, + } + + for path, want := range cases { + if got := isDefaultExclude(path, excludes); got != want { + t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want) + } + } +} + +func TestReadMountInfo(t *testing.T) { + t.Parallel() + + // The second-to-last line has no optional fields. /mnt/over has an + // ext4 filesystem mounted over its tmpfs. + mountinfo := strings.Join([]string{ + "22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw", + "23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw", + "24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw", + `25 22 0:23 / /mnt/ram\040disk rw shared:2 - tmpfs none rw`, + "26 22 0:24 / /mnt/over rw shared:3 - tmpfs none rw", + "27 26 8:2 / /mnt/over rw - ext4 /dev/sda2 rw", + "28 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw", + }, "\n") + + got, err := readMountInfo(strings.NewReader(mountinfo)) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "/": "ext4", "/proc": "proc", "/sys": "sysfs", + "/mnt/ram disk": "tmpfs", "/mnt/over": "ext4", "/srv/auto": "autofs", + } + if !maps.Equal(got, want) { + t.Errorf("readMountInfo = %q, want %q", got, want) + } + + _, err = readMountInfo(strings.NewReader("22 1 8:1 / / rw,relatime\n")) + if !errors.Is(err, errMountInfoLine) { + t.Errorf("readMountInfo of a line without a type = %v, want %v", + err, errMountInfoLine) + } +} + +func TestNonPersistentFilesystems(t *testing.T) { + t.Parallel() + + if runtime.GOOS != goosLinux { + t.Skip("the test knows a mount only on Linux: /proc") + } + + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfPath("/proc") + if !ok { + t.Fatal("no device for /proc") + } + + if got := nonPersistent[dev]; got != "proc" { + t.Errorf("filesystem of /proc = %q, want %q", got, "proc") + } +} + +func TestDefaultExcludesHelp(t *testing.T) { + t.Parallel() + + for _, goos := range []string{goosLinux, goosDarwin, goosWindows} { + help := defaultExcludesHelp(goos) + + for _, dir := range defaultExcludes(goos) { + if !strings.Contains(help, "\n "+dir+"\n") { + t.Errorf("help for %s does not list %s:\n%s", goos, dir, help) + } + } + + // Only Linux and macOS list their mounted filesystems. + for _, fsType := range nonPersistentFilesystemTypes() { + listed := strings.Contains(help, " "+fsType+",") || + strings.Contains(help, " "+fsType+"\n") + if listed != (goos != goosWindows) { + t.Errorf("help for %s lists %s: %v", goos, fsType, listed) + } + } + } + + want := "This system has no default excludes.\n" + if got := defaultExcludesHelp("freebsd"); got != want { + t.Errorf("help for freebsd = %q, want %q", got, want) + } + + // scan's help shows those of the system it runs on. + var stdout, stderr bytes.Buffer + + run([]string{cmdScan, "-h"}, &stdout, &stderr) + + help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS)) + if !strings.Contains(stderr.String(), help) { + t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help) + } +} + +// TestScanDoesNotEnterDefaultExclude checks that the walk does not +// enter a default exclude below an operand. It says so on a line of its +// own and does not count it as skipped. The records below it count as +// outside the operands: the record of a file gone from it is kept, and +// its size joins the size census, while a file added to it is not +// recorded. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanDoesNotEnterDefaultExclude(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + excluded := filepath.Join(dir, "tmp") + + // A scan with no default excludes records the file in excluded. + gone := writeFile(t, excluded, "gone", pattern(1, 100)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + unseen := writeFile(t, excluded, "new", pattern(2, 10)) + + // The same size as gone, so it is hashed only if the census holds the + // size of gone's record. + added := writeFile(t, dir, "added", pattern(3, 100)) + + st, err := syncScan(t.Context(), db, []string{dir}, 1, false, + walkRules{excludes: []string{excluded}}) + if err != nil { + t.Fatal(err) + } + + if st.skipped != 0 { + t.Errorf("skipped = %d, want 0", st.skipped) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, gone) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, gone) + } + + if slices.Contains(recordPaths(recs), unseen) { + t.Errorf("%s recorded: the walk entered %s", unseen, excluded) + } + + line := "\nscan: skipped " + excluded + " (default exclude)\n" + if !strings.Contains(stderr(), line) { + t.Errorf("stderr = %q, want %q", stderr(), line) + } +} + +// TestScanWalksOperandInDefaultExclude checks that an operand at or +// inside a default exclude is walked, also when it lies under another +// operand, whose walk does not enter the default exclude. +func TestScanWalksOperandInDefaultExclude(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + excluded := filepath.Join(dir, "tmp") + f := writeFile(t, excluded, "sub/f", pattern(1, 10)) + sub := filepath.Dir(f) + + cases := map[string][]string{ + "at it": {excluded}, + "inside it": {sub}, + "inside it under an operand": {dir, sub}, + } + + for name, roots := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + st, err := syncScan(t.Context(), db, roots, 1, false, + walkRules{excludes: []string{excluded}}) + if err != nil { + t.Fatal(err) + } + + if st.added != 1 { + t.Errorf("added = %d, want %s recorded once", st.added, f) + } + + recordByPath(t, dbRecords(t, db), f) + }) + } +} + +// TestScanNamesNoDefaultExcludeThatIsAnOperand checks that a default +// exclude given as an operand under another operand, which the walk of +// that other operand does not enter, gets no line saying it was +// skipped: it is walked as an operand of its own. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + excluded := filepath.Join(dir, "tmp") + f := writeFile(t, excluded, "f", pattern(1, 10)) + + st, err := syncScan(t.Context(), db, []string{dir, excluded}, 1, false, + walkRules{excludes: []string{excluded}}) + if err != nil { + t.Fatal(err) + } + + if st.added != 1 { + t.Errorf("added = %d, want %s recorded once", st.added, f) + } + + recordByPath(t, dbRecords(t, db), f) + + if strings.Contains(stderr(), "scan: skipped") { + t.Errorf("stderr = %q, want no line for %s", stderr(), excluded) + } +} + +// TestRunScanTempDirOperand scans a tree in the tests' temporary +// directory, which on Linux lies inside the default exclude /tmp unless +// TMPDIR names another place, with the system's own default excludes: +// the scan walks the whole tree and reports no default exclude. +func TestRunScanTempDirOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) { + t.Skip(dir + " is not inside a default exclude") + } + + f := writeFile(t, dir, "sub/f", pattern(1, 10)) + + stderr := scanOK(t, dir) + if strings.Contains(stderr, "scan: skipped") { + t.Errorf("stderr = %q, want no line saying a path was skipped", + stderr) + } + + assertRecorded(t, path, f, true) +} + +// TestRunScanNoDefaultExcludes scans /var through the command. Its +// /var/tmp, on a persistent filesystem, is a default exclude on Linux: +// without --no-default-excludes the walk does not enter it and says so, +// and with that flag the walk records a file the test puts there. +func TestRunScanNoDefaultExcludes(t *testing.T) { + if runtime.GOOS != goosLinux { + t.Skip("only Linux has /var/tmp among its default excludes") + } + + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + t.Fatal(err) + } + + rules := walkRules{nonPersistent: nonPersistent} + + for _, dir := range []string{"/var", "/var/tmp"} { + fi, lstatErr := os.Lstat(dir) + if lstatErr != nil || !fi.IsDir() || + rules.nonPersistentLine(dir, fi) != "" { + t.Skip(dir + " is not a directory on a persistent filesystem here") + } + } + + //nolint:usetesting // the directory must be inside /var/tmp + tmp, err := os.MkdirTemp("/var/tmp", "sfdupes-test-") + if err != nil { + t.Skip("cannot write to /var/tmp here: " + err.Error()) + } + + t.Cleanup(func() { _ = os.RemoveAll(tmp) }) + + f := writeFile(t, tmp, "f", pattern(1, 10)) + + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + line := "scan: skipped /var/tmp (default exclude)\n" + + if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) { + t.Errorf("scan /var: stderr = %q, want %q", stderr, line) + } + + assertRecorded(t, path, f, false) + + stderr := scanOK(t, "--no-default-excludes", "/var") + if strings.Contains(stderr, line) { + t.Errorf("scan --no-default-excludes /var: stderr = %q, want no %q", + stderr, line) + } + + assertRecorded(t, path, f, true) +} + +// TestRunScanSkipsNonPersistentOperand scans, through the command, a +// directory on a filesystem that is not persistent, with and without +// --no-default-excludes: either way it is skipped, with a line saying +// so, and nothing in it is recorded. +func TestRunScanSkipsNonPersistentOperand(t *testing.T) { + shm, fsType, _ := shmTempDir(t) + writeFile(t, shm, "f", pattern(1, 10)) + + t.Setenv(databaseEnv, testDBPath(t)) + + line := "scan: skipped " + shm + " (" + fsType + + ", not a persistent filesystem)\n" + summary := "scan: 0 files seen" + + for _, flags := range [][]string{nil, {"--no-default-excludes"}} { + stderr := scanOK(t, append(flags, shm)...) + if !strings.Contains(stderr, line) || + !strings.Contains(stderr, summary) { + t.Errorf("scan %q: stderr = %q, want %q and %q", flags, stderr, + line, summary) + } + } +} + +// TestScanKeepsRecordsOnNonPersistent scans a directory on a filesystem +// that is not persistent beside one on the tests' temporary directory. +// The records an earlier scan left in it count as outside the operands: +// the record of a file gone from it is kept, and its size joins the size +// census, while a file added to it is not recorded, and its size does +// not join the census. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanKeepsRecordsOnNonPersistent(t *testing.T) { + shm, fsType, rules := shmTempDir(t) + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + + // A scan that, without the list of filesystems, records gone. + gone := writeFile(t, shm, "gone", pattern(1, 100)) + syncTree(t, db, shm) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + unseen := writeFile(t, shm, "new", pattern(2, 10)) + + // added has the size of gone, and lonely that of unseen: each is + // hashed only if the census holds that size. + added := writeFile(t, dir, "added", pattern(3, 100)) + lonely := writeFile(t, dir, "lonely", pattern(4, 10)) + + st, err := syncScan(t.Context(), db, []string{shm, dir}, 1, false, rules) + if err != nil { + t.Fatal(err) + } + + if st.skipped != 0 { + t.Errorf("skipped = %d, want 0", st.skipped) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, gone) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, gone) + } + + if recordByPath(t, recs, lonely).head != "" { + t.Errorf("%s hashed: the census holds the size of %s", lonely, unseen) + } + + if slices.Contains(recordPaths(recs), unseen) { + t.Errorf("%s recorded", unseen) + } + + line := "scan: skipped " + shm + " (" + fsType + + ", not a persistent filesystem)\n" + if !strings.Contains(stderr(), line) { + t.Errorf("stderr = %q, want %q", stderr(), line) + } +} + +// TestWalkLeavesOutNonPersistent starts the walk itself in a directory +// on a filesystem that is not persistent, which a scan never does: it +// drops such an operand first. Below a directory on a persistent +// filesystem, one on another takes a mount, which a test cannot make. +// The walk enters no directory and records no file there, and names +// each. +func TestWalkLeavesOutNonPersistent(t *testing.T) { + t.Parallel() + + shm, fsType, rules := shmTempDir(t) + f := writeFile(t, shm, "f", pattern(1, 10)) + sub := filepath.Dir(writeFile(t, shm, "sub/g", pattern(1, 10))) + + var leftOut []string + + for ev := range startWalk(t.Context(), []string{shm}, false, rules, 1) { + if ev.rec.path != "" { + t.Errorf("walk recorded %s", ev.rec.path) + } + + if ev.leftOut == "" { + continue + } + + leftOut = append(leftOut, ev.leftOut) + + want := "scan: skipped " + ev.leftOut + " (" + fsType + + ", not a persistent filesystem)" + if ev.warn != want { + t.Errorf("line = %q, want %q", ev.warn, want) + } + } + + slices.Sort(leftOut) + + if want := []string{f, sub}; !slices.Equal(leftOut, want) { + t.Errorf("walk left out %q, want %q", leftOut, want) + } +} + +// shmTempDir returns a new directory in /dev/shm, removed when the test +// ends, the type of its filesystem, and the rules holding the real +// filesystems that are not persistent. It skips the test where /dev/shm +// is on a persistent filesystem, or cannot be written to. +func shmTempDir(t *testing.T) (string, string, walkRules) { + t.Helper() + + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + t.Fatal(err) + } + + dev, ok := deviceOfPath("/dev/shm") + + fsType, found := nonPersistent[dev] + if !ok || !found { + t.Skip("/dev/shm is not a memory-backed filesystem here") + } + + //nolint:usetesting // the directory must be inside /dev/shm + dir, err := os.MkdirTemp("/dev/shm", "sfdupes-test-") + if err != nil { + t.Skip("cannot write to /dev/shm here: " + err.Error()) + } + + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + return dir, fsType, walkRules{nonPersistent: nonPersistent} +} + +// assertRecorded checks that the database at dbPath holds a record for +// path when want is set, and none when it is not. +func assertRecorded(t *testing.T, dbPath, path string, want bool) { + t.Helper() + + db, err := openDB(dbPath, reportParams) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + if got := slices.Contains(recordPaths(dbRecords(t, db)), path); got != want { + t.Errorf("%s recorded: %v, want %v", path, got, want) + } +} diff --git a/main.go b/main.go index 8f33973..1c3843c 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ // // Usage: // -// sfdupes scan [--workers N] [-x] PATH... +// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv // sfdupes info @@ -182,17 +182,23 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { return root } -// newScanCommand builds the scan subcommand, with its --workers and -x -// flags. +// newScanCommand builds the scan subcommand, with its --workers, -x and +// --no-default-excludes flags. Its help lists the default excludes of +// the system it runs on, and the filesystem types it never records +// anything on. func newScanCommand() *cobra.Command { var ( - scanWorkers int - scanOneFS bool + scanWorkers int + scanOneFS bool + scanNoDefaultExcludes bool ) + short := "Walk trees and synchronize the scan database" + scanCmd := &cobra.Command{ - Use: cmdScan + " [--workers N] [-x] PATH...", - Short: "Walk trees and synchronize the scan database", + Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...", + Short: short, + Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS), Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, _ []string) error { return checkScanWorkers(cmd, scanWorkers) @@ -201,13 +207,16 @@ func newScanCommand() *cobra.Command { ctx, stop := interruptContext(ctx) defer stop() - return runScan(ctx, args, scanWorkers, scanOneFS) + return runScan(ctx, args, scanWorkers, scanOneFS, + scanNoDefaultExcludes) }), } scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(), "concurrent workers for the walk, hash, and content phases") scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false, "do not cross filesystem boundaries") + scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes", + false, "enter the default excludes, the paths listed above") return scanCmd } diff --git a/main_test.go b/main_test.go index f93fca2..8cd0b6f 100644 --- a/main_test.go +++ b/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "io/fs" "os" @@ -18,6 +19,36 @@ import ( // error prints and a runtime failure must not. const usageMarker = "Usage:" +// TestMain stops the tests at once, saying why, when their temporary +// directory is not on a persistent filesystem: scan records nothing +// there (README "Rules for the walk"), so most scan tests would fail +// without saying why. +func TestMain(m *testing.M) { + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + fmt.Fprintf(os.Stderr, "list the mounted filesystems: %v\n", err) + os.Exit(1) + } + + dir := os.TempDir() + + fi, err := os.Stat(dir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + dev, ok := deviceOfInfo(fi) + if fsType, found := nonPersistent[dev]; ok && found { + fmt.Fprintf(os.Stderr, "%s is on %s, not a persistent filesystem, "+ + "and scan records nothing there: set TMPDIR to a directory on a "+ + "persistent filesystem\n", dir, fsType) + os.Exit(1) + } + + os.Exit(m.Run()) +} + // walSuffixes are the SQLite sidecar files a WAL-mode database keeps // while it is open. A clean close checkpoints the WAL and removes // both; finding either afterwards means the database was never closed. @@ -535,7 +566,7 @@ func TestRunNamesDefaultDatabase(t *testing.T) { }{ cmdScan: { run: func() error { - return runScan(t.Context(), []string{missing}, 1, false) + return runScan(t.Context(), []string{missing}, 1, false, false) }, want: scanWant, }, diff --git a/scan.go b/scan.go index 642055b..ee6d6c4 100644 --- a/scan.go +++ b/scan.go @@ -13,6 +13,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -114,9 +115,12 @@ func scanPhase(name string) string { // SIGTERM that interruptContext catches, the scan keeps what it has // hashed (see syncScan), prints how many files its walk reached, and // returns errInterrupted. workers must be at least 1; the scan command -// rejects anything less. +// rejects anything less. Before the lock is taken, the scan lists the +// filesystems that are not persistent, on which the walk records +// nothing; unless noDefaultExcludes is set, the walk does not enter the +// default excludes either. func runScan(ctx context.Context, roots []string, workers int, - oneFS bool, + oneFS, noDefaultExcludes bool, ) error { dbPath, source, err := databasePath() if err != nil { @@ -139,6 +143,16 @@ func runScan(ctx context.Context, roots []string, workers int, return err } + nonPersistent, err := nonPersistentFilesystems() + if err != nil { + return fmt.Errorf("list the mounted filesystems: %w", err) + } + + rules := walkRules{nonPersistent: nonPersistent} + if !noDefaultExcludes { + rules.excludes = defaultExcludes(runtime.GOOS) + } + lock, err := lockScanDatabase(dbPath) if err != nil { return err @@ -159,7 +173,7 @@ func runScan(ctx context.Context, roots []string, workers int, defer closeScanDatabase(ctx, db, dbPath) - st, err := syncScan(ctx, db, roots, workers, oneFS) + st, err := syncScan(ctx, db, roots, workers, oneFS, rules) if errors.Is(err, context.Canceled) { return interrupted(st.walked) } @@ -231,14 +245,12 @@ func resolveRoots(roots []string) ([]string, error) { } // pruneRoots drops operands already covered by another operand: -// duplicates and any operand lying under another one. With -x, an -// operand under another one is kept when the walk of the deepest kept -// operand above it cannot reach it: when device puts any directory from -// it (from its directory, for a regular file) up to but not including -// that operand on a different filesystem than that operand. Every -// remaining file is then reachable through exactly one root, so walked -// paths are unique without keeping a scan-wide set of every path seen. -func pruneRoots(roots []string, oneFS bool, +// duplicates and any operand lying under another one. An operand under +// another one is kept, though, when the walk of the deepest kept operand +// above it does not reach it (see walkReaches). Every remaining file is +// then reachable through exactly one root, so walked paths are unique +// without keeping a scan-wide set of every path seen. +func pruneRoots(roots []string, oneFS bool, rules walkRules, device func(path string) (uint64, bool), ) []string { // Shorter-first ordering guarantees an ancestor is kept before any @@ -262,40 +274,57 @@ func pruneRoots(roots []string, oneFS bool, continue } - // Without -x, and always for a duplicate, r is dropped. - if !oneFS || above == r { - continue - } - - aboveDev, aboveOK := device(above) - if !aboveOK { - continue - } - - // The walk of above enters a directory only when it is on - // above's filesystem, and then reaches every regular file in it - // whatever device the file reports, so for a regular-file - // operand the directories checked start at its directory. - dir := r - - fi, err := os.Lstat(r) - if err == nil && fi.Mode().IsRegular() { - dir = filepath.Dir(r) - } - - for ; dir != above; dir = filepath.Dir(dir) { - dev, ok := device(dir) - if ok && dev != aboveDev { - kept = append(kept, r) - - break - } + // A duplicate is dropped, and so is an operand the walk of above + // reaches. + if above != r && !walkReaches(above, r, oneFS, rules, device) { + kept = append(kept, r) } } return kept } +// walkReaches reports whether the walk of operand above reaches r, an +// operand under it. That walk does not enter a default exclude in +// rules, nor a directory that device puts on a filesystem in rules that +// is not persistent, nor, with -x, one that device puts on a different +// filesystem than above, when device knows above's. It reaches every +// regular file in a directory it enters, so for a regular-file operand +// the directories checked start at its directory; they end below above. +func walkReaches(above, r string, oneFS bool, rules walkRules, + device func(path string) (uint64, bool), +) bool { + aboveDev, aboveDevOK := device(above) + + dir := r + + fi, err := os.Lstat(r) + if err == nil && fi.Mode().IsRegular() { + dir = filepath.Dir(r) + } + + for ; dir != above; dir = filepath.Dir(dir) { + if isDefaultExclude(dir, rules.excludes) { + return false + } + + dev, ok := device(dir) + if !ok { + continue + } + + if _, found := rules.nonPersistent[dev]; found { + return false + } + + if oneFS && aboveDevOK && dev != aboveDev { + return false + } + } + + return true +} + // deviceOfPath returns the filesystem device ID of the file at path, // when lstat succeeds and the platform exposes one. func deviceOfPath(path string) (uint64, bool) { @@ -337,11 +366,11 @@ type scanState struct { // the hashed records still waiting in the batch, starts no other write // or deletion, and returns the cancellation. func syncScan(ctx context.Context, db *sql.DB, roots []string, - workers int, oneFS bool, + workers int, oneFS bool, rules walkRules, ) (scanStats, error) { s := &scanState{db: db} - err := s.runPhases(ctx, roots, workers, oneFS) + err := s.runPhases(ctx, roots, workers, oneFS, rules) if err == nil || ctx.Err() == nil { return s.st, err } @@ -365,15 +394,18 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string, // and content (fill in the content hash of every record of headTailMin // or more whose size, head, and tail match another record's). Records // outside the roots are never touched, except that the content phase -// fills in their content hash. Operands the walk cannot start from are -// dropped first, so the records beneath them are treated as if those -// operands had not been given. +// fills in their content hash. Operands the walk cannot start from, or +// that are not on a persistent filesystem, are dropped first, so the +// records beneath them are treated as if those operands had not been +// given. Below an operand, the walk follows rules. func (s *scanState) runPhases(ctx context.Context, roots []string, - workers int, oneFS bool, + workers int, oneFS bool, rules walkRules, ) error { - // Types are checked before pruning so that an operand under a - // dropped one is still scanned, not dropped as lying under it. - roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath) + // Types and filesystems are checked before pruning so that an + // operand under a dropped one is still scanned, not dropped as lying + // under it. + roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules, + deviceOfPath) err := s.loadIndex(ctx, roots) if err != nil { @@ -381,7 +413,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, } changed, unhashed := s.walkPhase( - startWalk(ctx, roots, oneFS, workers), roots) + startWalk(ctx, roots, oneFS, rules, workers), roots) // A cancelled walk stops early, so its size census covers only part // of the roots, and every file it never reached would look vanished @@ -411,9 +443,11 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, // about, counted as skipped, and dropped. A dropped operand is no // longer a root, so the records stored beneath it are treated as if it // had not been given: those under no other root count as outside the -// roots and are not deleted as unverified. An operand that fails lstat -// here is kept, and the walk warns about it. -func (s *scanState) walkableRoots(roots []string) []string { +// roots and are not deleted as unverified. An operand on a filesystem +// in rules that is not persistent is dropped the same way, with a line +// saying so that is not a warning, and is not counted as skipped. An +// operand that fails lstat here is kept, and the walk warns about it. +func (s *scanState) walkableRoots(roots []string, rules walkRules) []string { kept := make([]string, 0, len(roots)) for _, root := range roots { @@ -427,6 +461,13 @@ func (s *scanState) walkableRoots(roots []string) []string { continue } + + line := rules.nonPersistentLine(root, fi) + if line != "" { + fmt.Fprintln(os.Stderr, escapePath(line)) + + continue + } } kept = append(kept, root) @@ -481,10 +522,12 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // (the content phase may still fill in its content hash). It returns // the new-or-changed files and the unchanged files whose records lack // hashes; both remain candidates until the census decides whether -// their sizes are shared. A record below a directory -x did not enter -// counts as outside the roots: it is not deleted, and its size joins -// the census. That does not hold below a root at or inside that -// directory, since that root's own walk verifies the records below it. +// their sizes are shared. A record at or below a path the walk left +// out, because -x stopped at it, it is a default exclude, or it is not +// on a persistent filesystem, counts as outside the roots: it is not +// deleted, and its size joins the census. That does not hold below a +// root at or inside that path, since that root's own walk verifies the +// records below it; nor is a line printed for such a root. func (s *scanState) walkPhase( events <-chan walkEvent, roots []string, ) ([]fileRec, []fileRec) { @@ -502,12 +545,25 @@ func (s *scanState) walkPhase( var ( changed, unhashed []fileRec - otherFSDirs []string + leftOut []string ) for ev := range events { if ev.otherFSDir != "" { - otherFSDirs = append(otherFSDirs, ev.otherFSDir) + leftOut = append(leftOut, ev.otherFSDir) + + continue + } + + // The line saying so is printed as a warning is, but what the + // walk leaves out is not counted as skipped. A default exclude + // that is itself a root is walked as one, so it gets no line. + if ev.leftOut != "" { + leftOut = append(leftOut, ev.leftOut) + + if !slices.Contains(roots, ev.leftOut) { + prog.warnf("%s", ev.warn) + } continue } @@ -542,12 +598,13 @@ func (s *scanState) walkPhase( prog.finish() for path, old := range s.existing { - // Kept only when a directory -x did not enter lies below the - // deepest root above the record. A root at or below that - // directory was walked on its own and did not find the file. - // Both contain the record, so the longer path is the deeper. - otherFSDir := deepestRoot(path, otherFSDirs) - if len(otherFSDir) > len(deepestRoot(path, roots)) { + // Kept only when a path the walk left out lies below the deepest + // root above the record, or is the record's own. A root at or + // below that path was walked on its own and did not find the + // file. Both contain the record, so the longer path is the + // deeper. + left := deepestRoot(path, leftOut) + if len(left) > len(deepestRoot(path, roots)) { delete(s.existing, path) s.sizes = append(s.sizes, old.size) @@ -1046,27 +1103,31 @@ type dirJob struct { } // walkEvent is one walk result delivered to the walk phase: a regular -// file's statted record, a warning when fail is set, or, when -// otherFSDir is set, a directory -x did not enter because it is on -// another filesystem than its operand. +// file's statted record, a warning when fail is set, or a path the walk +// left out: when otherFSDir is set, a directory -x did not enter +// because it is on another filesystem than its operand, and when +// leftOut is set, a default exclude or a directory or file on a +// filesystem that is not persistent, with warn holding the line that +// says so. type walkEvent struct { rec fileRec warn string fail bool otherFSDir string + leftOut string } // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning -// event per per-path error, and one event per directory -x did not -// enter. The channel is closed when the walk completes, and also when -// ctx is cancelled — every goroutine in the pool abandons its blocking -// send in that case, so the consumer sees a truncated but properly -// terminated stream instead of a stalled one. +// event per per-path error, and one event per path -x or rules kept the +// walk from entering or recording. The channel is closed when the walk +// completes, and also when ctx is cancelled — every goroutine in the +// pool abandons its blocking send in that case, so the consumer sees a +// truncated but properly terminated stream instead of a stalled one. func startWalk(ctx context.Context, roots []string, oneFS bool, - workers int, + rules walkRules, workers int, ) <-chan walkEvent { - jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS) + jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, rules) go func() { initial := make([]dirJob, 0, len(roots)) @@ -1177,7 +1238,7 @@ func seedRoot(ctx context.Context, root string, // workers drop the directories still queued rather than stop reading // jobs, so the range always runs out and the pool always tears down. func startWalkWorkers(ctx context.Context, workers int, - oneFS bool, + oneFS bool, rules walkRules, ) (chan dirJob, chan []dirJob, chan walkEvent) { jobs := make(chan dirJob, workQueueDepth) subdirs := make(chan []dirJob, workers) @@ -1193,7 +1254,7 @@ func startWalkWorkers(ctx context.Context, workers int, } select { - case subdirs <- walkOneDir(ctx, job, oneFS, events): + case subdirs <- walkOneDir(ctx, job, oneFS, rules, events): case <-ctx.Done(): } } @@ -1251,7 +1312,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob, // entry and a warning event per unreadable one, and returns the // subdirectories to descend into. func walkOneDir(ctx context.Context, job dirJob, oneFS bool, - events chan<- walkEvent, + rules walkRules, events chan<- walkEvent, ) []dirJob { entries, err := os.ReadDir(job.path) if err != nil { @@ -1275,7 +1336,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, p := filepath.Join(job.path, e.Name()) if e.IsDir() { - if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok { + sub, ok := subdirJob(ctx, p, e, job, oneFS, rules, events) + if ok { subs = append(subs, sub) } @@ -1287,7 +1349,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, continue } - emitFile(ctx, p, e, events) + emitFile(ctx, p, e, rules, events) } return subs @@ -1297,9 +1359,11 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, // The lstat happens here in the walk worker, while the directory's // metadata is still hot; a path that fails to stat (or stops being a // regular file) between the directory read and the lstat is warned -// about and skipped. +// about and skipped. A file on a filesystem in rules that is not +// persistent, as through a bind mount of one file, is not recorded: it +// gets an event naming it instead. func emitFile(ctx context.Context, p string, e fs.DirEntry, - events chan<- walkEvent, + rules walkRules, events chan<- walkEvent, ) { info, err := e.Info() if err != nil { @@ -1315,6 +1379,13 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, return } + line := rules.nonPersistentLine(p, info) + if line != "" { + sendEvent(ctx, events, walkEvent{leftOut: p, warn: line}) + + return + } + dev, ino := inodeOfInfo(info) sendEvent(ctx, events, walkEvent{rec: fileRec{ @@ -1328,22 +1399,20 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // subdirJob applies the descent rules to directory p: never enter // .zfs (ZFS snapshot pseudo-dirs would list every file once per -// snapshot), and with -x never enter a directory on a different -// filesystem than its operand, sending an event that names it so the -// scan keeps the records below it, except those below an operand at or +// snapshot), never enter a directory on a filesystem in rules that is +// not persistent, nor a default exclude in rules, and with -x never +// enter a directory on a different filesystem than its operand. For the +// last three it sends an event that names the directory, so the scan +// keeps the records below it, except those below an operand at or // inside it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, - parent dirJob, oneFS bool, events chan<- walkEvent, + parent dirJob, oneFS bool, rules walkRules, + events chan<- walkEvent, ) (dirJob, bool) { if e.Name() == ".zfs" { return dirJob{}, false } - job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK} - if !oneFS || !parent.rootDevOK { - return job, true - } - info, err := e.Info() if err != nil { sendEvent(ctx, events, walkEvent{ @@ -1354,6 +1423,27 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry, return dirJob{}, false } + line := rules.nonPersistentLine(p, info) + if line != "" { + sendEvent(ctx, events, walkEvent{leftOut: p, warn: line}) + + return dirJob{}, false + } + + if isDefaultExclude(p, rules.excludes) { + sendEvent(ctx, events, walkEvent{ + leftOut: p, + warn: fmt.Sprintf("scan: skipped %s (default exclude)", p), + }) + + return dirJob{}, false + } + + job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK} + if !oneFS || !parent.rootDevOK { + return job, true + } + if dev, ok := deviceOfInfo(info); ok && dev != parent.rootDev { sendEvent(ctx, events, walkEvent{otherFSDir: p}) diff --git a/scan_test.go b/scan_test.go index 1cfe8f2..3f45bae 100644 --- a/scan_test.go +++ b/scan_test.go @@ -782,7 +782,8 @@ func collectWalk(t *testing.T, roots []string, oneFS bool, errs int ) - for ev := range startWalk(t.Context(), roots, oneFS, workers) { + for ev := range startWalk(t.Context(), roots, oneFS, walkRules{}, + workers) { if ev.fail { errs++ @@ -1004,7 +1005,8 @@ func callSubdirJob(t *testing.T, p string, e fs.DirEntry, t.Helper() events := make(chan walkEvent, 1) - job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events) + job, ok := subdirJob(t.Context(), p, e, parent, oneFS, walkRules{}, + events) close(events) var evs []walkEvent @@ -1152,7 +1154,7 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, return dev, ok } - roots = pruneRoots(roots, true, device) + roots = pruneRoots(roots, true, walkRules{}, device) ctx := t.Context() s := &scanState{db: db} @@ -1162,7 +1164,7 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, t.Fatal(err) } - jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + jobs, subdirs, events := startWalkWorkers(ctx, 2, true, walkRules{}) // seedRoot emits a regular-file operand itself and returns a job // only for a directory operand. @@ -1463,7 +1465,7 @@ const smokeTreeFiles = 15 func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { t.Helper() - st, err := syncScan(t.Context(), db, roots, 4, false) + st, err := syncScan(t.Context(), db, roots, 4, false, walkRules{}) if err != nil { t.Fatalf("syncScan: %v", err) } @@ -2412,7 +2414,7 @@ func TestPruneRoots(t *testing.T) { // Duplicates and operands under other operands are dropped; /cc is // not under /c (sibling with a shared prefix). got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, - false, device) + false, walkRules{}, device) want := []string{"/a", "/c", "/cc"} if !slices.Equal(got, want) { @@ -2421,7 +2423,7 @@ func TestPruneRoots(t *testing.T) { nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"} - got = pruneRoots(nested, false, device) + got = pruneRoots(nested, false, walkRules{}, device) want = []string{"/p"} if !slices.Equal(got, want) { @@ -2429,12 +2431,31 @@ func TestPruneRoots(t *testing.T) { } // With -x, /p/q, on another filesystem than /p, is kept as well. - got = pruneRoots(nested, true, device) + got = pruneRoots(nested, true, walkRules{}, device) want = []string{"/p", pqRoot} if !slices.Equal(got, want) { t.Fatalf("pruneRoots with -x = %q, want %q", got, want) } + + // The walk of /p does not enter /p/q when it is a default exclude, so + // /p/q is kept, without -x too; /p/q/r lies under it and is dropped. + got = pruneRoots(nested, false, walkRules{excludes: []string{pqRoot}}, + device) + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want) + } + + // Nor does it enter /p/q when the filesystem of /p/q is not + // persistent, so /p/q/r/m, of no known device, is kept. + got = pruneRoots([]string{"/p", "/p/q/r/m"}, false, + walkRules{nonPersistent: map[uint64]string{2: "tmpfs"}}, device) + + want = []string{"/p", "/p/q/r/m"} + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots with %s not persistent = %q, want %q", + pqRoot, got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) {