diff --git a/README.md b/README.md index 4273341..13dcad6 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,10 @@ As a file `/etc/cron.d/sfdupes`: `/var/log/sfdupes.log`, holds a first line naming the database (see "Database"), plain progress lines naming each phase and its place among the five, as `[2/5] walk` (one as the phase starts, one every 5 seconds while it - runs, and one as it ends), a warning for each path it skips, and the summary - line (see "Progress" and "`scan` mode"). The log grows with every scan; rotate - it like any other. + runs, and one as it ends), a warning for each path it skips, a line for each + default exclude its walk does not enter (see "Rules for the walk"), and the + summary line (see "Progress" and "`scan` mode"). The log grows with every + scan; rotate it like any other. - Skipped paths do not fail a scan: it still exits 0, and cron sends nothing. A scan that fails, or is stopped by `SIGINT` or `SIGTERM`, exits 1 with the reason among the last lines of the log; `tail` prints them, and cron mails @@ -248,7 +249,7 @@ Four subcommands, all implemented: schema version and the counts of its records. ``` -sfdupes scan [--workers N] [-x] PATH... +sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... sfdupes report > dupes.tsv sfdupes trees > dupetrees.tsv sfdupes info @@ -257,8 +258,9 @@ sfdupes [command] --help ``` `--workers N` sets the size of each `scan` worker pool (default: the number of -CPUs), and `-x` (`--one-file-system`) keeps the walk of each operand on that -operand's filesystem; both are described under "`scan` mode". +CPUs), `-x` (`--one-file-system`) keeps the walk of each operand on that +operand's filesystem, and `--no-default-excludes` lets the walk enter the +default excludes; all three are described under "`scan` mode". `sfdupes --version` (or `-v`) prints one line, `sfdupes VERSION`, to stdout and exits 0, writing nothing to stderr. `-h` or `--help`, alone or after a subcommand, prints the help text to stderr and exits 0, writing nothing to @@ -439,11 +441,12 @@ results do not depend on the working directory. All operands belong to a single scan and are enumerated concurrently: every operand seeds the shared walk worker pool. Overlapping operands are harmless — an operand that duplicates another or lies under another is dropped before walking, so every file is reached exactly -once and produces one database record. With `-x`, though, an operand under -another one is kept and walked on its own when any directory from it (from its -directory, for a regular file) up to but not including the operand covering it -is on a different filesystem than that operand, because that operand's walk does -not enter such a directory and so does not reach it. +once and produces one database record. An operand under another one is kept, +though, and walked on its own when any directory from it (from its directory, +for a regular file) up to but not including the operand covering it is a default +exclude (see "Rules for the walk") or, with `-x`, is on a different filesystem +than that operand, because that operand's walk does not enter such a directory +and so does not reach it. An operand that is a symlink (never followed, not even as an operand), socket, FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints @@ -491,12 +494,13 @@ operands: a `content` hash is read for one, wherever it lies, once its size, `head`, and `tail` match another record's. If that file is gone or has changed since its record was written, the record is left as it is. -- With `-x`, a record below a directory the walk did not enter because that - directory is on another filesystem than its operand counts as outside the - scanned operands: it is not deleted, and its size joins the size census. A - filesystem mounted under an operand can therefore be scanned on its own - schedule into the same database. Records below an operand at or inside such a - directory are not kept this way, since that operand's own walk verifies them. +- A record below a directory the walk did not enter, because that directory is a + default exclude or, with `-x`, because it is on another filesystem than its + operand, counts as outside the scanned operands: it is not deleted, and its + size joins the size census. A filesystem mounted under an operand can + therefore be scanned on its own schedule into the same database. Records below + an operand at or inside such a directory are not kept this way, since that + operand's own walk verifies them. `scan` runs **five sequential phases over the whole scan**. Parallelism lives inside the walk, hash and content phases; batched database writes begin during @@ -572,6 +576,42 @@ Rules for the walk: unless they lie below an operand at or inside that directory, count as outside the scanned operands, as described in "`scan` mode" above: they are not deleted, and their sizes join the size census. +- Below an operand, never descend into a default exclude: a directory that holds + the kernel's view of the running system, or temporary files, so that + `sfdupes scan /` walks only the files worth comparing. The default excludes + depend on the system, and `sfdupes scan --help` lists those of the system it + runs on: + - On Linux: `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and `/var/tmp`, and the + mount point of every filesystem of type `proc`, `sysfs`, `devtmpfs`, + `devpts`, `tmpfs`, `ramfs`, `cgroup`, `cgroup2`, `debugfs`, `tracefs`, + `securityfs`, `pstore`, `bpf`, `configfs`, `fusectl`, `mqueue`, + `hugetlbfs`, `efivarfs`, `binfmt_misc`, `autofs`, `nsfs`, `rpc_pipefs` or + `selinuxfs`, wherever it is mounted. Where one filesystem is mounted over + another, the one mounted last counts. `scan` reads the mount points from + `/proc/self/mountinfo` once, after it names the database and before it + takes the lock; if it cannot read them, it fails (exit 1). + - On macOS: `/dev`, `/private/tmp`, `/private/var/tmp`, `/private/var/vm`, + `/System/Volumes/VM` and `/System/Volumes/Data`. A walk of `/` reaches + everything in `/System/Volumes/Data` through the firmlinks, so walking it + as well would record each of those files twice. + - On any other system there are none. + + `--no-default-excludes` turns them all off, and `scan` then does not read + `/proc/self/mountinfo`. An operand at or inside a default exclude is still + walked, so `sfdupes scan /tmp/x` works. For each default exclude its walk + does not enter, `scan` prints a line on stderr naming it and why, as it + prints a warning, so it never lands inside a progress display: + + ``` + scan: skipped /proc (default exclude, proc filesystem) + scan: skipped /tmp (default exclude) + ``` + + That line is not a warning, and the directory is not counted as skipped. The + records below it, unless they lie below an operand at or inside it, count as + outside the scanned operands, as described in "`scan` mode" above: they are + not deleted, and their sizes join the size census. + - On any per-path error (permission denied, file vanished between passes, unreadable): print a one-line warning to stderr, skip the path, and continue. Per-file errors never abort the run; the final summary reports how many were diff --git a/TODO.md b/TODO.md index b814411..17753e8 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,19 @@ # Completed Steps +- below an operand, `scan`'s walk does not enter the default excludes, so + `sfdupes scan /` skips the kernel's view of the running system and the + temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp`, + `/var/tmp` and the mount point of every kernel or memory-backed filesystem, + read from `/proc/self/mountinfo` as the scan starts; on macOS `/dev`, + `/private/tmp`, `/private/var/tmp`, `/private/var/vm`, `/System/Volumes/VM` + and `/System/Volumes/Data`. An operand at or inside one is still walked. The + scan prints a line for each it does not enter, as it prints a warning, without + counting it as skipped, and the records below it count as outside the scanned + operands, as for `-x`. `--no-default-excludes` turns them off, and + `scan --help` lists them (2026-10-16, + https://git.eeqj.de/sneak/sfdupes/issues/124) + - the rate a progress display shows covers the items completed over the last 5 seconds, or since the line before when stderr is not a terminal, so it falls to 0 while nothing completes, and a burst after a wait no longer reads as a diff --git a/cancel_test.go b/cancel_test.go index da543fd..894b57e 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -196,7 +196,7 @@ func TestSyncScanCancelledMidWalkKeepsRecords(t *testing.T) { base := baselineGoroutines(t) st, err := syncScan(newWalkClock(walkCancelAtDone), db, - []string{dir}, walkCancelWorkers, false) + []string{dir}, walkCancelWorkers, false, nil) assertWalkGuardAborted(t, st, err) assertRecordsIntact(t, db, before) @@ -268,7 +268,8 @@ func TestSyncScanCancelledBeforeLoadIndex(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false) + st, err := syncScan(ctx, db, []string{dir}, walkCancelWorkers, false, + nil) if !errors.Is(err, context.Canceled) { t.Fatalf("syncScan on a cancelled context = %v, want %v", err, context.Canceled) @@ -301,7 +302,7 @@ func TestSyncScanCancelledMidHashKeepsHashedRecords(t *testing.T) { db := openTestDB(t) st, err := syncScan(newWalkClock(hashCancelAtDone), db, - []string{dir}, walkCancelWorkers, false) + []string{dir}, walkCancelWorkers, false, nil) if !errors.Is(err, context.Canceled) { t.Fatalf("syncScan cancelled mid-hash = %v, want %v", err, context.Canceled) @@ -356,7 +357,8 @@ func TestRunScanInterrupted(t *testing.T) { stderr := captureStderr(t) dir := buildSmokeTree(t) - err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + err := runScan(t.Context(), []string{dir}, walkCancelWorkers, false, + false) if err != nil { t.Fatal(err) } @@ -380,7 +382,7 @@ func TestRunScanInterrupted(t *testing.T) { defer close(done) err = runScan(cancelledContext(t), []string{dir}, walkCancelWorkers, - false) + false, false) }() awaitReturn(t, done, "runScan") @@ -407,7 +409,8 @@ func TestRunScanInterrupted(t *testing.T) { got, before) } - err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false) + err = runScan(t.Context(), []string{dir}, walkCancelWorkers, false, + false) if err != nil { t.Fatal(err) } @@ -433,7 +436,7 @@ func TestRunScanInterruptedMidHash(t *testing.T) { dir := buildWalkCancelTree(t) err := runScan(newWalkClock(hashCancelAtDone), []string{dir}, - walkCancelWorkers, false) + walkCancelWorkers, false, false) if !errors.Is(err, errInterrupted) { t.Fatalf("runScan interrupted mid-hash = %v, want %v", err, errInterrupted) @@ -617,7 +620,8 @@ func TestWalkOneDirStopsWhenCancelled(t *testing.T) { // Unbuffered and unread: on a cancelled scan every send gives up. events := make(chan walkEvent) - subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, events) + subs := walkOneDir(cancelledContext(t), dirJob{path: dir}, false, nil, + events) if len(subs) != 0 { t.Errorf("cancelled walkOneDir returned %+v to descend into, "+ "want none", subs) @@ -637,7 +641,7 @@ func TestWalkWorkersDropQueuedDirs(t *testing.T) { missing := filepath.Join(t.TempDir(), "missing") - jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false) + jobs, _, events := startWalkWorkers(cancelledContext(t), 2, false, nil) for range 64 { jobs <- dirJob{path: missing} @@ -668,7 +672,7 @@ func TestWalkWorkerAbandonsSubdirHandoff(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - jobs, subdirs, events := startWalkWorkers(ctx, 1, false) + jobs, subdirs, events := startWalkWorkers(ctx, 1, false, nil) // Fill the hand-back channel to its capacity — one slot per worker // — so the worker's own hand-back is certain to block. diff --git a/exclude.go b/exclude.go new file mode 100644 index 0000000..b31fcf3 --- /dev/null +++ b/exclude.go @@ -0,0 +1,177 @@ +package main + +// This file holds the default excludes (README "Rules for the walk"): +// the directories a scan's walk does not enter below an operand, unless +// scan is given --no-default-excludes. + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "slices" + "strings" +) + +// The systems that have default excludes, as runtime.GOOS names them. +const ( + goosLinux = "linux" + goosDarwin = "darwin" +) + +// mountInfoPath is the file in which Linux lists the mounts the reading +// process sees. +const mountInfoPath = "/proc/self/mountinfo" + +// The fields of a line of mountInfoPath that kernelMounts reads, counted +// from 0 (see proc(5)): the mount point is the fifth field, and after +// the sixth come optional fields, then a field holding "-", then the +// filesystem type. +const ( + mountPointField = 4 + optionalFieldsStart = 6 +) + +// typesPerHelpLine is how many filesystem types the scan command's help +// lists on each line. +const typesPerHelpLine = 6 + +// errMountInfoLine reports a line of mountInfoPath without the fields +// kernelMounts reads. +var errMountInfoLine = errors.New("unexpected line") + +// defaultExcludes returns the directories that are default excludes on +// the system goos names, as runtime.GOOS does: a fixed list on Linux and +// on macOS, and none on any other system. On Linux the mount points +// kernelMounts finds are default excludes as well. +func defaultExcludes(goos string) []string { + switch goos { + case goosLinux: + return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"} + case goosDarwin: + // A walk of / reaches everything in /System/Volumes/Data through + // the firmlinks, so walking it as well would record each of + // those files twice. + return []string{ + "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", + "/System/Volumes/VM", "/System/Volumes/Data", + } + default: + return nil + } +} + +// kernelFilesystemTypes returns the Linux filesystem types whose mount +// points are default excludes: filesystems that the kernel makes up or +// that are kept in memory. +func kernelFilesystemTypes() []string { + return []string{ + "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "ramfs", + "cgroup", "cgroup2", "debugfs", "tracefs", "securityfs", "pstore", + "bpf", "configfs", "fusectl", "mqueue", "hugetlbfs", "efivarfs", + "binfmt_misc", "autofs", "nsfs", "rpc_pipefs", "selinuxfs", + } +} + +// findDefaultExcludes returns the default excludes of the system goos +// names, each with what the scan says about it on the line it prints +// when its walk does not enter it. On Linux it reads the mount points +// from mountInfoPath. +func findDefaultExcludes(goos string) (map[string]string, error) { + excludes := make(map[string]string) + + for _, dir := range defaultExcludes(goos) { + excludes[dir] = "default exclude" + } + + if goos != goosLinux { + return excludes, nil + } + + f, err := os.Open(mountInfoPath) + if err != nil { + return nil, err + } + + defer func() { _ = f.Close() }() + + mounts, err := kernelMounts(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", mountInfoPath, err) + } + + for dir, fsType := range mounts { + excludes[dir] = "default exclude, " + fsType + " filesystem" + } + + return excludes, nil +} + +// kernelMounts reads mountinfo, laid out as mountInfoPath is, and +// returns the mount point of every filesystem whose type is one of +// kernelFilesystemTypes, with that type. Where one filesystem is mounted +// over another, a later line names the one on top, so the last line for +// a mount point decides. +func kernelMounts(mountinfo io.Reader) (map[string]string, error) { + types := kernelFilesystemTypes() + + // The kernel writes a space, tab, newline or backslash in a mount + // point as an octal escape. + unescape := strings.NewReplacer( + `\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + + mounts := make(map[string]string) + lines := bufio.NewScanner(mountinfo) + + for lines.Scan() { + fields := strings.Fields(lines.Text()) + + sep := slices.Index(fields, "-") + if sep < optionalFieldsStart || sep+1 >= len(fields) { + return nil, fmt.Errorf("%w: %q", errMountInfoLine, lines.Text()) + } + + mountPoint := unescape.Replace(fields[mountPointField]) + + fsType := fields[sep+1] + if slices.Contains(types, fsType) { + mounts[mountPoint] = fsType + } else { + delete(mounts, mountPoint) + } + } + + return mounts, lines.Err() +} + +// defaultExcludesHelp describes the default excludes of the system goos +// names, for the scan command's help. +func defaultExcludesHelp(goos string) string { + dirs := defaultExcludes(goos) + if len(dirs) == 0 { + return "This system has no default excludes." + } + + var help strings.Builder + + help.WriteString("Below an operand, the walk does not enter these " + + "directories, the default\nexcludes, unless --no-default-excludes " + + "is given:\n\n") + + for _, dir := range dirs { + fmt.Fprintf(&help, " %s\n", dir) + } + + if goos == goosLinux { + help.WriteString("\nnor the mount point of any filesystem of " + + "these types:\n\n") + + types := kernelFilesystemTypes() + for line := range slices.Chunk(types, typesPerHelpLine) { + fmt.Fprintf(&help, " %s\n", strings.Join(line, ", ")) + } + } + + return help.String() +} diff --git a/exclude_test.go b/exclude_test.go new file mode 100644 index 0000000..8851787 --- /dev/null +++ b/exclude_test.go @@ -0,0 +1,266 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" +) + +// defaultExcludeWhy is what the scan says about a default exclude from +// the fixed list when its walk does not enter it. +const defaultExcludeWhy = "default exclude" + +func TestDefaultExcludes(t *testing.T) { + t.Parallel() + + // README "Rules for the walk": each system's list. + cases := map[string][]string{ + goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, + goosDarwin: { + "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", + "/System/Volumes/VM", "/System/Volumes/Data", + }, + "freebsd": nil, + } + + for goos, want := range cases { + if got := defaultExcludes(goos); !slices.Equal(got, want) { + t.Errorf("defaultExcludes(%q) = %q, want %q", goos, got, want) + } + } +} + +func TestKernelMounts(t *testing.T) { + t.Parallel() + + // The second-to-last line has no optional fields. /mnt/over has an + // ext4 filesystem mounted over its tmpfs, so it holds files. + mountinfo := strings.Join([]string{ + "22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw", + "23 22 0:21 / /proc rw,nosuid shared:12 - proc none rw", + "24 22 0:22 / /sys rw,nosuid shared:7 - sysfs none rw", + `25 22 0:23 / /mnt/ram\040disk rw shared:2 - tmpfs none rw`, + "26 22 0:24 / /mnt/over rw shared:3 - tmpfs none rw", + "27 26 8:2 / /mnt/over rw - ext4 /dev/sda2 rw", + "28 22 0:25 / /srv/auto rw shared:4 - autofs systemd-1 rw", + }, "\n") + + got, err := kernelMounts(strings.NewReader(mountinfo)) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "/proc": "proc", "/sys": "sysfs", "/mnt/ram disk": "tmpfs", + "/srv/auto": "autofs", + } + if !maps.Equal(got, want) { + t.Errorf("kernelMounts = %q, want %q", got, want) + } + + _, err = kernelMounts(strings.NewReader("22 1 8:1 / / rw,relatime\n")) + if !errors.Is(err, errMountInfoLine) { + t.Errorf("kernelMounts of a line without a type = %v, want %v", + err, errMountInfoLine) + } +} + +func TestFindDefaultExcludesReadsMounts(t *testing.T) { + t.Parallel() + + if runtime.GOOS != goosLinux { + t.Skip("only Linux lists its mounts in " + mountInfoPath) + } + + excludes, err := findDefaultExcludes(goosLinux) + if err != nil { + t.Fatal(err) + } + + for _, dir := range defaultExcludes(goosLinux) { + if _, ok := excludes[dir]; !ok { + t.Errorf("%s is not a default exclude", dir) + } + } + + want := defaultExcludeWhy + ", proc filesystem" + if got := excludes["/proc"]; got != want { + t.Errorf("/proc is %q, want %q", got, want) + } +} + +func TestDefaultExcludesHelp(t *testing.T) { + t.Parallel() + + for _, goos := range []string{goosLinux, goosDarwin} { + help := defaultExcludesHelp(goos) + + for _, dir := range defaultExcludes(goos) { + if !strings.Contains(help, "\n "+dir+"\n") { + t.Errorf("help for %s does not list %s:\n%s", goos, dir, help) + } + } + + // Only Linux has filesystem types among its default excludes. + for _, fsType := range kernelFilesystemTypes() { + listed := strings.Contains(help, " "+fsType+",") || + strings.Contains(help, " "+fsType+"\n") + if listed != (goos == goosLinux) { + t.Errorf("help for %s lists %s: %v", goos, fsType, listed) + } + } + } + + want := "This system has no default excludes." + if got := defaultExcludesHelp("freebsd"); got != want { + t.Errorf("help for freebsd = %q, want %q", got, want) + } + + // scan's help shows those of the system it runs on. + var stdout, stderr bytes.Buffer + + run([]string{cmdScan, "-h"}, &stdout, &stderr) + + help := strings.TrimSpace(defaultExcludesHelp(runtime.GOOS)) + if !strings.Contains(stderr.String(), help) { + t.Errorf("scan --help = %q, want it to hold %q", stderr.String(), help) + } +} + +// TestScanDoesNotEnterDefaultExclude checks that the walk does not +// enter a default exclude below an operand. It says so on a line of its +// own and does not count it as skipped. The records below it count as +// outside the operands: the record of a file gone from it is kept, and +// its size joins the size census, while a file added to it is not +// recorded. +// +//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr +func TestScanDoesNotEnterDefaultExclude(t *testing.T) { + stderr := captureStderr(t) + dir := t.TempDir() + db := openTestDB(t) + excluded := filepath.Join(dir, "tmp") + + // A scan with no default excludes records the file in excluded. + gone := writeFile(t, excluded, "gone", pattern(1, 100)) + syncTree(t, db, dir) + + err := os.Remove(gone) + if err != nil { + t.Fatal(err) + } + + unseen := writeFile(t, excluded, "new", pattern(2, 10)) + + // The same size as gone, so it is hashed only if the census holds the + // size of gone's record. + added := writeFile(t, dir, "added", pattern(3, 100)) + + st, err := syncScan(t.Context(), db, []string{dir}, 1, false, + map[string]string{excluded: defaultExcludeWhy}) + if err != nil { + t.Fatal(err) + } + + if st.skipped != 0 { + t.Errorf("skipped = %d, want 0", st.skipped) + } + + recs := dbRecords(t, db) + recordByPath(t, recs, gone) + + if recordByPath(t, recs, added).head == "" { + t.Errorf("%s not hashed: the census lacks the size of %s", + added, gone) + } + + if slices.Contains(recordPaths(recs), unseen) { + t.Errorf("%s recorded: the walk entered %s", unseen, excluded) + } + + line := "\nscan: skipped " + excluded + " (" + defaultExcludeWhy + ")\n" + if !strings.Contains(stderr(), line) { + t.Errorf("stderr = %q, want %q", stderr(), line) + } +} + +// TestScanWalksOperandInDefaultExclude checks that an operand at or +// inside a default exclude is walked, also when it lies under another +// operand, whose walk does not enter the default exclude. +func TestScanWalksOperandInDefaultExclude(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + excluded := filepath.Join(dir, "tmp") + f := writeFile(t, excluded, "sub/f", pattern(1, 10)) + sub := filepath.Dir(f) + + cases := map[string][]string{ + "at it": {excluded}, + "inside it": {sub}, + "inside it under an operand": {dir, sub}, + } + + for name, roots := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + st, err := syncScan(t.Context(), db, roots, 1, false, + map[string]string{excluded: defaultExcludeWhy}) + if err != nil { + t.Fatal(err) + } + + if st.added != 1 { + t.Errorf("added = %d, want %s recorded once", st.added, f) + } + + recordByPath(t, dbRecords(t, db), f) + }) + } +} + +// TestRunScanTempDirOperand scans a tree in the tests' temporary +// directory, which on Linux lies inside the default exclude /tmp unless +// TMPDIR names another place, with the system's own default excludes and +// then with --no-default-excludes: each scan walks the whole tree and +// reports no default exclude. +func TestRunScanTempDirOperand(t *testing.T) { + path := testDBPath(t) + t.Setenv(databaseEnv, path) + + dir := t.TempDir() + if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) { + t.Skip(dir + " is not inside a default exclude") + } + + for i, flags := range [][]string{nil, {"--no-default-excludes"}} { + // A file added before each scan shows that the scan walked the + // tree. + f := writeFile(t, dir, fmt.Sprintf("sub/%d", i), pattern(1, 10)) + + stderr := scanOK(t, append(flags, dir)...) + if strings.Contains(stderr, defaultExcludeWhy) { + t.Errorf("scan %q: stderr = %q, want no default exclude", + flags, stderr) + } + + db, err := openDB(path, reportParams) + if err != nil { + t.Fatal(err) + } + + recordByPath(t, dbRecords(t, db), f) + + _ = db.Close() + } +} diff --git a/main.go b/main.go index 8f33973..93fa0f5 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ // // Usage: // -// sfdupes scan [--workers N] [-x] PATH... +// sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH... // sfdupes report > dupes.tsv // sfdupes trees > dupetrees.tsv // sfdupes info @@ -182,17 +182,22 @@ func newRootCommand(stdout, stderr io.Writer) *cobra.Command { return root } -// newScanCommand builds the scan subcommand, with its --workers and -x -// flags. +// newScanCommand builds the scan subcommand, with its --workers, -x and +// --no-default-excludes flags. Its help lists the default excludes of +// the system it runs on. func newScanCommand() *cobra.Command { var ( - scanWorkers int - scanOneFS bool + scanWorkers int + scanOneFS bool + scanNoDefaultExcludes bool ) + short := "Walk trees and synchronize the scan database" + scanCmd := &cobra.Command{ - Use: cmdScan + " [--workers N] [-x] PATH...", - Short: "Walk trees and synchronize the scan database", + Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...", + Short: short, + Long: short + ".\n\n" + defaultExcludesHelp(runtime.GOOS), Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, _ []string) error { return checkScanWorkers(cmd, scanWorkers) @@ -201,13 +206,16 @@ func newScanCommand() *cobra.Command { ctx, stop := interruptContext(ctx) defer stop() - return runScan(ctx, args, scanWorkers, scanOneFS) + return runScan(ctx, args, scanWorkers, scanOneFS, + scanNoDefaultExcludes) }), } scanCmd.Flags().IntVar(&scanWorkers, "workers", runtime.NumCPU(), "concurrent workers for the walk, hash, and content phases") scanCmd.Flags().BoolVarP(&scanOneFS, "one-file-system", "x", false, "do not cross filesystem boundaries") + scanCmd.Flags().BoolVar(&scanNoDefaultExcludes, "no-default-excludes", + false, "enter the default excludes") return scanCmd } diff --git a/main_test.go b/main_test.go index f93fca2..492880a 100644 --- a/main_test.go +++ b/main_test.go @@ -535,7 +535,7 @@ func TestRunNamesDefaultDatabase(t *testing.T) { }{ cmdScan: { run: func() error { - return runScan(t.Context(), []string{missing}, 1, false) + return runScan(t.Context(), []string{missing}, 1, false, false) }, want: scanWant, }, diff --git a/scan.go b/scan.go index 642055b..ae85b8f 100644 --- a/scan.go +++ b/scan.go @@ -13,6 +13,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "slices" "strings" "sync" @@ -114,9 +115,10 @@ func scanPhase(name string) string { // SIGTERM that interruptContext catches, the scan keeps what it has // hashed (see syncScan), prints how many files its walk reached, and // returns errInterrupted. workers must be at least 1; the scan command -// rejects anything less. +// rejects anything less. Unless noDefaultExcludes is set, the walk does +// not enter the default excludes, found before the lock is taken. func runScan(ctx context.Context, roots []string, workers int, - oneFS bool, + oneFS, noDefaultExcludes bool, ) error { dbPath, source, err := databasePath() if err != nil { @@ -139,6 +141,16 @@ func runScan(ctx context.Context, roots []string, workers int, return err } + // With no excludes the walk enters every directory. + var excludes map[string]string + + if !noDefaultExcludes { + excludes, err = findDefaultExcludes(runtime.GOOS) + if err != nil { + return fmt.Errorf("find the default excludes: %w", err) + } + } + lock, err := lockScanDatabase(dbPath) if err != nil { return err @@ -159,7 +171,7 @@ func runScan(ctx context.Context, roots []string, workers int, defer closeScanDatabase(ctx, db, dbPath) - st, err := syncScan(ctx, db, roots, workers, oneFS) + st, err := syncScan(ctx, db, roots, workers, oneFS, excludes) if errors.Is(err, context.Canceled) { return interrupted(st.walked) } @@ -231,14 +243,12 @@ func resolveRoots(roots []string) ([]string, error) { } // pruneRoots drops operands already covered by another operand: -// duplicates and any operand lying under another one. With -x, an -// operand under another one is kept when the walk of the deepest kept -// operand above it cannot reach it: when device puts any directory from -// it (from its directory, for a regular file) up to but not including -// that operand on a different filesystem than that operand. Every -// remaining file is then reachable through exactly one root, so walked -// paths are unique without keeping a scan-wide set of every path seen. -func pruneRoots(roots []string, oneFS bool, +// duplicates and any operand lying under another one. An operand under +// another one is kept, though, when the walk of the deepest kept operand +// above it does not reach it (see walkReaches). Every remaining file is +// then reachable through exactly one root, so walked paths are unique +// without keeping a scan-wide set of every path seen. +func pruneRoots(roots []string, oneFS bool, excludes map[string]string, device func(path string) (uint64, bool), ) []string { // Shorter-first ordering guarantees an ancestor is kept before any @@ -262,40 +272,59 @@ func pruneRoots(roots []string, oneFS bool, continue } - // Without -x, and always for a duplicate, r is dropped. - if !oneFS || above == r { - continue - } - - aboveDev, aboveOK := device(above) - if !aboveOK { - continue - } - - // The walk of above enters a directory only when it is on - // above's filesystem, and then reaches every regular file in it - // whatever device the file reports, so for a regular-file - // operand the directories checked start at its directory. - dir := r - - fi, err := os.Lstat(r) - if err == nil && fi.Mode().IsRegular() { - dir = filepath.Dir(r) - } - - for ; dir != above; dir = filepath.Dir(dir) { - dev, ok := device(dir) - if ok && dev != aboveDev { - kept = append(kept, r) - - break - } + // A duplicate is dropped, and so is an operand the walk of above + // reaches. + if above != r && !walkReaches(above, r, oneFS, excludes, device) { + kept = append(kept, r) } } return kept } +// walkReaches reports whether the walk of operand above reaches r, an +// operand under it. That walk does not enter a directory in excludes, +// nor, with -x, one that device puts on a different filesystem than +// above, when device knows above's. It reaches every regular file in a +// directory it enters, whatever device the file reports, so for a +// regular-file operand the directories checked start at its directory; +// they end below above. +func walkReaches(above, r string, oneFS bool, excludes map[string]string, + device func(path string) (uint64, bool), +) bool { + var ( + aboveDev uint64 + checkDevice bool + ) + + if oneFS { + aboveDev, checkDevice = device(above) + } + + dir := r + + fi, err := os.Lstat(r) + if err == nil && fi.Mode().IsRegular() { + dir = filepath.Dir(r) + } + + for ; dir != above; dir = filepath.Dir(dir) { + if _, ok := excludes[dir]; ok { + return false + } + + if !checkDevice { + continue + } + + if dev, ok := device(dir); ok && dev != aboveDev { + return false + } + } + + return true +} + // deviceOfPath returns the filesystem device ID of the file at path, // when lstat succeeds and the platform exposes one. func deviceOfPath(path string) (uint64, bool) { @@ -337,11 +366,11 @@ type scanState struct { // the hashed records still waiting in the batch, starts no other write // or deletion, and returns the cancellation. func syncScan(ctx context.Context, db *sql.DB, roots []string, - workers int, oneFS bool, + workers int, oneFS bool, excludes map[string]string, ) (scanStats, error) { s := &scanState{db: db} - err := s.runPhases(ctx, roots, workers, oneFS) + err := s.runPhases(ctx, roots, workers, oneFS, excludes) if err == nil || ctx.Err() == nil { return s.st, err } @@ -367,13 +396,15 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string, // outside the roots are never touched, except that the content phase // fills in their content hash. Operands the walk cannot start from are // dropped first, so the records beneath them are treated as if those -// operands had not been given. +// operands had not been given. Below an operand, the walk does not +// enter the directories in excludes. func (s *scanState) runPhases(ctx context.Context, roots []string, - workers int, oneFS bool, + workers int, oneFS bool, excludes map[string]string, ) error { // Types are checked before pruning so that an operand under a // dropped one is still scanned, not dropped as lying under it. - roots = pruneRoots(s.walkableRoots(roots), oneFS, deviceOfPath) + roots = pruneRoots(s.walkableRoots(roots), oneFS, excludes, + deviceOfPath) err := s.loadIndex(ctx, roots) if err != nil { @@ -381,7 +412,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string, } changed, unhashed := s.walkPhase( - startWalk(ctx, roots, oneFS, workers), roots) + startWalk(ctx, roots, oneFS, excludes, workers), roots) // A cancelled walk stops early, so its size census covers only part // of the roots, and every file it never reached would look vanished @@ -481,10 +512,11 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // (the content phase may still fill in its content hash). It returns // the new-or-changed files and the unchanged files whose records lack // hashes; both remain candidates until the census decides whether -// their sizes are shared. A record below a directory -x did not enter -// counts as outside the roots: it is not deleted, and its size joins -// the census. That does not hold below a root at or inside that -// directory, since that root's own walk verifies the records below it. +// their sizes are shared. A record below a directory the walk did not +// enter, because -x stopped at it or it is a default exclude, counts as +// outside the roots: it is not deleted, and its size joins the census. +// That does not hold below a root at or inside that directory, since +// that root's own walk verifies the records below it. func (s *scanState) walkPhase( events <-chan walkEvent, roots []string, ) ([]fileRec, []fileRec) { @@ -502,12 +534,22 @@ func (s *scanState) walkPhase( var ( changed, unhashed []fileRec - otherFSDirs []string + notEntered []string ) for ev := range events { if ev.otherFSDir != "" { - otherFSDirs = append(otherFSDirs, ev.otherFSDir) + notEntered = append(notEntered, ev.otherFSDir) + + continue + } + + // The line saying so is printed as a warning is, but a default + // exclude is not counted as skipped. + if ev.excludedDir != "" { + notEntered = append(notEntered, ev.excludedDir) + + prog.warnf("%s", ev.warn) continue } @@ -542,12 +584,12 @@ func (s *scanState) walkPhase( prog.finish() for path, old := range s.existing { - // Kept only when a directory -x did not enter lies below the - // deepest root above the record. A root at or below that + // Kept only when a directory the walk did not enter lies below + // the deepest root above the record. A root at or below that // directory was walked on its own and did not find the file. // Both contain the record, so the longer path is the deeper. - otherFSDir := deepestRoot(path, otherFSDirs) - if len(otherFSDir) > len(deepestRoot(path, roots)) { + dir := deepestRoot(path, notEntered) + if len(dir) > len(deepestRoot(path, roots)) { delete(s.existing, path) s.sizes = append(s.sizes, old.size) @@ -1046,27 +1088,30 @@ type dirJob struct { } // walkEvent is one walk result delivered to the walk phase: a regular -// file's statted record, a warning when fail is set, or, when -// otherFSDir is set, a directory -x did not enter because it is on -// another filesystem than its operand. +// file's statted record, a warning when fail is set, or a directory the +// walk did not enter: when otherFSDir is set, one -x did not enter +// because it is on another filesystem than its operand, and when +// excludedDir is set, a default exclude, with warn holding the line +// that says so. type walkEvent struct { - rec fileRec - warn string - fail bool - otherFSDir string + rec fileRec + warn string + fail bool + otherFSDir string + excludedDir string } // startWalk seeds every root into the shared walk worker pool and // returns the event stream: one record per regular file, one warning -// event per per-path error, and one event per directory -x did not -// enter. The channel is closed when the walk completes, and also when -// ctx is cancelled — every goroutine in the pool abandons its blocking -// send in that case, so the consumer sees a truncated but properly -// terminated stream instead of a stalled one. +// event per per-path error, and one event per directory -x or excludes +// kept the walk from entering. The channel is closed when the walk +// completes, and also when ctx is cancelled — every goroutine in the +// pool abandons its blocking send in that case, so the consumer sees a +// truncated but properly terminated stream instead of a stalled one. func startWalk(ctx context.Context, roots []string, oneFS bool, - workers int, + excludes map[string]string, workers int, ) <-chan walkEvent { - jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS) + jobs, subdirs, events := startWalkWorkers(ctx, workers, oneFS, excludes) go func() { initial := make([]dirJob, 0, len(roots)) @@ -1177,7 +1222,7 @@ func seedRoot(ctx context.Context, root string, // workers drop the directories still queued rather than stop reading // jobs, so the range always runs out and the pool always tears down. func startWalkWorkers(ctx context.Context, workers int, - oneFS bool, + oneFS bool, excludes map[string]string, ) (chan dirJob, chan []dirJob, chan walkEvent) { jobs := make(chan dirJob, workQueueDepth) subdirs := make(chan []dirJob, workers) @@ -1193,7 +1238,7 @@ func startWalkWorkers(ctx context.Context, workers int, } select { - case subdirs <- walkOneDir(ctx, job, oneFS, events): + case subdirs <- walkOneDir(ctx, job, oneFS, excludes, events): case <-ctx.Done(): } } @@ -1251,7 +1296,7 @@ func dispatchDirs(ctx context.Context, initial []dirJob, // entry and a warning event per unreadable one, and returns the // subdirectories to descend into. func walkOneDir(ctx context.Context, job dirJob, oneFS bool, - events chan<- walkEvent, + excludes map[string]string, events chan<- walkEvent, ) []dirJob { entries, err := os.ReadDir(job.path) if err != nil { @@ -1275,7 +1320,8 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool, p := filepath.Join(job.path, e.Name()) if e.IsDir() { - if sub, ok := subdirJob(ctx, p, e, job, oneFS, events); ok { + sub, ok := subdirJob(ctx, p, e, job, oneFS, excludes, events) + if ok { subs = append(subs, sub) } @@ -1328,17 +1374,28 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, // subdirJob applies the descent rules to directory p: never enter // .zfs (ZFS snapshot pseudo-dirs would list every file once per -// snapshot), and with -x never enter a directory on a different -// filesystem than its operand, sending an event that names it so the +// snapshot), never enter a default exclude in excludes, and with -x +// never enter a directory on a different filesystem than its operand. +// For the last two it sends an event that names the directory, so the // scan keeps the records below it, except those below an operand at or // inside it. func subdirJob(ctx context.Context, p string, e fs.DirEntry, - parent dirJob, oneFS bool, events chan<- walkEvent, + parent dirJob, oneFS bool, excludes map[string]string, + events chan<- walkEvent, ) (dirJob, bool) { if e.Name() == ".zfs" { return dirJob{}, false } + if why, ok := excludes[p]; ok { + sendEvent(ctx, events, walkEvent{ + excludedDir: p, + warn: fmt.Sprintf("scan: skipped %s (%s)", p, why), + }) + + return dirJob{}, false + } + job := dirJob{path: p, rootDev: parent.rootDev, rootDevOK: parent.rootDevOK} if !oneFS || !parent.rootDevOK { return job, true diff --git a/scan_test.go b/scan_test.go index 1cfe8f2..f1ba3ae 100644 --- a/scan_test.go +++ b/scan_test.go @@ -782,7 +782,7 @@ func collectWalk(t *testing.T, roots []string, oneFS bool, errs int ) - for ev := range startWalk(t.Context(), roots, oneFS, workers) { + for ev := range startWalk(t.Context(), roots, oneFS, nil, workers) { if ev.fail { errs++ @@ -1004,7 +1004,7 @@ func callSubdirJob(t *testing.T, p string, e fs.DirEntry, t.Helper() events := make(chan walkEvent, 1) - job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events) + job, ok := subdirJob(t.Context(), p, e, parent, oneFS, nil, events) close(events) var evs []walkEvent @@ -1152,7 +1152,7 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, return dev, ok } - roots = pruneRoots(roots, true, device) + roots = pruneRoots(roots, true, nil, device) ctx := t.Context() s := &scanState{db: db} @@ -1162,7 +1162,7 @@ func syncOneFilesystem(t *testing.T, db *sql.DB, roots []string, t.Fatal(err) } - jobs, subdirs, events := startWalkWorkers(ctx, 2, true) + jobs, subdirs, events := startWalkWorkers(ctx, 2, true, nil) // seedRoot emits a regular-file operand itself and returns a job // only for a directory operand. @@ -1463,7 +1463,7 @@ const smokeTreeFiles = 15 func syncTree(t *testing.T, db *sql.DB, roots ...string) scanStats { t.Helper() - st, err := syncScan(t.Context(), db, roots, 4, false) + st, err := syncScan(t.Context(), db, roots, 4, false, nil) if err != nil { t.Fatalf("syncScan: %v", err) } @@ -2412,7 +2412,7 @@ func TestPruneRoots(t *testing.T) { // Duplicates and operands under other operands are dropped; /cc is // not under /c (sibling with a shared prefix). got := pruneRoots([]string{"/a/b", "/a", "/c", "/a", "/a/b/c", "/cc"}, - false, device) + false, nil, device) want := []string{"/a", "/c", "/cc"} if !slices.Equal(got, want) { @@ -2421,7 +2421,7 @@ func TestPruneRoots(t *testing.T) { nested := []string{"/p/q/r", pqRoot, "/p", "/p", "/p/s", "/p/t"} - got = pruneRoots(nested, false, device) + got = pruneRoots(nested, false, nil, device) want = []string{"/p"} if !slices.Equal(got, want) { @@ -2429,12 +2429,20 @@ func TestPruneRoots(t *testing.T) { } // With -x, /p/q, on another filesystem than /p, is kept as well. - got = pruneRoots(nested, true, device) + got = pruneRoots(nested, true, nil, device) want = []string{"/p", pqRoot} if !slices.Equal(got, want) { t.Fatalf("pruneRoots with -x = %q, want %q", got, want) } + + // The walk of /p does not enter /p/q when it is a default exclude, so + // /p/q is kept, without -x too; /p/q/r lies under it and is dropped. + got = pruneRoots(nested, false, + map[string]string{pqRoot: defaultExcludeWhy}, device) + if !slices.Equal(got, want) { + t.Fatalf("pruneRoots excluding %s = %q, want %q", pqRoot, got, want) + } } func TestReportsNeverTouchFilesystem(t *testing.T) {