diff --git a/README.md b/README.md index 0a36a64..f7f8adc 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ walk"). As the file `/etc/cron.d/sfdupes`: - one line naming the database; - progress lines, as described under "Progress"; - a warning for each path it skips; - - a line for each system or temporary directory the walk leaves out (see - "Rules for the walk"); + - a line for each system, temporary or cache directory the walk leaves out + (see "Rules for the walk"); - a line for each path left out because its filesystem is one the kernel, the core of the operating system, makes up or keeps in memory, such as `tmpfs`; @@ -300,8 +300,9 @@ The reference below is also the program's specification. of directories on other filesystems, as the same flag does for `du` and `rsync`. By default the walk also enters directories on other filesystems. - `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default - excludes. Those are the system and temporary directories the walk otherwise - leaves out, such as `/var/tmp` on Linux (see "Rules for the walk"). + excludes. Those are the system, temporary and cache directories the walk + otherwise leaves out, such as `/var/tmp` and `/var/cache` on Linux (see "Rules + for the walk"). - `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything below it. `PATH` may be a directory or a file. The flag may be given more than once. The walk knows `PATH` by its device ID and inode number, as it knows the @@ -571,12 +572,17 @@ walk practical on large or busy storage pools. `--no-default-excludes` does not change this rule. - Below an operand, the walk does not enter a default exclude. On Linux, the - default excludes hold temporary files and the kernel's view of the running - system. They depend on the system, and `sfdupes scan --help` lists those of - the system it runs on. On Linux they are: + default excludes hold the kernel's view of the running system, temporary + files, and caches. A cache here is data that programs and package managers + download or build again on their own. The default excludes depend on the + system, and `sfdupes scan --help` lists those of the system it runs on. In the + lists, `*` stands for any one directory name, so `/home/*/.cache` matches + `/home/alice/.cache`. On Linux they are: ``` - /proc, /sys, /dev, /run, /tmp, /var/tmp + /proc, /sys, /dev, /run, /tmp, /var/tmp, /var/cache + /root/.cache, /root/.npm, /root/.cargo/registry, /root/go/pkg/mod + /home/*/.cache, /home/*/.npm, /home/*/.cargo/registry, /home/*/go/pkg/mod ``` On macOS they are: @@ -584,13 +590,21 @@ walk practical on large or busy storage pools. ``` /dev, /private/tmp, /private/var/tmp, /private/var/vm /System/Volumes/VM, /System/Volumes/Data + /private/var/folders, /Library/Caches + /private/var/root/Library/Caches, /private/var/root/.cache + /private/var/root/.npm, /private/var/root/.cargo/registry + /private/var/root/go/pkg/mod + /Users/*/Library/Caches, /Users/*/.cache, /Users/*/.npm + /Users/*/.cargo/registry, /Users/*/go/pkg/mod ``` - A walk of `/` on macOS reaches everything in `/System/Volumes/Data` already, - through firmlinks. Firmlinks are the links macOS uses to show the data - volume inside the system volume. Walking `/System/Volumes/Data` as well - would record each of those files twice. Any other system has no default - excludes. + `/private/var/folders` holds each user's temporary files and caches on + macOS. Root's home on macOS is `/var/root`, which the walk reaches as + `/private/var/root`. A walk of `/` on macOS reaches everything in + `/System/Volumes/Data` already, through firmlinks. Firmlinks are the links + macOS uses to show the data volume inside the system volume. Walking + `/System/Volumes/Data` as well would record each of those files twice. Any + other system has no default excludes. An operand at or inside a default exclude is walked, so `sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem. diff --git a/TODO.md b/TODO.md index f2bafd6..f640646 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ Newest first. An entry ends with its date and its issue or branch, where those are known. +- The default excludes cover caches as well. On Linux they add `/var/cache`. In + `/root` and in each home in `/home` they add `.cache`, `.npm`, + `.cargo/registry` and `go/pkg/mod`. On macOS they add `/private/var/folders` + and `/Library/Caches`. In `/private/var/root` and in each home in `/Users` + they add `Library/Caches` and the same four. `--no-default-excludes` still + turns them all off (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/181) + - `dwim` takes the `scan` flags `--workers`, `-x`, `--no-default-excludes` and `--exclude`, with the same meaning. Both commands register and check them with the same code (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/162) diff --git a/internal/sfdupes/exclude.go b/internal/sfdupes/exclude.go index 195ac2f..da6a74a 100644 --- a/internal/sfdupes/exclude.go +++ b/internal/sfdupes/exclude.go @@ -72,14 +72,31 @@ type walkRules struct { func defaultExcludes(goos string) []string { switch goos { case goosLinux: - return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"} + // The kernel's view of the running system, temporary files, and + // caches that programs and package managers fill again on their + // own: the system's, root's and those in each home in /home. + return []string{ + "/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache", + "/root/.cache", "/root/.npm", "/root/.cargo/registry", + "/root/go/pkg/mod", + "/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry", + "/home/*/go/pkg/mod", + } case goosDarwin: // A walk of / reaches everything in /System/Volumes/Data through // the firmlinks, so walking it as well would record each of - // those files twice. + // those files twice. /private/var/folders holds each user's + // temporary files and caches. The walk reaches root's home, + // /var/root, as /private/var/root. return []string{ "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", "/System/Volumes/VM", "/System/Volumes/Data", + "/private/var/folders", "/Library/Caches", + "/private/var/root/Library/Caches", "/private/var/root/.cache", + "/private/var/root/.npm", "/private/var/root/.cargo/registry", + "/private/var/root/go/pkg/mod", + "/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm", + "/Users/*/.cargo/registry", "/Users/*/go/pkg/mod", } case goosWindows: // Relative to the root of the drive an operand lies on. The diff --git a/internal/sfdupes/exclude_test.go b/internal/sfdupes/exclude_test.go index f3262f8..782bf34 100644 --- a/internal/sfdupes/exclude_test.go +++ b/internal/sfdupes/exclude_test.go @@ -24,10 +24,22 @@ func TestDefaultExcludes(t *testing.T) { // on any other system. The Windows list exists in the code only: the // binary does not build for Windows, so the README leaves it out. cases := map[string][]string{ - goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, + goosLinux: { + "/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache", + "/root/.cache", "/root/.npm", "/root/.cargo/registry", + "/root/go/pkg/mod", + "/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry", + "/home/*/go/pkg/mod", + }, goosDarwin: { "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", "/System/Volumes/VM", "/System/Volumes/Data", + "/private/var/folders", "/Library/Caches", + "/private/var/root/Library/Caches", "/private/var/root/.cache", + "/private/var/root/.npm", "/private/var/root/.cargo/registry", + "/private/var/root/go/pkg/mod", + "/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm", + "/Users/*/.cargo/registry", "/Users/*/go/pkg/mod", }, goosWindows: { "pagefile.sys", "swapfile.sys", "hiberfil.sys", @@ -63,6 +75,29 @@ func TestIsDefaultExclude(t *testing.T) { t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want) } } + + // The Linux list matches the cache directories in /root and in each + // home in /home, but nothing below them and no .cache directory + // elsewhere. + linux := map[string]bool{ + "/home/u/.cache": true, + "/home/u/go/pkg/mod": true, + "/root/.cache": true, + "/var/cache": true, + "/home/u/.cache/pip": false, + "/home/u/src/.cache": false, + "/srv/home/u/.cache": false, + "/home/u/.cargo/bin": false, + "/home/u/go/pkg/mod/cache": false, + } + + for path, want := range linux { + got := isDefaultExclude(path, defaultExcludes(goosLinux)) + if got != want { + t.Errorf("isDefaultExclude(%q, Linux list) = %v, want %v", path, + got, want) + } + } } func TestReadMountInfo(t *testing.T) { @@ -290,7 +325,7 @@ func TestRunScanTempDirOperand(t *testing.T) { t.Setenv(databaseEnv, path) dir := realTempDir(t) - if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) { + if !insideDefaultExclude(dir, defaultExcludes(runtime.GOOS)) { t.Skip(dir + " is not inside a default exclude") } @@ -711,6 +746,20 @@ func shmTempDir(t *testing.T) (string, string, walkRules) { return dir, fsType, walkRules{nonPersistent: nonPersistent} } +// insideDefaultExclude reports whether path, which is absolute and +// clean, or a directory above it is one of excludes. +func insideDefaultExclude(path string, excludes []string) bool { + for dir := path; ; dir = filepath.Dir(dir) { + if isDefaultExclude(dir, excludes) { + return true + } + + if dir == "/" { + return false + } + } +} + // assertRecorded checks that the database at dbPath holds a record for // path when want is set, and none when it is not. func assertRecorded(t *testing.T, dbPath, path string, want bool) {