From 8ff9353da0e14ceb483300104ce725b323245eb1 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 05:41:30 +0000 Subject: [PATCH] Leave caches out of the walk by default (closes #181) The default excludes now cover caches that programs and package managers fill again on their own. On Linux: /var/cache and, for root and each user under /home, .cache, .npm, .cargo/registry and go/pkg/mod. On macOS: /private/var/folders, /Library/Caches and, for each user, Library/Caches plus the same four. README and scan --help list them; --no-default-excludes still turns them all off. Model: opus-5-5 --- README.md | 34 +++++++++++++++++++++----------- TODO.md | 6 ++++++ internal/sfdupes/exclude.go | 17 ++++++++++++++-- internal/sfdupes/exclude_test.go | 33 ++++++++++++++++++++++++++++++- 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index dc51fc8..4beaab0 100644 --- a/README.md +++ b/README.md @@ -129,8 +129,8 @@ walk"). As the file `/etc/cron.d/sfdupes`: - one line naming the database; - progress lines, as described under "Progress"; - a warning for each path it skips; - - a line for each system or temporary directory the walk leaves out (see - "Rules for the walk"); + - a line for each system, temporary or cache directory the walk leaves out + (see "Rules for the walk"); - a line for each path left out because its filesystem is one the kernel, the core of the operating system, makes up or keeps in memory, such as `tmpfs`; @@ -302,8 +302,9 @@ The reference below is also the program's specification. directories on other filesystems, as the same flag does for `du` and `rsync`. By default the walk also enters directories on other filesystems. - `--no-default-excludes` (`scan`) lets the walk enter the default excludes. - Those are the system and temporary directories the walk otherwise leaves out, - such as `/var/tmp` on Linux (see "Rules for the walk"). + Those are the system, temporary and cache directories the walk otherwise + leaves out, such as `/var/tmp` and `/var/cache` on Linux (see "Rules for the + walk"). - `--exclude PATH` (`scan`) keeps the walk out of `PATH` and everything below it. `PATH` may be a directory or a file. The flag may be given more than once. The walk knows `PATH` by its device ID and inode number, as it knows the @@ -570,13 +571,18 @@ walk practical on large or busy storage pools. it, as described under "What a scan does to the database". `--no-default-excludes` does not change this rule. -- Below an operand, the walk does not enter a default exclude. On Linux, the - default excludes hold temporary files and the kernel's view of the running - system. They depend on the system, and `sfdupes scan --help` lists those of - the system it runs on. On Linux they are: +- Below an operand, the walk does not enter a default exclude. The default + excludes hold the kernel's view of the running system, temporary files, and + caches. A cache here is data that programs and package managers download or + build again on their own. The default excludes depend on the system, and + `sfdupes scan --help` lists those of the system it runs on. In the lists, `*` + stands for any one directory name, so `/home/*/.cache` covers each user's + cache. On Linux they are: ``` - /proc, /sys, /dev, /run, /tmp, /var/tmp + /proc, /sys, /dev, /run, /tmp, /var/tmp, /var/cache + /root/.cache, /root/.npm, /root/.cargo/registry, /root/go/pkg/mod + /home/*/.cache, /home/*/.npm, /home/*/.cargo/registry, /home/*/go/pkg/mod ``` On macOS they are: @@ -584,11 +590,15 @@ walk practical on large or busy storage pools. ``` /dev, /private/tmp, /private/var/tmp, /private/var/vm /System/Volumes/VM, /System/Volumes/Data + /private/var/folders, /Library/Caches + /Users/*/Library/Caches, /Users/*/.cache, /Users/*/.npm + /Users/*/.cargo/registry, /Users/*/go/pkg/mod ``` - A walk of `/` on macOS reaches everything in `/System/Volumes/Data` already, - through firmlinks. Firmlinks are the links macOS uses to show the data - volume inside the system volume. Walking `/System/Volumes/Data` as well + `/private/var/folders` holds each user's temporary files and caches on + macOS. A walk of `/` on macOS reaches everything in `/System/Volumes/Data` + already, through firmlinks. Firmlinks are the links macOS uses to show the + data volume inside the system volume. Walking `/System/Volumes/Data` as well would record each of those files twice. Any other system has no default excludes. diff --git a/TODO.md b/TODO.md index 8fb200d..a495b8a 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,12 @@ Newest first. An entry ends with its date and its issue or branch, where those are known. +- The default excludes cover caches as well: `/var/cache`, and each user's + `.cache`, `.npm`, `.cargo/registry` and `go/pkg/mod`, root's included. On + macOS they add `/private/var/folders`, `/Library/Caches` and each user's + `Library/Caches` and the same four. `--no-default-excludes` still turns them + all off (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/181) + - New `scan` flag `--exclude PATH`, which may be repeated. The walk neither enters nor records an excluded path, and keeps the records below it. An operand at or below one is a fatal error (2026-11-11, diff --git a/internal/sfdupes/exclude.go b/internal/sfdupes/exclude.go index 8d57da1..39022fa 100644 --- a/internal/sfdupes/exclude.go +++ b/internal/sfdupes/exclude.go @@ -72,14 +72,27 @@ type walkRules struct { func defaultExcludes(goos string) []string { switch goos { case goosLinux: - return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"} + // The kernel's view of the running system, temporary files, and + // caches that programs and package managers fill again on their + // own: the system's, root's and each user's. + return []string{ + "/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache", + "/root/.cache", "/root/.npm", "/root/.cargo/registry", + "/root/go/pkg/mod", + "/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry", + "/home/*/go/pkg/mod", + } case goosDarwin: // A walk of / reaches everything in /System/Volumes/Data through // the firmlinks, so walking it as well would record each of - // those files twice. + // those files twice. /private/var/folders holds each user's + // temporary files and caches. return []string{ "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", "/System/Volumes/VM", "/System/Volumes/Data", + "/private/var/folders", "/Library/Caches", + "/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm", + "/Users/*/.cargo/registry", "/Users/*/go/pkg/mod", } case goosWindows: // Relative to the root of the drive an operand lies on. The diff --git a/internal/sfdupes/exclude_test.go b/internal/sfdupes/exclude_test.go index f3262f8..fd28963 100644 --- a/internal/sfdupes/exclude_test.go +++ b/internal/sfdupes/exclude_test.go @@ -24,10 +24,19 @@ func TestDefaultExcludes(t *testing.T) { // on any other system. The Windows list exists in the code only: the // binary does not build for Windows, so the README leaves it out. cases := map[string][]string{ - goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}, + goosLinux: { + "/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache", + "/root/.cache", "/root/.npm", "/root/.cargo/registry", + "/root/go/pkg/mod", + "/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry", + "/home/*/go/pkg/mod", + }, goosDarwin: { "/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm", "/System/Volumes/VM", "/System/Volumes/Data", + "/private/var/folders", "/Library/Caches", + "/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm", + "/Users/*/.cargo/registry", "/Users/*/go/pkg/mod", }, goosWindows: { "pagefile.sys", "swapfile.sys", "hiberfil.sys", @@ -63,6 +72,28 @@ func TestIsDefaultExclude(t *testing.T) { t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want) } } + + // The Linux list matches each user's cache directory itself, and no + // .cache directory elsewhere. + linux := map[string]bool{ + "/home/u/.cache": true, + "/home/u/go/pkg/mod": true, + "/root/.cache": true, + "/var/cache": true, + "/home/u/.cache/pip": false, + "/home/u/src/.cache": false, + "/srv/home/u/.cache": false, + "/home/u/.cargo/bin": false, + "/home/u/go/pkg/mod/cache": false, + } + + for path, want := range linux { + got := isDefaultExclude(path, defaultExcludes(goosLinux)) + if got != want { + t.Errorf("isDefaultExclude(%q, Linux list) = %v, want %v", path, + got, want) + } + } } func TestReadMountInfo(t *testing.T) {