Leave caches out of the walk by default (closes #181) #192
No files matched your search
@@ -128,8 +128,8 @@ walk"). As the file `/etc/cron.d/sfdupes`:
|
|||||||
- one line naming the database;
|
- one line naming the database;
|
||||||
- progress lines, as described under "Progress";
|
- progress lines, as described under "Progress";
|
||||||
- a warning for each path it skips;
|
- a warning for each path it skips;
|
||||||
- a line for each system or temporary directory the walk leaves out (see
|
- a line for each system, temporary or cache directory the walk leaves out
|
||||||
"Rules for the walk");
|
(see "Rules for the walk");
|
||||||
- a line for each path left out because its filesystem is one the kernel,
|
- a line for each path left out because its filesystem is one the kernel,
|
||||||
the core of the operating system, makes up or keeps in memory, such as
|
the core of the operating system, makes up or keeps in memory, such as
|
||||||
`tmpfs`;
|
`tmpfs`;
|
||||||
@@ -300,8 +300,9 @@ The reference below is also the program's specification.
|
|||||||
of directories on other filesystems, as the same flag does for `du` and
|
of directories on other filesystems, as the same flag does for `du` and
|
||||||
`rsync`. By default the walk also enters directories on other filesystems.
|
`rsync`. By default the walk also enters directories on other filesystems.
|
||||||
- `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
|
- `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
|
||||||
excludes. Those are the system and temporary directories the walk otherwise
|
excludes. Those are the system, temporary and cache directories the walk
|
||||||
leaves out, such as `/var/tmp` on Linux (see "Rules for the walk").
|
otherwise leaves out, such as `/var/tmp` and `/var/cache` on Linux (see "Rules
|
||||||
|
for the walk").
|
||||||
- `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
|
- `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
|
||||||
below it. `PATH` may be a directory or a file. The flag may be given more than
|
below it. `PATH` may be a directory or a file. The flag may be given more than
|
||||||
once. The walk knows `PATH` by its device ID and inode number, as it knows the
|
once. The walk knows `PATH` by its device ID and inode number, as it knows the
|
||||||
@@ -571,12 +572,17 @@ walk practical on large or busy storage pools.
|
|||||||
`--no-default-excludes` does not change this rule.
|
`--no-default-excludes` does not change this rule.
|
||||||
|
|
||||||
- Below an operand, the walk does not enter a default exclude. On Linux, the
|
- Below an operand, the walk does not enter a default exclude. On Linux, the
|
||||||
default excludes hold temporary files and the kernel's view of the running
|
default excludes hold the kernel's view of the running system, temporary
|
||||||
system. They depend on the system, and `sfdupes scan --help` lists those of
|
files, and caches. A cache here is data that programs and package managers
|
||||||
the system it runs on. On Linux they are:
|
download or build again on their own. The default excludes depend on the
|
||||||
|
system, and `sfdupes scan --help` lists those of the system it runs on. In the
|
||||||
|
lists, `*` stands for any one directory name, so `/home/*/.cache` matches
|
||||||
|
`/home/alice/.cache`. On Linux they are:
|
||||||
|
|
||||||
```
|
```
|
||||||
/proc, /sys, /dev, /run, /tmp, /var/tmp
|
/proc, /sys, /dev, /run, /tmp, /var/tmp, /var/cache
|
||||||
|
/root/.cache, /root/.npm, /root/.cargo/registry, /root/go/pkg/mod
|
||||||
|
/home/*/.cache, /home/*/.npm, /home/*/.cargo/registry, /home/*/go/pkg/mod
|
||||||
```
|
```
|
||||||
|
|
||||||
On macOS they are:
|
On macOS they are:
|
||||||
@@ -584,13 +590,21 @@ walk practical on large or busy storage pools.
|
|||||||
```
|
```
|
||||||
/dev, /private/tmp, /private/var/tmp, /private/var/vm
|
/dev, /private/tmp, /private/var/tmp, /private/var/vm
|
||||||
/System/Volumes/VM, /System/Volumes/Data
|
/System/Volumes/VM, /System/Volumes/Data
|
||||||
|
/private/var/folders, /Library/Caches
|
||||||
|
/private/var/root/Library/Caches, /private/var/root/.cache
|
||||||
|
/private/var/root/.npm, /private/var/root/.cargo/registry
|
||||||
|
/private/var/root/go/pkg/mod
|
||||||
|
/Users/*/Library/Caches, /Users/*/.cache, /Users/*/.npm
|
||||||
|
/Users/*/.cargo/registry, /Users/*/go/pkg/mod
|
||||||
```
|
```
|
||||||
|
|
||||||
A walk of `/` on macOS reaches everything in `/System/Volumes/Data` already,
|
`/private/var/folders` holds each user's temporary files and caches on
|
||||||
through firmlinks. Firmlinks are the links macOS uses to show the data
|
macOS. Root's home on macOS is `/var/root`, which the walk reaches as
|
||||||
volume inside the system volume. Walking `/System/Volumes/Data` as well
|
`/private/var/root`. A walk of `/` on macOS reaches everything in
|
||||||
would record each of those files twice. Any other system has no default
|
`/System/Volumes/Data` already, through firmlinks. Firmlinks are the links
|
||||||
excludes.
|
macOS uses to show the data volume inside the system volume. Walking
|
||||||
|
`/System/Volumes/Data` as well would record each of those files twice. Any
|
||||||
|
other system has no default excludes.
|
||||||
|
|
||||||
An operand at or inside a default exclude is walked, so
|
An operand at or inside a default exclude is walked, so
|
||||||
`sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem.
|
`sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem.
|
||||||
|
|||||||
@@ -72,14 +72,31 @@ type walkRules struct {
|
|||||||
func defaultExcludes(goos string) []string {
|
func defaultExcludes(goos string) []string {
|
||||||
switch goos {
|
switch goos {
|
||||||
case goosLinux:
|
case goosLinux:
|
||||||
return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
|
// The kernel's view of the running system, temporary files, and
|
||||||
|
// caches that programs and package managers fill again on their
|
||||||
|
// own: the system's, root's and those in each home in /home.
|
||||||
|
return []string{
|
||||||
|
"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
|
||||||
|
"/root/.cache", "/root/.npm", "/root/.cargo/registry",
|
||||||
|
"/root/go/pkg/mod",
|
||||||
|
"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
|
||||||
|
"/home/*/go/pkg/mod",
|
||||||
|
}
|
||||||
case goosDarwin:
|
case goosDarwin:
|
||||||
// A walk of / reaches everything in /System/Volumes/Data through
|
// A walk of / reaches everything in /System/Volumes/Data through
|
||||||
// the firmlinks, so walking it as well would record each of
|
// the firmlinks, so walking it as well would record each of
|
||||||
// those files twice.
|
// those files twice. /private/var/folders holds each user's
|
||||||
|
// temporary files and caches. The walk reaches root's home,
|
||||||
|
// /var/root, as /private/var/root.
|
||||||
return []string{
|
return []string{
|
||||||
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
||||||
"/System/Volumes/VM", "/System/Volumes/Data",
|
"/System/Volumes/VM", "/System/Volumes/Data",
|
||||||
|
"/private/var/folders", "/Library/Caches",
|
||||||
|
"/private/var/root/Library/Caches", "/private/var/root/.cache",
|
||||||
|
"/private/var/root/.npm", "/private/var/root/.cargo/registry",
|
||||||
|
"/private/var/root/go/pkg/mod",
|
||||||
|
"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
|
||||||
|
"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
|
||||||
}
|
}
|
||||||
case goosWindows:
|
case goosWindows:
|
||||||
// Relative to the root of the drive an operand lies on. The
|
// Relative to the root of the drive an operand lies on. The
|
||||||
|
|||||||
@@ -24,10 +24,22 @@ func TestDefaultExcludes(t *testing.T) {
|
|||||||
// on any other system. The Windows list exists in the code only: the
|
// on any other system. The Windows list exists in the code only: the
|
||||||
// binary does not build for Windows, so the README leaves it out.
|
// binary does not build for Windows, so the README leaves it out.
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
goosLinux: {
|
||||||
|
"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
|
||||||
|
"/root/.cache", "/root/.npm", "/root/.cargo/registry",
|
||||||
|
"/root/go/pkg/mod",
|
||||||
|
"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
|
||||||
|
"/home/*/go/pkg/mod",
|
||||||
|
},
|
||||||
goosDarwin: {
|
goosDarwin: {
|
||||||
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
||||||
"/System/Volumes/VM", "/System/Volumes/Data",
|
"/System/Volumes/VM", "/System/Volumes/Data",
|
||||||
|
"/private/var/folders", "/Library/Caches",
|
||||||
|
"/private/var/root/Library/Caches", "/private/var/root/.cache",
|
||||||
|
"/private/var/root/.npm", "/private/var/root/.cargo/registry",
|
||||||
|
"/private/var/root/go/pkg/mod",
|
||||||
|
"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
|
||||||
|
"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
|
||||||
},
|
},
|
||||||
goosWindows: {
|
goosWindows: {
|
||||||
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
|
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
|
||||||
@@ -63,6 +75,29 @@ func TestIsDefaultExclude(t *testing.T) {
|
|||||||
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
|
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The Linux list matches the cache directories in /root and in each
|
||||||
|
// home in /home, but nothing below them and no .cache directory
|
||||||
|
// elsewhere.
|
||||||
|
linux := map[string]bool{
|
||||||
|
"/home/u/.cache": true,
|
||||||
|
"/home/u/go/pkg/mod": true,
|
||||||
|
"/root/.cache": true,
|
||||||
|
"/var/cache": true,
|
||||||
|
"/home/u/.cache/pip": false,
|
||||||
|
"/home/u/src/.cache": false,
|
||||||
|
"/srv/home/u/.cache": false,
|
||||||
|
"/home/u/.cargo/bin": false,
|
||||||
|
"/home/u/go/pkg/mod/cache": false,
|
||||||
|
}
|
||||||
|
|
||||||
|
for path, want := range linux {
|
||||||
|
got := isDefaultExclude(path, defaultExcludes(goosLinux))
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("isDefaultExclude(%q, Linux list) = %v, want %v", path,
|
||||||
|
got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReadMountInfo(t *testing.T) {
|
func TestReadMountInfo(t *testing.T) {
|
||||||
@@ -290,7 +325,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
|
|||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := realTempDir(t)
|
||||||
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
if !insideDefaultExclude(dir, defaultExcludes(runtime.GOOS)) {
|
||||||
t.Skip(dir + " is not inside a default exclude")
|
t.Skip(dir + " is not inside a default exclude")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -711,6 +746,20 @@ func shmTempDir(t *testing.T) (string, string, walkRules) {
|
|||||||
return dir, fsType, walkRules{nonPersistent: nonPersistent}
|
return dir, fsType, walkRules{nonPersistent: nonPersistent}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// insideDefaultExclude reports whether path, which is absolute and
|
||||||
|
// clean, or a directory above it is one of excludes.
|
||||||
|
func insideDefaultExclude(path string, excludes []string) bool {
|
||||||
|
for dir := path; ; dir = filepath.Dir(dir) {
|
||||||
|
if isDefaultExclude(dir, excludes) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if dir == "/" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// assertRecorded checks that the database at dbPath holds a record for
|
// assertRecorded checks that the database at dbPath holds a record for
|
||||||
// path when want is set, and none when it is not.
|
// path when want is set, and none when it is not.
|
||||||
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
|
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
|
||||||
|
|||||||
Reference in new issue
Block a user