Compare commits
3
Commits
049a74a10b
..
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e16c46804d | ||
|
|
317abb834c | ||
|
|
652df1c382 |
No files matched your search
@@ -136,8 +136,8 @@ point of each disk at or below `/srv` with its own `mountpoint -q`, joined with
|
|||||||
- one line naming the database;
|
- one line naming the database;
|
||||||
- progress lines, as described under "Progress";
|
- progress lines, as described under "Progress";
|
||||||
- a warning for each path it skips;
|
- a warning for each path it skips;
|
||||||
- a line for each system or temporary directory the walk leaves out (see
|
- a line for each system, temporary or cache directory the walk leaves out
|
||||||
"Rules for the walk");
|
(see "Rules for the walk");
|
||||||
- a line for each path left out because its filesystem is one the kernel,
|
- a line for each path left out because its filesystem is one the kernel,
|
||||||
the core of the operating system, makes up or keeps in memory, such as
|
the core of the operating system, makes up or keeps in memory, such as
|
||||||
`tmpfs`;
|
`tmpfs`;
|
||||||
@@ -309,8 +309,9 @@ The reference below is also the program's specification.
|
|||||||
of directories on other filesystems, as the same flag does for `du` and
|
of directories on other filesystems, as the same flag does for `du` and
|
||||||
`rsync`. By default the walk also enters directories on other filesystems.
|
`rsync`. By default the walk also enters directories on other filesystems.
|
||||||
- `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
|
- `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
|
||||||
excludes. Those are the system and temporary directories the walk otherwise
|
excludes. Those are the system, temporary and cache directories the walk
|
||||||
leaves out, such as `/var/tmp` on Linux (see "Rules for the walk").
|
otherwise leaves out, such as `/var/tmp` and `/var/cache` on Linux (see "Rules
|
||||||
|
for the walk").
|
||||||
- `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
|
- `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
|
||||||
below it. `PATH` may be a directory or a file. The flag may be given more than
|
below it. `PATH` may be a directory or a file. The flag may be given more than
|
||||||
once. The walk knows `PATH` by its device ID and inode number, as it knows the
|
once. The walk knows `PATH` by its device ID and inode number, as it knows the
|
||||||
@@ -562,7 +563,7 @@ walk practical on large or busy storage pools.
|
|||||||
`scan` prints a line on stderr naming the path and its filesystem type:
|
`scan` prints a line on stderr naming the path and its filesystem type:
|
||||||
|
|
||||||
```
|
```
|
||||||
scan: skipped /dev/shm (tmpfs, not a persistent filesystem)
|
scan: left out /dev/shm (tmpfs, not a persistent filesystem)
|
||||||
```
|
```
|
||||||
|
|
||||||
The path is not counted as skipped. The records below it are kept, as
|
The path is not counted as skipped. The records below it are kept, as
|
||||||
@@ -572,7 +573,7 @@ walk practical on large or busy storage pools.
|
|||||||
For each one it reaches, `scan` prints a line on stderr naming it:
|
For each one it reaches, `scan` prints a line on stderr naming it:
|
||||||
|
|
||||||
```
|
```
|
||||||
scan: skipped /srv/cache (--exclude)
|
scan: left out /srv/cache (--exclude)
|
||||||
```
|
```
|
||||||
|
|
||||||
The path is not counted as skipped. The scan keeps the records at and below
|
The path is not counted as skipped. The scan keeps the records at and below
|
||||||
@@ -580,12 +581,17 @@ walk practical on large or busy storage pools.
|
|||||||
`--no-default-excludes` does not change this rule.
|
`--no-default-excludes` does not change this rule.
|
||||||
|
|
||||||
- Below an operand, the walk does not enter a default exclude. On Linux, the
|
- Below an operand, the walk does not enter a default exclude. On Linux, the
|
||||||
default excludes hold temporary files and the kernel's view of the running
|
default excludes hold the kernel's view of the running system, temporary
|
||||||
system. They depend on the system, and `sfdupes scan --help` lists those of
|
files, and caches. A cache here is data that programs and package managers
|
||||||
the system it runs on. On Linux they are:
|
download or build again on their own. The default excludes depend on the
|
||||||
|
system, and `sfdupes scan --help` lists those of the system it runs on. In the
|
||||||
|
lists, `*` stands for any one directory name, so `/home/*/.cache` matches
|
||||||
|
`/home/alice/.cache`. On Linux they are:
|
||||||
|
|
||||||
```
|
```
|
||||||
/proc, /sys, /dev, /run, /tmp, /var/tmp
|
/proc, /sys, /dev, /run, /tmp, /var/tmp, /var/cache
|
||||||
|
/root/.cache, /root/.npm, /root/.cargo/registry, /root/go/pkg/mod
|
||||||
|
/home/*/.cache, /home/*/.npm, /home/*/.cargo/registry, /home/*/go/pkg/mod
|
||||||
```
|
```
|
||||||
|
|
||||||
On macOS they are:
|
On macOS they are:
|
||||||
@@ -593,13 +599,21 @@ walk practical on large or busy storage pools.
|
|||||||
```
|
```
|
||||||
/dev, /private/tmp, /private/var/tmp, /private/var/vm
|
/dev, /private/tmp, /private/var/tmp, /private/var/vm
|
||||||
/System/Volumes/VM, /System/Volumes/Data
|
/System/Volumes/VM, /System/Volumes/Data
|
||||||
|
/private/var/folders, /Library/Caches
|
||||||
|
/private/var/root/Library/Caches, /private/var/root/.cache
|
||||||
|
/private/var/root/.npm, /private/var/root/.cargo/registry
|
||||||
|
/private/var/root/go/pkg/mod
|
||||||
|
/Users/*/Library/Caches, /Users/*/.cache, /Users/*/.npm
|
||||||
|
/Users/*/.cargo/registry, /Users/*/go/pkg/mod
|
||||||
```
|
```
|
||||||
|
|
||||||
A walk of `/` on macOS reaches everything in `/System/Volumes/Data` already,
|
`/private/var/folders` holds each user's temporary files and caches on
|
||||||
through firmlinks. Firmlinks are the links macOS uses to show the data
|
macOS. Root's home on macOS is `/var/root`, which the walk reaches as
|
||||||
volume inside the system volume. Walking `/System/Volumes/Data` as well
|
`/private/var/root`. A walk of `/` on macOS reaches everything in
|
||||||
would record each of those files twice. Any other system has no default
|
`/System/Volumes/Data` already, through firmlinks. Firmlinks are the links
|
||||||
excludes.
|
macOS uses to show the data volume inside the system volume. Walking
|
||||||
|
`/System/Volumes/Data` as well would record each of those files twice. Any
|
||||||
|
other system has no default excludes.
|
||||||
|
|
||||||
An operand at or inside a default exclude is walked, so
|
An operand at or inside a default exclude is walked, so
|
||||||
`sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem.
|
`sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem.
|
||||||
@@ -608,7 +622,7 @@ walk practical on large or busy storage pools.
|
|||||||
not enter, `scan` prints a line on stderr naming it:
|
not enter, `scan` prints a line on stderr naming it:
|
||||||
|
|
||||||
```
|
```
|
||||||
scan: skipped /var/tmp (default exclude)
|
scan: left out /var/tmp (default exclude)
|
||||||
```
|
```
|
||||||
|
|
||||||
The directory is not counted as skipped. `scan` prints the filesystem line
|
The directory is not counted as skipped. `scan` prints the filesystem line
|
||||||
@@ -668,6 +682,18 @@ scan: 123400 files seen (1200 added, 34 updated, 56 removed, 122166 unchanged),
|
|||||||
|
|
||||||
`removed` counts deleted records, which are not part of the files seen.
|
`removed` counts deleted records, which are not part of the files seen.
|
||||||
|
|
||||||
|
The warning for a skipped path names the path once and gives the reason:
|
||||||
|
|
||||||
|
```
|
||||||
|
scan: skipped /srv/a/closed (permission denied)
|
||||||
|
scan: skipped /srv/a/pipe (a FIFO)
|
||||||
|
```
|
||||||
|
|
||||||
|
The summary counts each path named in such a warning. A line for a path left out
|
||||||
|
on purpose starts with `scan: left out` instead, and is not counted. A search of
|
||||||
|
the log for `scan: skipped` therefore finds the paths the summary counts. When a
|
||||||
|
file with several hard links cannot be read, each of its paths gets a warning.
|
||||||
|
|
||||||
#### Stopping a scan
|
#### Stopping a scan
|
||||||
|
|
||||||
`scan` stops cleanly on `SIGINT`, the signal Ctrl-C sends, or `SIGTERM`, the
|
`scan` stops cleanly on `SIGINT`, the signal Ctrl-C sends, or `SIGTERM`, the
|
||||||
|
|||||||
@@ -1013,8 +1013,7 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
|||||||
|
|
||||||
stderr := scanOK(t, dir)
|
stderr := scanOK(t, dir)
|
||||||
|
|
||||||
warning := "hash " + unreadable + ": open " + unreadable +
|
warning := "scan: skipped " + unreadable + " (permission denied)\n"
|
||||||
": permission denied\n"
|
|
||||||
if !strings.Contains(stderr, warning) {
|
if !strings.Contains(stderr, warning) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr, warning)
|
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||||
}
|
}
|
||||||
@@ -1286,7 +1285,7 @@ func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
|
|||||||
|
|
||||||
stderr := scanOK(t, link)
|
stderr := scanOK(t, link)
|
||||||
|
|
||||||
warning := "walk " + fifo + ": skipping FIFO operand\n"
|
warning := "scan: skipped " + fifo + " (a FIFO)\n"
|
||||||
if !strings.Contains(stderr, warning) {
|
if !strings.Contains(stderr, warning) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr, warning)
|
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||||
}
|
}
|
||||||
@@ -1321,7 +1320,7 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
|||||||
|
|
||||||
stderr := scanOK(t, operand)
|
stderr := scanOK(t, operand)
|
||||||
|
|
||||||
warning := "walk " + operand + ": skipping " + kind + " operand\n"
|
warning := "scan: skipped " + operand + " (a " + kind + ")\n"
|
||||||
if !strings.Contains(stderr, warning) {
|
if !strings.Contains(stderr, warning) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr, warning)
|
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||||
}
|
}
|
||||||
@@ -1355,7 +1354,7 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|||||||
writeFile(t, hourly, "f", pattern(1, 10))
|
writeFile(t, hourly, "f", pattern(1, 10))
|
||||||
f := writeFile(t, dir, "f", pattern(2, 10))
|
f := writeFile(t, dir, "f", pattern(2, 10))
|
||||||
|
|
||||||
warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
|
warning := "scan: skipped " + hourly + " (inside a .zfs directory)\n"
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
operands []string
|
operands []string
|
||||||
@@ -1407,7 +1406,7 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
|||||||
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
||||||
|
|
||||||
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
||||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
warning := "scan: skipped " + daily + " (inside a .zfs directory)\n"
|
||||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||||
"0 unchanged), 1 skipped\n"
|
"0 unchanged), 1 skipped\n"
|
||||||
|
|
||||||
|
|||||||
@@ -121,9 +121,9 @@ func TestRunDwimExcludeLeavesTreeOut(t *testing.T) {
|
|||||||
t.Errorf("group = %q, want it to start %q", lines[1], want)
|
t.Errorf("group = %q, want it to start %q", lines[1], want)
|
||||||
}
|
}
|
||||||
|
|
||||||
skipped := "scan: skipped " + excluded + " (--exclude)\n"
|
leftOut := "scan: left out " + excluded + " (--exclude)\n"
|
||||||
if got := stderr(); !strings.Contains(got, skipped) {
|
if got := stderr(); !strings.Contains(got, leftOut) {
|
||||||
t.Errorf("stderr = %q, want it to contain %q", got, skipped)
|
t.Errorf("stderr = %q, want it to contain %q", got, leftOut)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -72,14 +72,31 @@ type walkRules struct {
|
|||||||
func defaultExcludes(goos string) []string {
|
func defaultExcludes(goos string) []string {
|
||||||
switch goos {
|
switch goos {
|
||||||
case goosLinux:
|
case goosLinux:
|
||||||
return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
|
// The kernel's view of the running system, temporary files, and
|
||||||
|
// caches that programs and package managers fill again on their
|
||||||
|
// own: the system's, root's and those in each home in /home.
|
||||||
|
return []string{
|
||||||
|
"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
|
||||||
|
"/root/.cache", "/root/.npm", "/root/.cargo/registry",
|
||||||
|
"/root/go/pkg/mod",
|
||||||
|
"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
|
||||||
|
"/home/*/go/pkg/mod",
|
||||||
|
}
|
||||||
case goosDarwin:
|
case goosDarwin:
|
||||||
// A walk of / reaches everything in /System/Volumes/Data through
|
// A walk of / reaches everything in /System/Volumes/Data through
|
||||||
// the firmlinks, so walking it as well would record each of
|
// the firmlinks, so walking it as well would record each of
|
||||||
// those files twice.
|
// those files twice. /private/var/folders holds each user's
|
||||||
|
// temporary files and caches. The walk reaches root's home,
|
||||||
|
// /var/root, as /private/var/root.
|
||||||
return []string{
|
return []string{
|
||||||
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
||||||
"/System/Volumes/VM", "/System/Volumes/Data",
|
"/System/Volumes/VM", "/System/Volumes/Data",
|
||||||
|
"/private/var/folders", "/Library/Caches",
|
||||||
|
"/private/var/root/Library/Caches", "/private/var/root/.cache",
|
||||||
|
"/private/var/root/.npm", "/private/var/root/.cargo/registry",
|
||||||
|
"/private/var/root/go/pkg/mod",
|
||||||
|
"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
|
||||||
|
"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
|
||||||
}
|
}
|
||||||
case goosWindows:
|
case goosWindows:
|
||||||
// Relative to the root of the drive an operand lies on. The
|
// Relative to the root of the drive an operand lies on. The
|
||||||
@@ -186,7 +203,7 @@ func (rules walkRules) excludePathLine(path string, fi fs.FileInfo) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("scan: skipped %s (--exclude)", path)
|
return fmt.Sprintf("scan: left out %s (--exclude)", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// nonPersistentFilesystemTypes returns the filesystem types that are
|
// nonPersistentFilesystemTypes returns the filesystem types that are
|
||||||
@@ -216,7 +233,7 @@ func (rules walkRules) nonPersistentLine(path string, fi fs.FileInfo) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("scan: skipped %s (%s, not a persistent filesystem)",
|
return fmt.Sprintf("scan: left out %s (%s, not a persistent filesystem)",
|
||||||
path, fsType)
|
path, fsType)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,10 +24,22 @@ func TestDefaultExcludes(t *testing.T) {
|
|||||||
// on any other system. The Windows list exists in the code only: the
|
// on any other system. The Windows list exists in the code only: the
|
||||||
// binary does not build for Windows, so the README leaves it out.
|
// binary does not build for Windows, so the README leaves it out.
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
goosLinux: {
|
||||||
|
"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
|
||||||
|
"/root/.cache", "/root/.npm", "/root/.cargo/registry",
|
||||||
|
"/root/go/pkg/mod",
|
||||||
|
"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
|
||||||
|
"/home/*/go/pkg/mod",
|
||||||
|
},
|
||||||
goosDarwin: {
|
goosDarwin: {
|
||||||
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
|
||||||
"/System/Volumes/VM", "/System/Volumes/Data",
|
"/System/Volumes/VM", "/System/Volumes/Data",
|
||||||
|
"/private/var/folders", "/Library/Caches",
|
||||||
|
"/private/var/root/Library/Caches", "/private/var/root/.cache",
|
||||||
|
"/private/var/root/.npm", "/private/var/root/.cargo/registry",
|
||||||
|
"/private/var/root/go/pkg/mod",
|
||||||
|
"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
|
||||||
|
"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
|
||||||
},
|
},
|
||||||
goosWindows: {
|
goosWindows: {
|
||||||
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
|
"pagefile.sys", "swapfile.sys", "hiberfil.sys",
|
||||||
@@ -63,6 +75,29 @@ func TestIsDefaultExclude(t *testing.T) {
|
|||||||
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
|
t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The Linux list matches the cache directories in /root and in each
|
||||||
|
// home in /home, but nothing below them and no .cache directory
|
||||||
|
// elsewhere.
|
||||||
|
linux := map[string]bool{
|
||||||
|
"/home/u/.cache": true,
|
||||||
|
"/home/u/go/pkg/mod": true,
|
||||||
|
"/root/.cache": true,
|
||||||
|
"/var/cache": true,
|
||||||
|
"/home/u/.cache/pip": false,
|
||||||
|
"/home/u/src/.cache": false,
|
||||||
|
"/srv/home/u/.cache": false,
|
||||||
|
"/home/u/.cargo/bin": false,
|
||||||
|
"/home/u/go/pkg/mod/cache": false,
|
||||||
|
}
|
||||||
|
|
||||||
|
for path, want := range linux {
|
||||||
|
got := isDefaultExclude(path, defaultExcludes(goosLinux))
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("isDefaultExclude(%q, Linux list) = %v, want %v", path,
|
||||||
|
got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReadMountInfo(t *testing.T) {
|
func TestReadMountInfo(t *testing.T) {
|
||||||
@@ -207,7 +242,7 @@ func TestScanDoesNotEnterDefaultExclude(t *testing.T) {
|
|||||||
t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
|
t.Errorf("%s recorded: the walk entered %s", unseen, excluded)
|
||||||
}
|
}
|
||||||
|
|
||||||
line := "\nscan: skipped " + excluded + " (default exclude)\n"
|
line := "\nscan: left out " + excluded + " (default exclude)\n"
|
||||||
if !strings.Contains(stderr(), line) {
|
if !strings.Contains(stderr(), line) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr(), line)
|
t.Errorf("stderr = %q, want %q", stderr(), line)
|
||||||
}
|
}
|
||||||
@@ -276,7 +311,7 @@ func TestScanNamesNoDefaultExcludeThatIsAnOperand(t *testing.T) {
|
|||||||
|
|
||||||
recordByPath(t, dbRecords(t, db), f)
|
recordByPath(t, dbRecords(t, db), f)
|
||||||
|
|
||||||
if strings.Contains(stderr(), "scan: skipped") {
|
if strings.Contains(stderr(), "scan: left out") {
|
||||||
t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
|
t.Errorf("stderr = %q, want no line for %s", stderr(), excluded)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -290,15 +325,15 @@ func TestRunScanTempDirOperand(t *testing.T) {
|
|||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
dir := realTempDir(t)
|
dir := realTempDir(t)
|
||||||
if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
|
if !insideDefaultExclude(dir, defaultExcludes(runtime.GOOS)) {
|
||||||
t.Skip(dir + " is not inside a default exclude")
|
t.Skip(dir + " is not inside a default exclude")
|
||||||
}
|
}
|
||||||
|
|
||||||
f := writeFile(t, dir, "sub/f", pattern(1, 10))
|
f := writeFile(t, dir, "sub/f", pattern(1, 10))
|
||||||
|
|
||||||
stderr := scanOK(t, dir)
|
stderr := scanOK(t, dir)
|
||||||
if strings.Contains(stderr, "scan: skipped") {
|
if strings.Contains(stderr, "scan: left out") {
|
||||||
t.Errorf("stderr = %q, want no line saying a path was skipped",
|
t.Errorf("stderr = %q, want no line saying a path was left out",
|
||||||
stderr)
|
stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +377,7 @@ func TestRunScanNoDefaultExcludes(t *testing.T) {
|
|||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
line := "scan: skipped /var/tmp (default exclude)\n"
|
line := "scan: left out /var/tmp (default exclude)\n"
|
||||||
|
|
||||||
if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
|
if stderr := scanOK(t, "/var"); !strings.Contains(stderr, line) {
|
||||||
t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
|
t.Errorf("scan /var: stderr = %q, want %q", stderr, line)
|
||||||
@@ -383,7 +418,7 @@ func TestRunScanExcludeNested(t *testing.T) {
|
|||||||
|
|
||||||
stderr := scanOK(t, excludeFlag, excluded, dir)
|
stderr := scanOK(t, excludeFlag, excluded, dir)
|
||||||
|
|
||||||
line := "scan: skipped " + excluded + " (--exclude)\n"
|
line := "scan: left out " + excluded + " (--exclude)\n"
|
||||||
if !strings.Contains(stderr, line) {
|
if !strings.Contains(stderr, line) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr, line)
|
t.Errorf("stderr = %q, want %q", stderr, line)
|
||||||
}
|
}
|
||||||
@@ -445,8 +480,8 @@ func TestRunScanExcludeReachedByAnotherPath(t *testing.T) {
|
|||||||
stderr := scanOK(t, excludeFlag, filepath.Join(link, "cache"),
|
stderr := scanOK(t, excludeFlag, filepath.Join(link, "cache"),
|
||||||
excludeFlag, dotDot, dir)
|
excludeFlag, dotDot, dir)
|
||||||
|
|
||||||
for _, skipped := range []string{filepath.Dir(inDir), file, hardLink} {
|
for _, leftOut := range []string{filepath.Dir(inDir), file, hardLink} {
|
||||||
line := "scan: skipped " + skipped + " (--exclude)\n"
|
line := "scan: left out " + leftOut + " (--exclude)\n"
|
||||||
if !strings.Contains(stderr, line) {
|
if !strings.Contains(stderr, line) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr, line)
|
t.Errorf("stderr = %q, want %q", stderr, line)
|
||||||
}
|
}
|
||||||
@@ -580,7 +615,7 @@ func TestRunScanSkipsNonPersistentOperand(t *testing.T) {
|
|||||||
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|
||||||
line := "scan: skipped " + shm + " (" + fsType +
|
line := "scan: left out " + shm + " (" + fsType +
|
||||||
", not a persistent filesystem)\n"
|
", not a persistent filesystem)\n"
|
||||||
summary := "scan: 0 files seen"
|
summary := "scan: 0 files seen"
|
||||||
|
|
||||||
@@ -634,7 +669,7 @@ func TestScanKeepsRecordsOnNonPersistent(t *testing.T) {
|
|||||||
t.Errorf("%s recorded", unseen)
|
t.Errorf("%s recorded", unseen)
|
||||||
}
|
}
|
||||||
|
|
||||||
line := "scan: skipped " + shm + " (" + fsType +
|
line := "scan: left out " + shm + " (" + fsType +
|
||||||
", not a persistent filesystem)\n"
|
", not a persistent filesystem)\n"
|
||||||
if !strings.Contains(stderr(), line) {
|
if !strings.Contains(stderr(), line) {
|
||||||
t.Errorf("stderr = %q, want %q", stderr(), line)
|
t.Errorf("stderr = %q, want %q", stderr(), line)
|
||||||
@@ -667,7 +702,7 @@ func TestWalkLeavesOutNonPersistent(t *testing.T) {
|
|||||||
|
|
||||||
leftOut = append(leftOut, ev.leftOut)
|
leftOut = append(leftOut, ev.leftOut)
|
||||||
|
|
||||||
want := "scan: skipped " + ev.leftOut + " (" + fsType +
|
want := "scan: left out " + ev.leftOut + " (" + fsType +
|
||||||
", not a persistent filesystem)"
|
", not a persistent filesystem)"
|
||||||
if ev.warn != want {
|
if ev.warn != want {
|
||||||
t.Errorf("line = %q, want %q", ev.warn, want)
|
t.Errorf("line = %q, want %q", ev.warn, want)
|
||||||
@@ -711,6 +746,20 @@ func shmTempDir(t *testing.T) (string, string, walkRules) {
|
|||||||
return dir, fsType, walkRules{nonPersistent: nonPersistent}
|
return dir, fsType, walkRules{nonPersistent: nonPersistent}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// insideDefaultExclude reports whether path, which is absolute and
|
||||||
|
// clean, or a directory above it is one of excludes.
|
||||||
|
func insideDefaultExclude(path string, excludes []string) bool {
|
||||||
|
for dir := path; ; dir = filepath.Dir(dir) {
|
||||||
|
if isDefaultExclude(dir, excludes) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if dir == "/" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// assertRecorded checks that the database at dbPath holds a record for
|
// assertRecorded checks that the database at dbPath holds a record for
|
||||||
// path when want is set, and none when it is not.
|
// path when want is set, and none when it is not.
|
||||||
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
|
func assertRecorded(t *testing.T, dbPath, path string, want bool) {
|
||||||
|
|||||||
+28
-12
@@ -664,8 +664,8 @@ func isDatabaseFile(f fileRec, rules walkRules) bool {
|
|||||||
// inode run, committing completed records to the database in batches
|
// inode run, committing completed records to the database in batches
|
||||||
// as results arrive, so a long scan persists its progress as it goes
|
// as results arrive, so a long scan persists its progress as it goes
|
||||||
// (an interrupted scan resumes cheaply: the next run skips everything
|
// (an interrupted scan resumes cheaply: the next run skips everything
|
||||||
// already recorded). A run that fails to hash is warned about and
|
// already recorded). Each path of a run that fails to hash is warned
|
||||||
// counted as skipped; stale records for its paths, if any, are deleted
|
// about and counted as skipped; stale records for them, if any, are deleted
|
||||||
// by the update phase. A run whose file no longer exists gets no record
|
// by the update phase. A run whose file no longer exists gets no record
|
||||||
// and no warning, and is not counted as skipped. The total counts actual
|
// and no warning, and is not counted as skipped. The total counts actual
|
||||||
// reads, so the display's time remaining is meaningful.
|
// reads, so the display's time remaining is meaningful.
|
||||||
@@ -706,9 +706,12 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
s.st.skipped += len(r.run)
|
// Every path in the run is counted, so every path gets a line.
|
||||||
|
for _, rec := range r.run {
|
||||||
|
s.st.skipped++
|
||||||
|
|
||||||
prog.warnf("hash %s: %v", r.run[0].path, r.err)
|
prog.warnf("%s", skippedLine(rec.path, r.err))
|
||||||
|
}
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -915,8 +918,7 @@ func operandWarning(root string, fi fs.FileInfo) string {
|
|||||||
// The walk never enters a .zfs directory, so it starts from nothing
|
// The walk never enters a .zfs directory, so it starts from nothing
|
||||||
// inside one either.
|
// inside one either.
|
||||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
||||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
return fmt.Sprintf("scan: skipped %s (inside a .zfs directory)", root)
|
||||||
root)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var kind string
|
var kind string
|
||||||
@@ -942,7 +944,21 @@ func operandWarning(root string, fi fs.FileInfo) string {
|
|||||||
kind = "non-regular file"
|
kind = "non-regular file"
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("walk %s: skipping %s operand", root, kind)
|
return fmt.Sprintf("scan: skipped %s (a %s)", root, kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
// skippedLine returns the warning for a path the scan skips because of
|
||||||
|
// err. The reason is the error without its path, which the line already
|
||||||
|
// names.
|
||||||
|
func skippedLine(path string, err error) string {
|
||||||
|
reason := err.Error()
|
||||||
|
|
||||||
|
var pathErr *fs.PathError
|
||||||
|
if errors.As(err, &pathErr) {
|
||||||
|
reason = pathErr.Err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf("scan: skipped %s (%s)", path, reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedRoot turns one PATH operand into the walk's starting state: a
|
// seedRoot turns one PATH operand into the walk's starting state: a
|
||||||
@@ -965,7 +981,7 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("walk %s: %v", root, err),
|
warn: skippedLine(root, err),
|
||||||
fail: true,
|
fail: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -1094,7 +1110,7 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("walk %s: %v", job.path, err),
|
warn: skippedLine(job.path, err),
|
||||||
fail: true,
|
fail: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -1153,7 +1169,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("stat %s: %v", p, err),
|
warn: skippedLine(p, err),
|
||||||
fail: true,
|
fail: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -1214,7 +1230,7 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
warn: fmt.Sprintf("walk %s: %v", p, err),
|
warn: skippedLine(p, err),
|
||||||
fail: true,
|
fail: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -1235,7 +1251,7 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
if isDefaultExclude(p, rules.defaultExcludes) {
|
if isDefaultExclude(p, rules.defaultExcludes) {
|
||||||
sendEvent(ctx, events, walkEvent{
|
sendEvent(ctx, events, walkEvent{
|
||||||
leftOut: p,
|
leftOut: p,
|
||||||
warn: fmt.Sprintf("scan: skipped %s (default exclude)", p),
|
warn: fmt.Sprintf("scan: left out %s (default exclude)", p),
|
||||||
})
|
})
|
||||||
|
|
||||||
return dirJob{}, false
|
return dirJob{}, false
|
||||||
|
|||||||
@@ -752,8 +752,9 @@ func TestWalkPassesOverRemovedPaths(t *testing.T) {
|
|||||||
|
|
||||||
// TestScanWarnsAboutUnreadablePaths checks that a permission error is
|
// TestScanWarnsAboutUnreadablePaths checks that a permission error is
|
||||||
// still warned about and counted as skipped, unlike a removed path. The
|
// still warned about and counted as skipped, unlike a removed path. The
|
||||||
// walk reaches an unreadable directory, and a file in a directory that
|
// walk reaches an unreadable directory. It also reaches a file and a
|
||||||
// can be read but not searched. Another such file is an operand.
|
// subdirectory in a directory that can be read but not searched. Another
|
||||||
|
// such file is an operand.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
|
func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
|
||||||
@@ -765,6 +766,7 @@ func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
tree := filepath.Join(dir, "tree")
|
tree := filepath.Join(dir, "tree")
|
||||||
walked := writeFile(t, tree, "unsearchable/f", []byte("x"))
|
walked := writeFile(t, tree, "unsearchable/f", []byte("x"))
|
||||||
|
subdir := filepath.Join(filepath.Dir(walked), "d")
|
||||||
operand := writeFile(t, dir, "unsearchable/f", []byte("y"))
|
operand := writeFile(t, dir, "unsearchable/f", []byte("y"))
|
||||||
unreadable := filepath.Join(tree, "unreadable")
|
unreadable := filepath.Join(tree, "unreadable")
|
||||||
|
|
||||||
@@ -773,6 +775,11 @@ func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = os.Mkdir(subdir, 0o750)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
modes := map[string]fs.FileMode{
|
modes := map[string]fs.FileMode{
|
||||||
unreadable: 0,
|
unreadable: 0,
|
||||||
filepath.Dir(walked): 0o400,
|
filepath.Dir(walked): 0o400,
|
||||||
@@ -793,15 +800,16 @@ func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
st := syncTree(t, openTestDB(t), tree, operand)
|
st := syncTree(t, openTestDB(t), tree, operand)
|
||||||
if st != (scanStats{skipped: 3}) {
|
if st != (scanStats{skipped: 4}) {
|
||||||
t.Errorf("stats = %+v, want 3 skipped and nothing else", st)
|
t.Errorf("stats = %+v, want 4 skipped and nothing else", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := stderr()
|
got := stderr()
|
||||||
for _, warning := range []string{
|
for _, warning := range []string{
|
||||||
"walk " + unreadable + ": open " + unreadable + ": permission denied\n",
|
"scan: skipped " + unreadable + " (permission denied)\n",
|
||||||
"stat " + walked + ": lstat " + walked + ": permission denied\n",
|
"scan: skipped " + walked + " (permission denied)\n",
|
||||||
"walk " + operand + ": lstat " + operand + ": permission denied\n",
|
"scan: skipped " + subdir + " (permission denied)\n",
|
||||||
|
"scan: skipped " + operand + " (permission denied)\n",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(got, warning) {
|
if !strings.Contains(got, warning) {
|
||||||
t.Errorf("stderr = %q, want %q", got, warning)
|
t.Errorf("stderr = %q, want %q", got, warning)
|
||||||
@@ -1941,14 +1949,19 @@ func TestScanHardlinksReadOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestScanHardlinkRunFailsTogether makes a file with two hard links
|
||||||
|
// unreadable. Both paths are counted as skipped, and each gets its own
|
||||||
|
// warning.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestScanHardlinkRunFailsTogether(t *testing.T) {
|
func TestScanHardlinkRunFailsTogether(t *testing.T) {
|
||||||
t.Parallel()
|
stderr := captureStderr(t)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
db := openTestDB(t)
|
db := openTestDB(t)
|
||||||
a := writeFile(t, dir, "a.bin", pattern(1, 300))
|
a := writeFile(t, dir, "a.bin", pattern(1, 300))
|
||||||
|
b := filepath.Join(dir, "b.bin")
|
||||||
|
|
||||||
err := os.Link(a, filepath.Join(dir, "b.bin"))
|
err := os.Link(a, b)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1964,6 +1977,15 @@ func TestScanHardlinkRunFailsTogether(t *testing.T) {
|
|||||||
if st.skipped != 2 || st.added != 0 {
|
if st.skipped != 2 || st.added != 0 {
|
||||||
t.Fatalf("stats = %+v, want both hardlink paths skipped", st)
|
t.Fatalf("stats = %+v, want both hardlink paths skipped", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
got := stderr()
|
||||||
|
|
||||||
|
for _, p := range []string{a, b} {
|
||||||
|
warning := "scan: skipped " + p + " (permission denied)\n"
|
||||||
|
if !strings.Contains(got, warning) {
|
||||||
|
t.Errorf("stderr = %q, want %q", got, warning)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file
|
// TestHashPhaseSkipsFileReplacedByFIFO replaces a walked regular file
|
||||||
@@ -2001,7 +2023,7 @@ func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) {
|
|||||||
s.st.skipped, len(s.batch))
|
s.st.skipped, len(s.batch))
|
||||||
}
|
}
|
||||||
|
|
||||||
warning := "hash " + p + ": not a regular file\n"
|
warning := "scan: skipped " + p + " (not a regular file)\n"
|
||||||
if got := stderr(); strings.Count(got, warning) != 1 {
|
if got := stderr(); strings.Count(got, warning) != 1 {
|
||||||
t.Errorf("stderr = %q, want %q once", got, warning)
|
t.Errorf("stderr = %q, want %q once", got, warning)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user