trees: count a nested dupe tree once in the reclaimable total (closes #168)

The total summed `size` over every row. It now leaves out a row whose
`dupe` lies inside another row's `dupe`. Before, such a row's bytes were
counted again in each row whose `dupe` holds it. The rows are unchanged.
README defines the total to match.

Model: opus-5-5

README.md

@@ -128,8 +128,8 @@ walk"). As the file `/etc/cron.d/sfdupes`:
     - one line naming the database;
     - progress lines, as described under "Progress";
     - a warning for each path it skips;
-    - a line for each system, temporary or cache directory the walk leaves out
+    - a line for each system or temporary directory the walk leaves out (see
-      (see "Rules for the walk");
+      "Rules for the walk");
     - a line for each path left out because its filesystem is one the kernel,
       the core of the operating system, makes up or keeps in memory, such as
       `tmpfs`;
@@ -300,9 +300,8 @@ The reference below is also the program's specification.
   of directories on other filesystems, as the same flag does for `du` and
   `rsync`. By default the walk also enters directories on other filesystems.
 - `--no-default-excludes` (`scan`, `dwim`) lets the walk enter the default
-  excludes. Those are the system, temporary and cache directories the walk
+  excludes. Those are the system and temporary directories the walk otherwise
-  otherwise leaves out, such as `/var/tmp` and `/var/cache` on Linux (see "Rules
+  leaves out, such as `/var/tmp` on Linux (see "Rules for the walk").
-  for the walk").
 - `--exclude PATH` (`scan`, `dwim`) keeps the walk out of `PATH` and everything
   below it. `PATH` may be a directory or a file. The flag may be given more than
   once. The walk knows `PATH` by its device ID and inode number, as it knows the
@@ -572,17 +571,12 @@ walk practical on large or busy storage pools.
     `--no-default-excludes` does not change this rule.
 
 - Below an operand, the walk does not enter a default exclude. On Linux, the
-  default excludes hold the kernel's view of the running system, temporary
+  default excludes hold temporary files and the kernel's view of the running
-  files, and caches. A cache here is data that programs and package managers
+  system. They depend on the system, and `sfdupes scan --help` lists those of
-  download or build again on their own. The default excludes depend on the
+  the system it runs on. On Linux they are:
-  system, and `sfdupes scan --help` lists those of the system it runs on. In the
-  lists, `*` stands for any one directory name, so `/home/*/.cache` matches
-  `/home/alice/.cache`. On Linux they are:
 
     ```
-    /proc, /sys, /dev, /run, /tmp, /var/tmp, /var/cache
+    /proc, /sys, /dev, /run, /tmp, /var/tmp
-    /root/.cache, /root/.npm, /root/.cargo/registry, /root/go/pkg/mod
-    /home/*/.cache, /home/*/.npm, /home/*/.cargo/registry, /home/*/go/pkg/mod
     ```
 
     On macOS they are:
@@ -590,21 +584,13 @@ walk practical on large or busy storage pools.
     ```
     /dev, /private/tmp, /private/var/tmp, /private/var/vm
     /System/Volumes/VM, /System/Volumes/Data
-    /private/var/folders, /Library/Caches
-    /private/var/root/Library/Caches, /private/var/root/.cache
-    /private/var/root/.npm, /private/var/root/.cargo/registry
-    /private/var/root/go/pkg/mod
-    /Users/*/Library/Caches, /Users/*/.cache, /Users/*/.npm
-    /Users/*/.cargo/registry, /Users/*/go/pkg/mod
     ```
 
-    `/private/var/folders` holds each user's temporary files and caches on
+    A walk of `/` on macOS reaches everything in `/System/Volumes/Data` already,
-    macOS. Root's home on macOS is `/var/root`, which the walk reaches as
+    through firmlinks. Firmlinks are the links macOS uses to show the data
-    `/private/var/root`. A walk of `/` on macOS reaches everything in
+    volume inside the system volume. Walking `/System/Volumes/Data` as well
-    `/System/Volumes/Data` already, through firmlinks. Firmlinks are the links
+    would record each of those files twice. Any other system has no default
-    macOS uses to show the data volume inside the system volume. Walking
+    excludes.
-    `/System/Volumes/Data` as well would record each of those files twice. Any
-    other system has no default excludes.
 
     An operand at or inside a default exclude is walked, so
     `sfdupes scan /tmp/x` works when `/tmp/x` is on a persistent filesystem.
@@ -862,7 +848,8 @@ The summary line on stderr gives:
 - the records read;
 - the duplicate-tree groups;
 - the dupe trees, which is the number of rows;
-- the reclaimable bytes, the sum of `size` over all rows.
+- the reclaimable bytes, the sum of `size` over all rows, leaving out a row
+  whose `dupe` lies inside another row's `dupe`.
 
 ```
 trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable

internal/sfdupes/exclude.go

@@ -72,31 +72,14 @@ type walkRules struct {
 func defaultExcludes(goos string) []string {
 	switch goos {
 	case goosLinux:
-		// The kernel's view of the running system, temporary files, and
+		return []string{"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"}
-		// caches that programs and package managers fill again on their
-		// own: the system's, root's and those in each home in /home.
-		return []string{
-			"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
-			"/root/.cache", "/root/.npm", "/root/.cargo/registry",
-			"/root/go/pkg/mod",
-			"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
-			"/home/*/go/pkg/mod",
-		}
 	case goosDarwin:
 		// A walk of / reaches everything in /System/Volumes/Data through
 		// the firmlinks, so walking it as well would record each of
-		// those files twice. /private/var/folders holds each user's
+		// those files twice.
-		// temporary files and caches. The walk reaches root's home,
-		// /var/root, as /private/var/root.
 		return []string{
 			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
 			"/System/Volumes/VM", "/System/Volumes/Data",
-			"/private/var/folders", "/Library/Caches",
-			"/private/var/root/Library/Caches", "/private/var/root/.cache",
-			"/private/var/root/.npm", "/private/var/root/.cargo/registry",
-			"/private/var/root/go/pkg/mod",
-			"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
-			"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
 		}
 	case goosWindows:
 		// Relative to the root of the drive an operand lies on. The

internal/sfdupes/exclude_test.go

@@ -24,22 +24,10 @@ func TestDefaultExcludes(t *testing.T) {
 	// on any other system. The Windows list exists in the code only: the
 	// binary does not build for Windows, so the README leaves it out.
 	cases := map[string][]string{
-		goosLinux: {
+		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
-			"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp", "/var/cache",
-			"/root/.cache", "/root/.npm", "/root/.cargo/registry",
-			"/root/go/pkg/mod",
-			"/home/*/.cache", "/home/*/.npm", "/home/*/.cargo/registry",
-			"/home/*/go/pkg/mod",
-		},
 		goosDarwin: {
 			"/dev", "/private/tmp", "/private/var/tmp", "/private/var/vm",
 			"/System/Volumes/VM", "/System/Volumes/Data",
-			"/private/var/folders", "/Library/Caches",
-			"/private/var/root/Library/Caches", "/private/var/root/.cache",
-			"/private/var/root/.npm", "/private/var/root/.cargo/registry",
-			"/private/var/root/go/pkg/mod",
-			"/Users/*/Library/Caches", "/Users/*/.cache", "/Users/*/.npm",
-			"/Users/*/.cargo/registry", "/Users/*/go/pkg/mod",
 		},
 		goosWindows: {
 			"pagefile.sys", "swapfile.sys", "hiberfil.sys",
@@ -75,29 +63,6 @@ func TestIsDefaultExclude(t *testing.T) {
 			t.Errorf("isDefaultExclude(%q) = %v, want %v", path, got, want)
 		}
 	}
-
-	// The Linux list matches the cache directories in /root and in each
-	// home in /home, but nothing below them and no .cache directory
-	// elsewhere.
-	linux := map[string]bool{
-		"/home/u/.cache":           true,
-		"/home/u/go/pkg/mod":       true,
-		"/root/.cache":             true,
-		"/var/cache":               true,
-		"/home/u/.cache/pip":       false,
-		"/home/u/src/.cache":       false,
-		"/srv/home/u/.cache":       false,
-		"/home/u/.cargo/bin":       false,
-		"/home/u/go/pkg/mod/cache": false,
-	}
-
-	for path, want := range linux {
-		got := isDefaultExclude(path, defaultExcludes(goosLinux))
-		if got != want {
-			t.Errorf("isDefaultExclude(%q, Linux list) = %v, want %v", path,
-				got, want)
-		}
-	}
 }
 
 func TestReadMountInfo(t *testing.T) {
@@ -325,7 +290,7 @@ func TestRunScanTempDirOperand(t *testing.T) {
 	t.Setenv(databaseEnv, path)
 
 	dir := realTempDir(t)
-	if !insideDefaultExclude(dir, defaultExcludes(runtime.GOOS)) {
+	if !underAnyRoot(dir, defaultExcludes(runtime.GOOS)) {
 		t.Skip(dir + " is not inside a default exclude")
 	}
 
@@ -746,20 +711,6 @@ func shmTempDir(t *testing.T) (string, string, walkRules) {
 	return dir, fsType, walkRules{nonPersistent: nonPersistent}
 }
 
-// insideDefaultExclude reports whether path, which is absolute and
-// clean, or a directory above it is one of excludes.
-func insideDefaultExclude(path string, excludes []string) bool {
-	for dir := path; ; dir = filepath.Dir(dir) {
-		if isDefaultExclude(dir, excludes) {
-			return true
-		}
-
-		if dir == "/" {
-			return false
-		}
-	}
-}
-
 // assertRecorded checks that the database at dbPath holds a record for
 // path when want is set, and none when it is not.
 func assertRecorded(t *testing.T, dbPath, path string, want bool) {

internal/sfdupes/trees.go

@@ -101,7 +101,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
 		stalledOn: "out",
 	})
-	dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
+	dupeTrees, err := writeTreeGroups(stdout, dupes, prog)
 
 	prog.finish()
 
@@ -112,7 +112,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 	fmt.Fprintf(os.Stderr,
 		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
 			"%s reclaimable\n",
-		records, len(dupes), dupeTrees, humanBytes(reclaimable))
+		records, len(dupes), dupeTrees, humanBytes(reclaimableBytes(dupes)))
 
 	return nil
 }
@@ -146,22 +146,19 @@ func readTree(ctx context.Context, db *sql.DB,
 // a row for every member of each group but its first, counting each
 // group on prog once its rows are written. The disk_size column is the
 // dupe's own. stdout may be the terminal prog is drawn on, so the lines
-// go through prog's writer. It returns the number of rows (dupe trees)
+// go through prog's writer. It returns the number of rows (dupe trees).
-// and the bytes they could reclaim.
 func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 	prog *progress,
-) (int, int64, error) {
+) (int, error) {
 	out := prog.writer(stdout)
 
 	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
 	if err != nil {
-		return 0, 0, fmt.Errorf("write stdout: %w", err)
+		return 0, fmt.Errorf("write stdout: %w", err)
 	}
 
 	dupeTrees := 0
 
-	var reclaimable int64
-
 	for _, g := range dupes {
 		first := g[0]
 		for _, n := range g[1:] {
@@ -169,11 +166,10 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 				escapePath(first.path), escapePath(n.path),
 				first.fileCount, first.totalSize, n.diskSize)
 			if err != nil {
-				return 0, 0, fmt.Errorf("write stdout: %w", err)
+				return 0, fmt.Errorf("write stdout: %w", err)
 			}
 
 			dupeTrees++
-			reclaimable += first.totalSize
 		}
 
 		prog.increment()
@@ -181,10 +177,44 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 
 	err = out.Flush()
 	if err != nil {
-		return 0, 0, fmt.Errorf("write stdout: %w", err)
+		return 0, fmt.Errorf("write stdout: %w", err)
 	}
 
-	return dupeTrees, reclaimable, nil
+	return dupeTrees, nil
+}
+
+// reclaimableBytes returns the sum of the size column over the rows
+// that writeTreeGroups writes for dupes, leaving out a row whose dupe
+// lies inside another row's dupe.
+func reclaimableBytes(dupes [][]*treeNode) int64 {
+	isDupe := make(map[*treeNode]bool)
+
+	for _, g := range dupes {
+		for _, n := range g[1:] {
+			isDupe[n] = true
+		}
+	}
+
+	var total int64
+
+	for n := range isDupe {
+		if !insideDupe(n, isDupe) {
+			total += n.totalSize
+		}
+	}
+
+	return total
+}
+
+// insideDupe reports whether a directory above n is in isDupe.
+func insideDupe(n *treeNode, isDupe map[*treeNode]bool) bool {
+	for p := n.parent; p != nil; p = p.parent {
+		if isDupe[p] {
+			return true
+		}
+	}
+
+	return false
 }
 
 // treeBuilder reconstructs the directory hierarchy from records added

internal/sfdupes/trees_test.go

@@ -313,7 +313,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
 		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
 	}, terminalWidth, time.Hour, time.Now)
 
-	_, _, err := writeTreeGroups(os.Stderr, dupes, p)
+	_, err := writeTreeGroups(os.Stderr, dupes, p)
 
 	p.finish()
 
@@ -587,3 +587,64 @@ func TestCollectTreeGroupsKeepsGroupLeftWithTwoCopies(t *testing.T) {
 		t.Errorf("groups = %v, want %v", got, want)
 	}
 }
+
+// TestReclaimableBytesCountsNestedDupeOnce checks that a dupe tree
+// inside another dupe tree adds nothing to the reclaimable bytes. /B/x
+// lies in /B, so the total adds the sizes of /B and /C/x, 1500 + 1000.
+func TestReclaimableBytesCountsNestedDupeOnce(t *testing.T) {
+	t.Parallel()
+
+	fh, ft, fc := testHash("fh"), testHash("ft"), testHash("fc")
+	gh, gt, gc := testHash("gh"), testHash("gt"), testHash("gc")
+	recs := []scanRec{
+		{size: 1000, head: fh, tail: ft, content: fc, path: "/A/x/f"},
+		{size: 500, head: gh, tail: gt, content: gc, path: "/A/y/g"},
+		{size: 1000, head: fh, tail: ft, content: fc, path: "/B/x/f"},
+		{size: 500, head: gh, tail: gt, content: gc, path: "/B/y/g"},
+		{size: 1000, head: fh, tail: ft, content: fc, path: "/C/x/f"},
+	}
+
+	super, dirs := treeOf(t, recs)
+	dupes := collectTreeGroups(dirs, super, everyPathExists, nil)
+
+	wantGroups := [][]string{{"/A", "/B"}, {"/A/x", "/B/x", "/C/x"}}
+	if got := groupPaths(dupes); !slices.EqualFunc(got, wantGroups,
+		slices.Equal) {
+		t.Fatalf("groups = %v, want %v", got, wantGroups)
+	}
+
+	if got := reclaimableBytes(dupes); got != 2500 {
+		t.Errorf("reclaimableBytes = %d, want 2500", got)
+	}
+}
+
+// TestReclaimableBytesCountsDupeInsideGrandparentDupeOnce checks that a
+// dupe tree adds nothing when a dupe tree above its parent holds it.
+// /B/m/x lies in /B, and /B/m is no dupe, since the group of /A and /B
+// implies that of /A/m and /B/m. The total adds the sizes of /B and
+// /C/x, 1300 + 1000.
+func TestReclaimableBytesCountsDupeInsideGrandparentDupeOnce(t *testing.T) {
+	t.Parallel()
+
+	f, w := testHash("f"), testHash("w")
+	recs := []scanRec{
+		{size: 1000, head: f, tail: f, content: f, path: "/A/m/x/f"},
+		{size: 300, head: w, tail: w, content: w, path: "/A/m/w"},
+		{size: 1000, head: f, tail: f, content: f, path: "/B/m/x/f"},
+		{size: 300, head: w, tail: w, content: w, path: "/B/m/w"},
+		{size: 1000, head: f, tail: f, content: f, path: "/C/x/f"},
+	}
+
+	super, dirs := treeOf(t, recs)
+	dupes := collectTreeGroups(dirs, super, everyPathExists, nil)
+
+	wantGroups := [][]string{{"/A", "/B"}, {"/A/m/x", "/B/m/x", "/C/x"}}
+	if got := groupPaths(dupes); !slices.EqualFunc(got, wantGroups,
+		slices.Equal) {
+		t.Fatalf("groups = %v, want %v", got, wantGroups)
+	}
+
+	if got := reclaimableBytes(dupes); got != 2300 {
+		t.Errorf("reclaimableBytes = %d, want 2300", got)
+	}
+}