Compare commits

...
Author SHA1 Message Date
sneak 6732172ed8 trees: count a nested dupe tree once in the reclaimable total (closes #168)
check / check (push) Canceled after 0s
The total summed `size` over every row, so a dupe tree inside another
reported dupe tree was counted twice. It is now what removing every
`dupe` path frees: a dupe tree with a dupe tree above it adds nothing.
The rows are unchanged. README defines the total to match.

Model: opus-5-5
2026-11-11 03:55:17 +00:00
4 changed files with 78 additions and 14 deletions

No files matched your search

+2 -1
View File
@@ -802,7 +802,8 @@ The summary line on stderr gives:
- the records read; - the records read;
- the duplicate-tree groups; - the duplicate-tree groups;
- the dupe trees, which is the number of rows; - the dupe trees, which is the number of rows;
- the reclaimable bytes, the sum of `size` over all rows. - the reclaimable bytes, what removing every `dupe` path would free. A `dupe`
path inside another `dupe` path adds nothing, so each file is counted once.
``` ```
trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable
+3
View File
@@ -29,6 +29,9 @@
Newest first. An entry ends with its date and its issue or branch, where those Newest first. An entry ends with its date and its issue or branch, where those
are known. are known.
- The reclaimable total of `trees` counts each file once. A `dupe` tree inside
another `dupe` tree adds nothing, so the total is what removing every `dupe`
path frees (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/168)
- `report` and `trees` leave out empty files. `report` writes no group of size - `report` and `trees` leave out empty files. `report` writes no group of size
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still 0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
counts toward a tree's digest (2026-11-11, counts toward a tree's digest (2026-11-11,
+42 -12
View File
@@ -100,7 +100,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)), label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
stalledOn: "out", stalledOn: "out",
}) })
dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog) dupeTrees, err := writeTreeGroups(stdout, dupes, prog)
prog.finish() prog.finish()
@@ -111,7 +111,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
fmt.Fprintf(os.Stderr, fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+ "trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n", "%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimable)) records, len(dupes), dupeTrees, humanBytes(reclaimableBytes(dupes)))
return nil return nil
} }
@@ -145,22 +145,19 @@ func readTree(ctx context.Context, db *sql.DB,
// a row for every member of each group but its first, counting each // a row for every member of each group but its first, counting each
// group on prog once its rows are written. The disk_size column is the // group on prog once its rows are written. The disk_size column is the
// dupe's own. stdout may be the terminal prog is drawn on, so the lines // dupe's own. stdout may be the terminal prog is drawn on, so the lines
// go through prog's writer. It returns the number of rows (dupe trees) // go through prog's writer. It returns the number of rows (dupe trees).
// and the bytes they could reclaim.
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode, func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
prog *progress, prog *progress,
) (int, int64, error) { ) (int, error) {
out := prog.writer(stdout) out := prog.writer(stdout)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size") _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return 0, fmt.Errorf("write stdout: %w", err)
} }
dupeTrees := 0 dupeTrees := 0
var reclaimable int64
for _, g := range dupes { for _, g := range dupes {
first := g[0] first := g[0]
for _, n := range g[1:] { for _, n := range g[1:] {
@@ -168,11 +165,10 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
escapePath(first.path), escapePath(n.path), escapePath(first.path), escapePath(n.path),
first.fileCount, first.totalSize, n.diskSize) first.fileCount, first.totalSize, n.diskSize)
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return 0, fmt.Errorf("write stdout: %w", err)
} }
dupeTrees++ dupeTrees++
reclaimable += first.totalSize
} }
prog.increment() prog.increment()
@@ -180,10 +176,44 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
err = out.Flush() err = out.Flush()
if err != nil { if err != nil {
return 0, 0, fmt.Errorf("write stdout: %w", err) return 0, fmt.Errorf("write stdout: %w", err)
} }
return dupeTrees, reclaimable, nil return dupeTrees, nil
}
// reclaimableBytes returns the bytes that removing every dupe tree in
// dupes would free. A dupe tree inside another dupe tree adds nothing,
// since the outer one already counts its files.
func reclaimableBytes(dupes [][]*treeNode) int64 {
isDupe := make(map[*treeNode]bool)
for _, g := range dupes {
for _, n := range g[1:] {
isDupe[n] = true
}
}
var total int64
for n := range isDupe {
if !insideDupe(n, isDupe) {
total += n.totalSize
}
}
return total
}
// insideDupe reports whether a directory above n is in isDupe.
func insideDupe(n *treeNode, isDupe map[*treeNode]bool) bool {
for p := n.parent; p != nil; p = p.parent {
if isDupe[p] {
return true
}
}
return false
} }
// treeBuilder reconstructs the directory hierarchy from records added // treeBuilder reconstructs the directory hierarchy from records added
+31 -1
View File
@@ -303,7 +303,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)), label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
}, terminalWidth, time.Hour, time.Now) }, terminalWidth, time.Hour, time.Now)
_, _, err := writeTreeGroups(os.Stderr, dupes, p) _, err := writeTreeGroups(os.Stderr, dupes, p)
p.finish() p.finish()
@@ -487,3 +487,33 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
t.Fatalf("groups = %v, want %v", got, want) t.Fatalf("groups = %v, want %v", got, want)
} }
} }
// TestReclaimableBytesCountsNestedDupeOnce checks that a dupe tree
// inside another dupe tree adds nothing to the reclaimable bytes:
// removing /B and /C/x frees 1500 + 1000 bytes, and /B/x lies in /B.
func TestReclaimableBytesCountsNestedDupeOnce(t *testing.T) {
t.Parallel()
fh, ft, fc := testHash("fh"), testHash("ft"), testHash("fc")
gh, gt, gc := testHash("gh"), testHash("gt"), testHash("gc")
recs := []scanRec{
{size: 1000, head: fh, tail: ft, content: fc, path: "/A/x/f"},
{size: 500, head: gh, tail: gt, content: gc, path: "/A/y/g"},
{size: 1000, head: fh, tail: ft, content: fc, path: "/B/x/f"},
{size: 500, head: gh, tail: gt, content: gc, path: "/B/y/g"},
{size: 1000, head: fh, tail: ft, content: fc, path: "/C/x/f"},
}
super, dirs := treeOf(t, recs)
dupes := collectTreeGroups(dirs, super, nil)
wantGroups := [][]string{{"/A", "/B"}, {"/A/x", "/B/x", "/C/x"}}
if got := groupPaths(dupes); !slices.EqualFunc(got, wantGroups,
slices.Equal) {
t.Fatalf("groups = %v, want %v", got, wantGroups)
}
if got := reclaimableBytes(dupes); got != 2500 {
t.Errorf("reclaimableBytes = %d, want 2500", got)
}
}