Compare commits

..
Author SHA1 Message Date
sneak bcd373da7a Add scan --exclude PATH (closes #163)
check / check (push) Successful in 2m43s
The walk neither enters nor records an --exclude path or anything
below it, and prints "scan: skipped PATH (--exclude)" for each one it
reaches. The flag repeats. The walk knows each --exclude path by
device ID and inode number, as it knows the database files, so it also
leaves one out when it reaches it by another path, such as a hard
link. The records below an excluded path are kept, as for a default
exclude. An empty value is a usage error. An --exclude path whose real
path scan cannot find, or an operand at or below one, is a fatal error
before the scan starts. walkRules names its two lists excludePaths and
defaultExcludes. printDatabasePath, split out of runScan, keeps runScan
under the lint length limit.

Model: opus-5-5
2026-11-11 05:19:08 +00:00
4 changed files with 14 additions and 110 deletions

No files matched your search

+1 -2
View File
@@ -848,8 +848,7 @@ The summary line on stderr gives:
- the records read; - the records read;
- the duplicate-tree groups; - the duplicate-tree groups;
- the dupe trees, which is the number of rows; - the dupe trees, which is the number of rows;
- the reclaimable bytes, the sum of `size` over all rows, leaving out a row - the reclaimable bytes, the sum of `size` over all rows.
whose `dupe` lies inside another row's `dupe`.
``` ```
trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable trees: 14 records read, 1 duplicate tree groups, 1 dupe trees, 3.0 KiB reclaimable
-4
View File
@@ -29,10 +29,6 @@
Newest first. An entry ends with its date and its issue or branch, where those Newest first. An entry ends with its date and its issue or branch, where those
are known. are known.
- The reclaimable total of `trees` is the sum of `size` over all rows, leaving
out a row whose `dupe` lies inside another row's `dupe`. Before, the bytes of
such a row were counted twice (2026-11-11,
https://git.eeqj.de/sneak/sfdupes/issues/168)
- New `scan` flag `--exclude PATH`, which may be repeated. The walk neither - New `scan` flag `--exclude PATH`, which may be repeated. The walk neither
enters nor records an excluded path, and keeps the records below it. An enters nor records an excluded path, and keeps the records below it. An
operand at or below one is a fatal error (2026-11-11, operand at or below one is a fatal error (2026-11-11,
+12 -42
View File
@@ -101,7 +101,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)), label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
stalledOn: "out", stalledOn: "out",
}) })
dupeTrees, err := writeTreeGroups(stdout, dupes, prog) dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
prog.finish() prog.finish()
@@ -112,7 +112,7 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
fmt.Fprintf(os.Stderr, fmt.Fprintf(os.Stderr,
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+ "trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
"%s reclaimable\n", "%s reclaimable\n",
records, len(dupes), dupeTrees, humanBytes(reclaimableBytes(dupes))) records, len(dupes), dupeTrees, humanBytes(reclaimable))
return nil return nil
} }
@@ -146,19 +146,22 @@ func readTree(ctx context.Context, db *sql.DB,
// a row for every member of each group but its first, counting each // a row for every member of each group but its first, counting each
// group on prog once its rows are written. The disk_size column is the // group on prog once its rows are written. The disk_size column is the
// dupe's own. stdout may be the terminal prog is drawn on, so the lines // dupe's own. stdout may be the terminal prog is drawn on, so the lines
// go through prog's writer. It returns the number of rows (dupe trees). // go through prog's writer. It returns the number of rows (dupe trees)
// and the bytes they could reclaim.
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode, func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
prog *progress, prog *progress,
) (int, error) { ) (int, int64, error) {
out := prog.writer(stdout) out := prog.writer(stdout)
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size") _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
if err != nil { if err != nil {
return 0, fmt.Errorf("write stdout: %w", err) return 0, 0, fmt.Errorf("write stdout: %w", err)
} }
dupeTrees := 0 dupeTrees := 0
var reclaimable int64
for _, g := range dupes { for _, g := range dupes {
first := g[0] first := g[0]
for _, n := range g[1:] { for _, n := range g[1:] {
@@ -166,10 +169,11 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
escapePath(first.path), escapePath(n.path), escapePath(first.path), escapePath(n.path),
first.fileCount, first.totalSize, n.diskSize) first.fileCount, first.totalSize, n.diskSize)
if err != nil { if err != nil {
return 0, fmt.Errorf("write stdout: %w", err) return 0, 0, fmt.Errorf("write stdout: %w", err)
} }
dupeTrees++ dupeTrees++
reclaimable += first.totalSize
} }
prog.increment() prog.increment()
@@ -177,44 +181,10 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
err = out.Flush() err = out.Flush()
if err != nil { if err != nil {
return 0, fmt.Errorf("write stdout: %w", err) return 0, 0, fmt.Errorf("write stdout: %w", err)
} }
return dupeTrees, nil return dupeTrees, reclaimable, nil
}
// reclaimableBytes returns the sum of the size column over the rows
// that writeTreeGroups writes for dupes, leaving out a row whose dupe
// lies inside another row's dupe.
func reclaimableBytes(dupes [][]*treeNode) int64 {
isDupe := make(map[*treeNode]bool)
for _, g := range dupes {
for _, n := range g[1:] {
isDupe[n] = true
}
}
var total int64
for n := range isDupe {
if !insideDupe(n, isDupe) {
total += n.totalSize
}
}
return total
}
// insideDupe reports whether a directory above n is in isDupe.
func insideDupe(n *treeNode, isDupe map[*treeNode]bool) bool {
for p := n.parent; p != nil; p = p.parent {
if isDupe[p] {
return true
}
}
return false
} }
// treeBuilder reconstructs the directory hierarchy from records added // treeBuilder reconstructs the directory hierarchy from records added
+1 -62
View File
@@ -313,7 +313,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
label: treesPhase("write"), unit: "groups", total: int64(len(dupes)), label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
}, terminalWidth, time.Hour, time.Now) }, terminalWidth, time.Hour, time.Now)
_, err := writeTreeGroups(os.Stderr, dupes, p) _, _, err := writeTreeGroups(os.Stderr, dupes, p)
p.finish() p.finish()
@@ -587,64 +587,3 @@ func TestCollectTreeGroupsKeepsGroupLeftWithTwoCopies(t *testing.T) {
t.Errorf("groups = %v, want %v", got, want) t.Errorf("groups = %v, want %v", got, want)
} }
} }
// TestReclaimableBytesCountsNestedDupeOnce checks that a dupe tree
// inside another dupe tree adds nothing to the reclaimable bytes. /B/x
// lies in /B, so the total adds the sizes of /B and /C/x, 1500 + 1000.
func TestReclaimableBytesCountsNestedDupeOnce(t *testing.T) {
t.Parallel()
fh, ft, fc := testHash("fh"), testHash("ft"), testHash("fc")
gh, gt, gc := testHash("gh"), testHash("gt"), testHash("gc")
recs := []scanRec{
{size: 1000, head: fh, tail: ft, content: fc, path: "/A/x/f"},
{size: 500, head: gh, tail: gt, content: gc, path: "/A/y/g"},
{size: 1000, head: fh, tail: ft, content: fc, path: "/B/x/f"},
{size: 500, head: gh, tail: gt, content: gc, path: "/B/y/g"},
{size: 1000, head: fh, tail: ft, content: fc, path: "/C/x/f"},
}
super, dirs := treeOf(t, recs)
dupes := collectTreeGroups(dirs, super, everyPathExists, nil)
wantGroups := [][]string{{"/A", "/B"}, {"/A/x", "/B/x", "/C/x"}}
if got := groupPaths(dupes); !slices.EqualFunc(got, wantGroups,
slices.Equal) {
t.Fatalf("groups = %v, want %v", got, wantGroups)
}
if got := reclaimableBytes(dupes); got != 2500 {
t.Errorf("reclaimableBytes = %d, want 2500", got)
}
}
// TestReclaimableBytesCountsDupeInsideGrandparentDupeOnce checks that a
// dupe tree adds nothing when a dupe tree above its parent holds it.
// /B/m/x lies in /B, and /B/m is no dupe, since the group of /A and /B
// implies that of /A/m and /B/m. The total adds the sizes of /B and
// /C/x, 1300 + 1000.
func TestReclaimableBytesCountsDupeInsideGrandparentDupeOnce(t *testing.T) {
t.Parallel()
f, w := testHash("f"), testHash("w")
recs := []scanRec{
{size: 1000, head: f, tail: f, content: f, path: "/A/m/x/f"},
{size: 300, head: w, tail: w, content: w, path: "/A/m/w"},
{size: 1000, head: f, tail: f, content: f, path: "/B/m/x/f"},
{size: 300, head: w, tail: w, content: w, path: "/B/m/w"},
{size: 1000, head: f, tail: f, content: f, path: "/C/x/f"},
}
super, dirs := treeOf(t, recs)
dupes := collectTreeGroups(dirs, super, everyPathExists, nil)
wantGroups := [][]string{{"/A", "/B"}, {"/A/m/x", "/B/m/x", "/C/x"}}
if got := groupPaths(dupes); !slices.EqualFunc(got, wantGroups,
slices.Equal) {
t.Fatalf("groups = %v, want %v", got, wantGroups)
}
if got := reclaimableBytes(dupes); got != 2300 {
t.Errorf("reclaimableBytes = %d, want 2300", got)
}
}