Leave empty files out of report and trees (closes #178) #183

Merged
clawbot merged 1 commits from issue-178-leave-out-empty-files into next 2026-11-11 03:52:45 +01:00
7 changed files with 79 additions and 14 deletions

No files matched your search

+8 -4
View File
@@ -665,6 +665,7 @@ Processing:
- Records are grouped by signature, the four values
`(size, head, tail, content)`.
- Every group with two or more paths is a duplicate group.
- Empty files form no group, because removing one frees nothing.
- Within a group, paths are sorted in byte order. The first is `first`, and
every other path is a `dupe`.
- Groups are ordered by size, largest first, then by `first` path. The output is
@@ -713,7 +714,7 @@ The summary line on stderr gives:
- the reclaimable bytes, the sum of `size` over all rows.
```
report: 14 records read, 5 duplicate groups, 8 dupe files, 10.0 KiB reclaimable
report: 14 records read, 4 duplicate groups, 7 dupe files, 10.0 KiB reclaimable
```
### `trees` mode
@@ -741,6 +742,9 @@ Processing:
- `trees` builds the directory tree and computes every directory's digest. It
then groups the directories by digest. Every group with two or more
directories is a duplicate-tree group.
- `trees` leaves out a group whose trees hold 0 bytes, because removing one
frees nothing. An empty file still counts toward the digest of a tree that
holds other files.
- `trees` leaves out a group that a group of its members' parents already
implies. A group is left out when its members' parents are distinct
directories that all have the same digest. The parents' group, or an
@@ -1382,7 +1386,7 @@ All of the following, run in this directory, must pass:
head -c 2000 /dev/urandom > "$d/a/unique.bin" # same size, different content
printf 'x' > "$d/tiny1"; printf 'x' > "$d/tiny2" # 1-byte duplicates
printf 'y' > "$d/tiny3" # 1-byte non-duplicate
: > "$d/empty1"; : > "$d/empty2" # empty duplicates
: > "$d/empty1"; : > "$d/empty2" # empty files
# duplicate trees: t1 and t2 are identical; t3 differs by one filename
mkdir -p "$d/t1/sub" "$d/t2/sub" "$d/t3/sub"
head -c 3000 /dev/urandom > "$d/t1/f1"
@@ -1407,8 +1411,8 @@ All of the following, run in this directory, must pass:
is the path that sorts first.
- `t1/f1`, `t2/f1` and `t3/f1` form one group.
- `t1/sub/f2`, `t2/sub/f2` and `t3/sub/f2renamed` form one group.
- `tiny1` and `tiny2` form a pair, and so do `empty1` and `empty2`.
- `unique.bin` and `tiny3` appear nowhere.
- `tiny1` and `tiny2` form a pair.
- `unique.bin`, `tiny3`, `empty1` and `empty2` appear nowhere.
- The groups come in order of size, largest first.
`trees` prints exactly one row, `$d/t1` as `first` and `$d/t2` as `dupe`.
+4
View File
@@ -29,6 +29,10 @@
Newest first. An entry ends with its date and its issue or branch, where those
are known.
- `report` and `trees` leave out empty files. `report` writes no group of size
0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
counts toward a tree's digest (2026-11-11,
https://git.eeqj.de/sneak/sfdupes/issues/178)
- The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the
command, and it calls `Main` in the package `internal/sfdupes`. `main.go`
became `internal/sfdupes/cli.go`. Every other file kept its name. The install
+4 -3
View File
@@ -195,10 +195,11 @@ func (w *reportWriter) add(r scanRec) error {
}
// endGroup ends the paths read with w.sig: two or more are a duplicate
// group, held with its paths sorted until writeGroups writes it.
// group, held with its paths sorted until writeGroups writes it. Empty
// files form no group, since removing one frees nothing.
func (w *reportWriter) endGroup() {
if len(w.paths) < minGroupSize {
// A path with no duplicate is dropped, and its slice reused.
if len(w.paths) < minGroupSize || w.sig.size == 0 {
// The paths are dropped, and their slice reused.
w.paths = w.paths[:0]
return
+21
View File
@@ -434,6 +434,27 @@ func TestDupeGroups(t *testing.T) {
}
}
// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
// duplicate group, while 1-byte files still do.
func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
t.Parallel()
e, x := sha256.Sum256(nil), testHash("x")
recs := []scanRec{
{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
}
groups := dupeGroupsOf(t, recs)
if len(groups) != 1 || groups[0].size != 1 ||
!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
}
}
func TestDupeGroupsContentSeparates(t *testing.T) {
t.Parallel()
+6 -5
View File
@@ -1185,11 +1185,12 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
t.Helper()
groups := dupeGroups(t, db)
if len(groups) != 5 {
t.Fatalf("len(groups) = %d, want 5", len(groups))
if len(groups) != 4 {
t.Fatalf("len(groups) = %d, want 4", len(groups))
}
wantSizes := []int64{3000, 2000, 100, 1, 0}
// The empty files empty1 and empty2 form no group.
wantSizes := []int64{3000, 2000, 100, 1}
for i, g := range groups {
if g.size != wantSizes[i] {
t.Errorf("groups[%d].size = %d, want %d",
@@ -2367,8 +2368,8 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
report := runStdout(t, cmdReport)
if strings.Contains(report, "db.sqlite") ||
strings.Contains(report, hardLink) ||
!strings.Contains(report, filepath.Join(tree, "empty2")) {
t.Errorf("report = %q, want the empty files and none of "+
!strings.Contains(report, filepath.Join(tree, "b")) {
t.Errorf("report = %q, want a and b and none of "+
"the database's files", report)
}
})
+6 -2
View File
@@ -301,7 +301,9 @@ func fileEntry(name string, r scanRec) string {
// collectTreeGroups groups directories by digest, counting each one on
// prog as it is grouped, and returns every maximal group with two or
// more members, each group's members sorted by path, groups ordered by
// tree size descending then by first path ascending.
// tree size descending then by first path ascending. A group whose
// trees hold 0 bytes is left out, since removing one frees nothing. The
// groups it implies hold 0 bytes too, so leaving it out hides nothing.
func collectTreeGroups(allDirs []*treeNode, super *treeNode,
prog *progress,
) [][]*treeNode {
@@ -315,7 +317,9 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
var dupes [][]*treeNode
for _, g := range groups {
if len(g) < minGroupSize || suppressed(g, super) {
// The members share a digest, so they share a total size.
if len(g) < minGroupSize || g[0].totalSize == 0 ||
suppressed(g, super) {
continue
}
+30
View File
@@ -2,6 +2,7 @@ package sfdupes //nolint:testpackage // checks the unexported treeNode results
import (
"bytes"
"crypto/sha256"
"database/sql"
"os"
"slices"
@@ -435,6 +436,35 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
}
}
// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
// holding only empty files form no group, and that an empty file still
// counts toward the digest of a tree that holds other files.
func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
t.Parallel()
h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
sha256.Sum256(nil)
recs := []scanRec{
{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
// Without the empty file, /p/x3 is no copy of /p/x1.
{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
}
super, dirs := treeOf(t, recs)
got := groupPaths(collectTreeGroups(dirs, super, nil))
want := [][]string{{"/p/x1", "/p/x2"}}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("groups = %v, want %v", got, want)
}
}
func TestCollectTreeGroupsDifferingParents(t *testing.T) {
t.Parallel()