Leave removed copies out of report and trees (closes #173)

A deleted duplicate stayed in both reports until a scan covered it, and
a missing scan operand is a fatal error. report now runs lstat on each
path of a possible duplicate group and drops paths that are gone; trees
does the same for the directory at the top of each copy, never the
files inside. A group left with one copy is not reported. Only "does not
exist" and "not a directory" count as gone, so a permission error keeps
the copy for an unprivileged user. README says a stale report is fixed
by scanning again.

Model: opus-5-5

README.md

@@ -665,7 +665,7 @@ each path in a group still exists, and looks at no other path. The check is
 `lstat`, the system call that looks up a path without following a symlink:
 
 - A path is gone when `lstat` finds that it, or a directory above it, does not
-  exist.
+  exist. A path on a filesystem that is not mounted is gone too.
 - Any other error, such as a permission error, counts as present. So a user who
   may not look inside a directory still sees its copies.
 
@@ -678,7 +678,6 @@ Processing:
   `(size, head, tail, content)`.
 - In a group with two or more paths, a path that is gone is left out. A group
   that still has two or more paths is a duplicate group.
-- Empty files form no group, because removing one frees nothing.
 - Within a group, paths are sorted in byte order. The first is `first`, and
   every other path is a `dupe`.
 - Groups are ordered by size, largest first, then by `first` path. The output is
@@ -727,7 +726,7 @@ The summary line on stderr gives:
 - the reclaimable bytes, the sum of `size` over all rows.
 
 ```
-report: 14 records read, 4 duplicate groups, 7 dupe files, 10.0 KiB reclaimable
+report: 14 records read, 5 duplicate groups, 8 dupe files, 10.0 KiB reclaimable
 ```
 
 ### `trees` mode
@@ -757,9 +756,6 @@ Processing:
 - `trees` builds the directory tree and computes every directory's digest. It
   then groups the directories by digest. Every group with two or more
   directories is a duplicate-tree group.
-- `trees` leaves out a group whose trees hold 0 bytes, because removing one
-  frees nothing. An empty file still counts toward the digest of a tree that
-  holds other files.
 - `trees` leaves out a group that a group of its members' parents already
   implies. A group is left out when its members' parents are distinct
   directories that all have the same digest. The parents' group, or an
@@ -1406,7 +1402,7 @@ All of the following, run in this directory, must pass:
     head -c 2000 /dev/urandom > "$d/a/unique.bin"     # same size, different content
     printf 'x' > "$d/tiny1"; printf 'x' > "$d/tiny2"  # 1-byte duplicates
     printf 'y' > "$d/tiny3"                           # 1-byte non-duplicate
-    : > "$d/empty1"; : > "$d/empty2"                  # empty files
+    : > "$d/empty1"; : > "$d/empty2"                  # empty duplicates
     # duplicate trees: t1 and t2 are identical; t3 differs by one filename
     mkdir -p "$d/t1/sub" "$d/t2/sub" "$d/t3/sub"
     head -c 3000 /dev/urandom > "$d/t1/f1"
@@ -1431,8 +1427,8 @@ All of the following, run in this directory, must pass:
       is the path that sorts first.
     - `t1/f1`, `t2/f1` and `t3/f1` form one group.
     - `t1/sub/f2`, `t2/sub/f2` and `t3/sub/f2renamed` form one group.
-    - `tiny1` and `tiny2` form a pair.
-    - `unique.bin`, `tiny3`, `empty1` and `empty2` appear nowhere.
+    - `tiny1` and `tiny2` form a pair, and so do `empty1` and `empty2`.
+    - `unique.bin` and `tiny3` appear nowhere.
     - The groups come in order of size, largest first.
 
     `trees` prints exactly one row, `$d/t1` as `first` and `$d/t2` as `dupe`.

TODO.md

@@ -35,11 +35,6 @@ are known.
   operand is still a fatal error (2026-11-11,
   https://git.eeqj.de/sneak/sfdupes/issues/173)
 
-- `report` and `trees` leave out empty files. `report` writes no group of size
-  0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
-  counts toward a tree's digest (2026-11-11,
-  https://git.eeqj.de/sneak/sfdupes/issues/178)
-
 - The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the
   command, and it calls `Main` in the package `internal/sfdupes`. `main.go`
   became `internal/sfdupes/cli.go`. Every other file kept its name. The install

internal/sfdupes/report.go

@@ -205,18 +205,17 @@ func (w *reportWriter) add(r scanRec) error {
 
 // endGroup ends the paths read with w.sig: two or more that are still
 // on disk are a duplicate group, held with its paths sorted until
-// writeGroups writes it. Empty files form no group, since removing one
-// frees nothing. Only the paths of a possible group are checked, never
-// a path with no duplicate or an empty file.
+// writeGroups writes it. Only the paths of a possible group are
+// checked, never a path with no duplicate.
 func (w *reportWriter) endGroup() {
-	if len(w.paths) >= minGroupSize && w.sig.size > 0 {
+	if len(w.paths) >= minGroupSize {
 		w.paths = slices.DeleteFunc(w.paths, func(p string) bool {
 			return !w.exists(p)
 		})
 	}
 
-	if len(w.paths) < minGroupSize || w.sig.size == 0 {
-		// The paths are dropped, and their slice reused.
+	if len(w.paths) < minGroupSize {
+		// A path with no duplicate is dropped, and its slice reused.
 		w.paths = w.paths[:0]
 
 		return

internal/sfdupes/report_test.go

@@ -462,71 +462,6 @@ func TestDupeGroups(t *testing.T) {
 	}
 }
 
-// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
-// duplicate group, while 1-byte files still do.
-func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
-	t.Parallel()
-
-	e, x := sha256.Sum256(nil), testHash("x")
-	recs := []scanRec{
-		{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
-		{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
-		{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
-	}
-
-	groups := dupeGroupsOf(t, recs)
-	if len(groups) != 1 || groups[0].size != 1 ||
-		!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
-		t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
-	}
-}
-
-// TestReadReportChecksOnlyPossibleDuplicates checks that report asks
-// whether a path is on disk only when its signature has two or more
-// records, and never for an empty file.
-func TestReadReportChecksOnlyPossibleDuplicates(t *testing.T) {
-	t.Parallel()
-
-	a, u, e := testHash("a"), testHash("u"), sha256.Sum256(nil)
-	recs := []scanRec{
-		{size: 9, head: a, tail: a, content: a, path: "/x/1"},
-		{size: 9, head: a, tail: a, content: a, path: "/x/2"},
-		{size: 9, head: a, tail: a, content: a, path: "/x/3"},
-		{size: 9, head: u, tail: u, content: u, path: "/lonely"},
-		{size: 0, head: e, tail: e, content: e, path: "/empty1"},
-		{size: 0, head: e, tail: e, content: e, path: "/empty2"},
-	}
-
-	db := openTestDB(t)
-
-	err := applyChanges(t.Context(), db, recs, nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var checked []string
-
-	exists := func(path string) bool {
-		checked = append(checked, path)
-
-		return true
-	}
-
-	_, err = readReport(t.Context(), db, "", io.Discard, exists, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	slices.Sort(checked)
-
-	want := []string{"/x/1", "/x/2", "/x/3"}
-	if !slices.Equal(checked, want) {
-		t.Errorf("paths checked on disk = %q, want %q", checked, want)
-	}
-}
-
 func TestDupeGroupsContentSeparates(t *testing.T) {
 	t.Parallel()
 

internal/sfdupes/scan_test.go

@@ -1185,12 +1185,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
 	t.Helper()
 
 	groups := dupeGroups(t, db)
-	if len(groups) != 4 {
-		t.Fatalf("len(groups) = %d, want 4", len(groups))
+	if len(groups) != 5 {
+		t.Fatalf("len(groups) = %d, want 5", len(groups))
 	}
 
-	// The empty files empty1 and empty2 form no group.
-	wantSizes := []int64{3000, 2000, 100, 1}
+	wantSizes := []int64{3000, 2000, 100, 1, 0}
 	for i, g := range groups {
 		if g.size != wantSizes[i] {
 			t.Errorf("groups[%d].size = %d, want %d",
@@ -2290,7 +2289,8 @@ func TestRunReportsLeaveOutRemovedCopies(t *testing.T) {
 		p("t1/f1") + "\t" + p("t2/f1") + "\t3000\n" +
 		p("t1/f1") + "\t" + p("t3/f1") + "\t3000\n" +
 		p("a/one.bin") + "\t" + p("b/copy.bin") + "\t2000\n" +
-		p("t1/sub/f2") + "\t" + p("t3/sub/f2renamed") + "\t100\n"
+		p("t1/sub/f2") + "\t" + p("t3/sub/f2renamed") + "\t100\n" +
+		p("empty1") + "\t" + p("empty2") + "\t0\n"
 	if got := runStdout(t, cmdReport); got != want {
 		t.Errorf("report stdout = %q, want %q", got, want)
 	}
@@ -2431,8 +2431,8 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
 			report := runStdout(t, cmdReport)
 			if strings.Contains(report, "db.sqlite") ||
 				strings.Contains(report, hardLink) ||
-				!strings.Contains(report, filepath.Join(tree, "b")) {
-				t.Errorf("report = %q, want a and b and none of "+
+				!strings.Contains(report, filepath.Join(tree, "empty2")) {
+				t.Errorf("report = %q, want the empty files and none of "+
 					"the database's files", report)
 			}
 		})

internal/sfdupes/trees.go

@@ -303,10 +303,8 @@ func fileEntry(name string, r scanRec) string {
 // prog as it is grouped, and returns every maximal group with two or
 // more members for which exists returns true, each group's members
 // sorted by path, groups ordered by tree size descending then by first
-// path ascending. A group whose trees hold 0 bytes is left out, since
-// removing one frees nothing. The groups it implies hold 0 bytes too,
-// so leaving it out hides nothing. exists is called only on the members
-// of maximal groups.
+// path ascending. exists is called only on the members of maximal
+// groups.
 func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 	exists func(path string) bool, prog *progress,
 ) [][]*treeNode {
@@ -320,9 +318,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 	var dupes [][]*treeNode
 
 	for _, g := range groups {
-		// The members share a digest, so they share a total size.
-		if len(g) < minGroupSize || g[0].totalSize == 0 ||
-			suppressed(g, super) {
+		if len(g) < minGroupSize || suppressed(g, super) {
 			continue
 		}
 

internal/sfdupes/trees_test.go

@@ -2,7 +2,6 @@ package sfdupes //nolint:testpackage // checks the unexported treeNode results
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"database/sql"
 	"os"
 	"slices"
@@ -448,35 +447,6 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
 	}
 }
 
-// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
-// holding only empty files form no group, and that an empty file still
-// counts toward the digest of a tree that holds other files.
-func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
-	t.Parallel()
-
-	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
-		sha256.Sum256(nil)
-	recs := []scanRec{
-		{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
-		{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
-		{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
-		// Without the empty file, /p/x3 is no copy of /p/x1.
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
-	}
-
-	super, dirs := treeOf(t, recs)
-
-	got := groupPaths(collectTreeGroups(dirs, super, everyPathExists, nil))
-
-	want := [][]string{{"/p/x1", "/p/x2"}}
-	if !slices.EqualFunc(got, want, slices.Equal) {
-		t.Fatalf("groups = %v, want %v", got, want)
-	}
-}
-
 func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 	t.Parallel()
 
@@ -499,49 +469,3 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 		t.Fatalf("groups = %v, want %v", got, want)
 	}
 }
-
-// TestCollectTreeGroupsChecksOnlyReportedCopies checks that trees asks
-// whether a directory is on disk only for the members of the groups it
-// returns. It never asks about /a/proj/sub, which lies inside a copy
-// and whose group the group of /a/proj implies, nor about /a/empty,
-// which holds 0 bytes.
-func TestCollectTreeGroupsChecksOnlyReportedCopies(t *testing.T) {
-	t.Parallel()
-
-	f, g, u, e := testHash("f"), testHash("g"), testHash("u"),
-		sha256.Sum256(nil)
-	recs := []scanRec{
-		{size: 10, head: f, tail: f, content: f, path: "/a/proj/f"},
-		{size: 20, head: g, tail: g, content: g, path: "/a/proj/sub/g"},
-		{size: 0, head: e, tail: e, content: e, path: "/a/empty/.gitkeep"},
-		{size: 10, head: f, tail: f, content: f, path: "/b/proj/f"},
-		{size: 20, head: g, tail: g, content: g, path: "/b/proj/sub/g"},
-		{size: 0, head: e, tail: e, content: e, path: "/b/empty/.gitkeep"},
-		// /a lacks this file, so /a and /b are no copies of each other.
-		{size: 30, head: u, tail: u, content: u, path: "/b/lonely"},
-	}
-
-	super, dirs := treeOf(t, recs)
-
-	var checked []string
-
-	exists := func(path string) bool {
-		checked = append(checked, path)
-
-		return true
-	}
-
-	got := groupPaths(collectTreeGroups(dirs, super, exists, nil))
-
-	want := [][]string{{"/a/proj", "/b/proj"}}
-	if !slices.EqualFunc(got, want, slices.Equal) {
-		t.Fatalf("groups = %v, want %v", got, want)
-	}
-
-	slices.Sort(checked)
-
-	if !slices.Equal(checked, want[0]) {
-		t.Errorf("directories checked on disk = %q, want %q",
-			checked, want[0])
-	}
-}