Merge pull request 'next into main: the command moves to cmd/sfdupes' (#159) from next into main

Reviewed-on: #159

README.md

@@ -112,20 +112,15 @@ docker run --rm -v /var/lib/sfdupes:/var/lib/sfdupes sfdupes report > dupes.tsv
 Run `scan` as root, so that it can read every file. `scan` warns about a
 directory it cannot read, and about a new or changed file it cannot read. It
 skips the path and deletes the records at or below it (see "Rules for the
-walk").
+walk"). As the file `/etc/cron.d/sfdupes`:
-
-An unmounted disk leaves its mount point empty, so `scan` deletes every record
-under it, and the next scan reads every file on the disk again. For each other
-disk mounted below `/srv`, add `&& mountpoint -q` and its mount point after the
-first test. As the file `/etc/cron.d/sfdupes`:
 
 ```
-30 3 * * * root if mountpoint -q /srv; then /usr/local/bin/sfdupes scan /srv 2>>/var/log/sfdupes.log || tail -n 3 /var/log/sfdupes.log; else echo "a disk is not mounted; scan of /srv skipped"; fi
+30 3 * * * root /usr/local/bin/sfdupes scan /srv 2>>/var/log/sfdupes.log || tail -n 3 /var/log/sfdupes.log
 ```
 
 - The first scan creates `/var/lib/sfdupes/db.sqlite` and its directory. To keep
-  the database elsewhere, put `SFDUPES_DATABASE=/path/to/db.sqlite` directly
+  the database elsewhere, put `SFDUPES_DATABASE=/path/to/db.sqlite` before the
-  before `/usr/local/bin/sfdupes`.
+  command on the same line.
 - `scan` writes nothing to stdout. Its stderr, appended here to
   `/var/log/sfdupes.log`, holds:
     - one line naming the database;
@@ -213,8 +208,8 @@ makes the comparison exact. Reading samples cuts a 150 GiB file to about 150 MiB
 of reads, at the price of a comparison that is not exact.
 
 The hashes live in a database that is kept between runs. A rescan reads only new
-files, and files whose size or modification time (mtime) differs from their
+files, and files whose size changed or whose modification time (mtime) is newer
-record. All analysis then works from the database alone.
+than recorded. All analysis then works from the database alone.
 
 The goal is whole duplicated trees, not only single files. A download extracted
 twice, or a project copied once too often, is such a tree. It can be removed as
@@ -415,16 +410,17 @@ operands:
 
 - A file not yet in the database is hashed and inserted.
 - A file already in the database is not read again when its size and mtime match
-  its record. This is what makes a daily rescan cheap.
+  its record. The mtime matches when it is not newer than the recorded one. This
+  is what makes a daily rescan cheap.
 - When such a file's size on disk differs from its recorded `disk_size`, `scan`
   updates `disk_size` alone. It still does not read the file, and counts it as
   unchanged. ZFS, for example, counts a new file's blocks up to seconds after
   the write. A block is the unit in which a filesystem allocates disk space. A
   file scanned within those seconds gets its real `disk_size` at the next scan.
-- `scan` hashes a file again when its size or its mtime differs from its record.
+- `scan` hashes a file again when its size differs or its mtime is newer. The
-  An older mtime counts as much as a newer one. A file put back by `cp -p`,
+  mtime is compared to the nanosecond, as finely as the filesystem records it. A
-  `rsync -a` or a restore from backup often has an older mtime. The mtime is
+  same-size rewrite counts as a change whenever the filesystem gives it a later
-  compared to the nanosecond, as finely as the filesystem records it.
+  mtime, even within the same second.
 - The scan deletes each record under a scanned operand whose file it could not
   verify. That covers deleted files, and paths the scan could not stat or read
   this run. To stat a file is to read its details, such as its size, without
@@ -669,7 +665,6 @@ Processing:
 - Records are grouped by signature, the four values
   `(size, head, tail, content)`.
 - Every group with two or more paths is a duplicate group.
-- Empty files form no group, because removing one frees nothing.
 - Within a group, paths are sorted in byte order. The first is `first`, and
   every other path is a `dupe`.
 - Groups are ordered by size, largest first, then by `first` path. The output is
@@ -718,7 +713,7 @@ The summary line on stderr gives:
 - the reclaimable bytes, the sum of `size` over all rows.
 
 ```
-report: 14 records read, 4 duplicate groups, 7 dupe files, 10.0 KiB reclaimable
+report: 14 records read, 5 duplicate groups, 8 dupe files, 10.0 KiB reclaimable
 ```
 
 ### `trees` mode
@@ -746,9 +741,6 @@ Processing:
 - `trees` builds the directory tree and computes every directory's digest. It
   then groups the directories by digest. Every group with two or more
   directories is a duplicate-tree group.
-- `trees` leaves out a group whose trees hold 0 bytes, because removing one
-  frees nothing. An empty file still counts toward the digest of a tree that
-  holds other files.
 - `trees` leaves out a group that a group of its members' parents already
   implies. A group is left out when its members' parents are distinct
   directories that all have the same digest. The parents' group, or an
@@ -1390,7 +1382,7 @@ All of the following, run in this directory, must pass:
     head -c 2000 /dev/urandom > "$d/a/unique.bin"     # same size, different content
     printf 'x' > "$d/tiny1"; printf 'x' > "$d/tiny2"  # 1-byte duplicates
     printf 'y' > "$d/tiny3"                           # 1-byte non-duplicate
-    : > "$d/empty1"; : > "$d/empty2"                  # empty files
+    : > "$d/empty1"; : > "$d/empty2"                  # empty duplicates
     # duplicate trees: t1 and t2 are identical; t3 differs by one filename
     mkdir -p "$d/t1/sub" "$d/t2/sub" "$d/t3/sub"
     head -c 3000 /dev/urandom > "$d/t1/f1"
@@ -1415,8 +1407,8 @@ All of the following, run in this directory, must pass:
       is the path that sorts first.
     - `t1/f1`, `t2/f1` and `t3/f1` form one group.
     - `t1/sub/f2`, `t2/sub/f2` and `t3/sub/f2renamed` form one group.
-    - `tiny1` and `tiny2` form a pair.
+    - `tiny1` and `tiny2` form a pair, and so do `empty1` and `empty2`.
-    - `unique.bin`, `tiny3`, `empty1` and `empty2` appear nowhere.
+    - `unique.bin` and `tiny3` appear nowhere.
     - The groups come in order of size, largest first.
 
     `trees` prints exactly one row, `$d/t1` as `first` and `$d/t2` as `dupe`.

TODO.md

@@ -29,16 +29,6 @@
 Newest first. An entry ends with its date and its issue or branch, where those
 are known.
 
-- `README.md` says what a disk that is not mounted does to the database. The
-  daily `cron` line runs `scan` only when `mountpoint` reports `/srv` as mounted
-  (2026-11-11, https://git.eeqj.de/sneak/sfdupes/issues/175)
-- The usage line of `scan` no longer ends in `[flags]`. It is the line
-  `README.md` shows under "Subcommands" (2026-11-11,
-  https://git.eeqj.de/sneak/sfdupes/issues/169)
-- `report` and `trees` leave out empty files. `report` writes no group of size
-  0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
-  counts toward a tree's digest (2026-11-11,
-  https://git.eeqj.de/sneak/sfdupes/issues/178)
 - The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the
   command, and it calls `Main` in the package `internal/sfdupes`. `main.go`
   became `internal/sfdupes/cli.go`. Every other file kept its name. The install

internal/sfdupes/cli.go

@@ -215,13 +215,10 @@ func newScanCommand() *cobra.Command {
 	short := "Walk trees and synchronize the scan database"
 
 	scanCmd := &cobra.Command{
-		Use: cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
+		Use:   cmdScan + " [--workers N] [-x] [--no-default-excludes] PATH...",
-		// The Use line already names every flag, so cobra must not
+		Short: short,
-		// append its generic "[flags]" to it.
+		Long:  short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
-		DisableFlagsInUseLine: true,
+		Args:  cobra.MinimumNArgs(1),
-		Short:                 short,
-		Long:                  short + ".\n\n" + defaultExcludesHelp(runtime.GOOS),
-		Args:                  cobra.MinimumNArgs(1),
 		PreRunE: func(cmd *cobra.Command, _ []string) error {
 			return checkScanWorkers(cmd, scanWorkers)
 		},

internal/sfdupes/cli_test.go

@@ -714,34 +714,6 @@ func TestRunHelp(t *testing.T) {
 	}
 }
 
-func TestRunScanUsageLine(t *testing.T) {
-	t.Parallel()
-
-	// README §Subcommands shows this usage line for scan, and the help
-	// and the usage error both print it.
-	const want = "\n  sfdupes scan [--workers N] [-x] [--no-default-excludes] PATH...\n"
-
-	cases := map[string][]string{
-		"help":          {cmdScan, "-h"},
-		"missing paths": {cmdScan},
-	}
-
-	for name, args := range cases {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			var stdout, stderr bytes.Buffer
-
-			run(args, &stdout, &stderr)
-
-			if !strings.Contains(stderr.String(), want) {
-				t.Errorf("run(%v) stderr = %q, want the usage line %q",
-					args, stderr.String(), want)
-			}
-		})
-	}
-}
-
 func TestRunVersion(t *testing.T) {
 	t.Parallel()
 

internal/sfdupes/report.go

@@ -195,11 +195,10 @@ func (w *reportWriter) add(r scanRec) error {
 }
 
 // endGroup ends the paths read with w.sig: two or more are a duplicate
-// group, held with its paths sorted until writeGroups writes it. Empty
+// group, held with its paths sorted until writeGroups writes it.
-// files form no group, since removing one frees nothing.
 func (w *reportWriter) endGroup() {
-	if len(w.paths) < minGroupSize || w.sig.size == 0 {
+	if len(w.paths) < minGroupSize {
-		// The paths are dropped, and their slice reused.
+		// A path with no duplicate is dropped, and its slice reused.
 		w.paths = w.paths[:0]
 
 		return

internal/sfdupes/report_test.go

@@ -434,27 +434,6 @@ func TestDupeGroups(t *testing.T) {
 	}
 }
 
-// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
-// duplicate group, while 1-byte files still do.
-func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
-	t.Parallel()
-
-	e, x := sha256.Sum256(nil), testHash("x")
-	recs := []scanRec{
-		{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
-		{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
-		{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
-	}
-
-	groups := dupeGroupsOf(t, recs)
-	if len(groups) != 1 || groups[0].size != 1 ||
-		!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
-		t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
-	}
-}
-
 func TestDupeGroupsContentSeparates(t *testing.T) {
 	t.Parallel()
 

internal/sfdupes/scan.go

@@ -545,7 +545,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
 		prog.increment()
 
 		old, ok := s.existing[ev.rec.path]
-		if !ok || old.size != ev.rec.size || !ev.rec.mtime.Equal(old.mtime) {
+		if !ok || old.size != ev.rec.size || mtimeAfter(ev.rec.mtime, old.mtime) {
 			s.toHash = append(s.toHash, ev.rec)
 
 			continue
@@ -780,6 +780,16 @@ func (s *scanState) updatePhase(ctx context.Context) error {
 	return applyChanges(ctx, s.db, nil, deletes, prog)
 }
 
+// mtimeAfter reports whether mtime a is later than mtime b.
+// Not a.After(b): time.Time wraps an mtime past year 292 billion; Unix() undoes it.
+func mtimeAfter(a, b time.Time) bool {
+	if a.Unix() != b.Unix() {
+		return a.Unix() > b.Unix()
+	}
+
+	return a.Nanosecond() > b.Nanosecond()
+}
+
 // underAnyRoot reports whether path is any of the roots or lies under
 // one of them.
 func underAnyRoot(path string, roots []string) bool {

internal/sfdupes/scan_test.go

@@ -1185,12 +1185,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
 	t.Helper()
 
 	groups := dupeGroups(t, db)
-	if len(groups) != 4 {
+	if len(groups) != 5 {
-		t.Fatalf("len(groups) = %d, want 4", len(groups))
+		t.Fatalf("len(groups) = %d, want 5", len(groups))
 	}
 
-	// The empty files empty1 and empty2 form no group.
+	wantSizes := []int64{3000, 2000, 100, 1, 0}
-	wantSizes := []int64{3000, 2000, 100, 1}
 	for i, g := range groups {
 		if g.size != wantSizes[i] {
 			t.Errorf("groups[%d].size = %d, want %d",
@@ -1384,47 +1383,6 @@ func TestSyncScanMtimeBump(t *testing.T) {
 	}
 }
 
-// TestSyncScanMtimeMovedBack replaces a file with other content of the
-// same size and an older mtime, as cp -p or a restore from backup does.
-// The next scan must hash the file again.
-func TestSyncScanMtimeMovedBack(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	db := openTestDB(t)
-	a := writeFile(t, dir, "a.bin", pattern(1, 500))
-
-	recorded := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
-	err := os.Chtimes(a, recorded, recorded)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	syncTree(t, db, dir)
-
-	replaced := pattern(3, 500)
-	writeFile(t, dir, "a.bin", replaced)
-
-	older := recorded.Add(-time.Hour)
-
-	err = os.Chtimes(a, older, older)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	st := syncTree(t, db, dir)
-	if st != (scanStats{walked: 1, updated: 1}) {
-		t.Fatalf("rescan stats = %+v, want 1 updated", st)
-	}
-
-	assertWholeFileHashed(t, db, a, replaced)
-
-	if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(older) {
-		t.Fatalf("mtime = %v, want %v", r.mtime, older)
-	}
-}
-
 // assertWholeFileHashed fails unless the record for path holds the
 // whole-file hash of data as its head, tail, and content.
 func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
@@ -2409,8 +2367,8 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
 			report := runStdout(t, cmdReport)
 			if strings.Contains(report, "db.sqlite") ||
 				strings.Contains(report, hardLink) ||
-				!strings.Contains(report, filepath.Join(tree, "b")) {
+				!strings.Contains(report, filepath.Join(tree, "empty2")) {
-				t.Errorf("report = %q, want a and b and none of "+
+				t.Errorf("report = %q, want the empty files and none of "+
 					"the database's files", report)
 			}
 		})

internal/sfdupes/trees.go

@@ -301,9 +301,7 @@ func fileEntry(name string, r scanRec) string {
 // collectTreeGroups groups directories by digest, counting each one on
 // prog as it is grouped, and returns every maximal group with two or
 // more members, each group's members sorted by path, groups ordered by
-// tree size descending then by first path ascending. A group whose
+// tree size descending then by first path ascending.
-// trees hold 0 bytes is left out, since removing one frees nothing. The
-// groups it implies hold 0 bytes too, so leaving it out hides nothing.
 func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 	prog *progress,
 ) [][]*treeNode {
@@ -317,9 +315,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 	var dupes [][]*treeNode
 
 	for _, g := range groups {
-		// The members share a digest, so they share a total size.
+		if len(g) < minGroupSize || suppressed(g, super) {
-		if len(g) < minGroupSize || g[0].totalSize == 0 ||
-			suppressed(g, super) {
 			continue
 		}
 

internal/sfdupes/trees_test.go

@@ -2,7 +2,6 @@ package sfdupes //nolint:testpackage // checks the unexported treeNode results
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"database/sql"
 	"os"
 	"slices"
@@ -436,35 +435,6 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
 	}
 }
 
-// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
-// holding only empty files form no group, and that an empty file still
-// counts toward the digest of a tree that holds other files.
-func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
-	t.Parallel()
-
-	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
-		sha256.Sum256(nil)
-	recs := []scanRec{
-		{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
-		{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
-		{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
-		// Without the empty file, /p/x3 is no copy of /p/x1.
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
-	}
-
-	super, dirs := treeOf(t, recs)
-
-	got := groupPaths(collectTreeGroups(dirs, super, nil))
-
-	want := [][]string{{"/p/x1", "/p/x2"}}
-	if !slices.EqualFunc(got, want, slices.Equal) {
-		t.Fatalf("groups = %v, want %v", got, want)
-	}
-}
-
 func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 	t.Parallel()