Merge pull request 'next into main: the command moves to cmd/sfdupes' (#159) from next into main

Reviewed-on: #159

README.md

@@ -665,7 +665,6 @@ Processing:
 - Records are grouped by signature, the four values
   `(size, head, tail, content)`.
 - Every group with two or more paths is a duplicate group.
-- Empty files form no group, because removing one frees nothing.
 - Within a group, paths are sorted in byte order. The first is `first`, and
   every other path is a `dupe`.
 - Groups are ordered by size, largest first, then by `first` path. The output is
@@ -714,7 +713,7 @@ The summary line on stderr gives:
 - the reclaimable bytes, the sum of `size` over all rows.
 
 ```
-report: 14 records read, 4 duplicate groups, 7 dupe files, 10.0 KiB reclaimable
+report: 14 records read, 5 duplicate groups, 8 dupe files, 10.0 KiB reclaimable
 ```
 
 ### `trees` mode
@@ -742,9 +741,6 @@ Processing:
 - `trees` builds the directory tree and computes every directory's digest. It
   then groups the directories by digest. Every group with two or more
   directories is a duplicate-tree group.
-- `trees` leaves out a group whose trees hold 0 bytes, because removing one
-  frees nothing. An empty file still counts toward the digest of a tree that
-  holds other files.
 - `trees` leaves out a group that a group of its members' parents already
   implies. A group is left out when its members' parents are distinct
   directories that all have the same digest. The parents' group, or an
@@ -1143,14 +1139,6 @@ On a terminal:
     ⠹ [3/4] hash: [12345/98765], 92 files/s, elapsed 2m32s, remain 17m44s            12% |█         |
     ```
 
-- During `scan`'s `walk`, the line below the display shows the path of the last
-  file the walk found. Each draw shows the path of that moment, cut short before
-  the terminal's last column. The path is escaped as in the reports (see "Report
-  output format"). Each control character left after that is written as `\x` and
-  its code in two hexadecimal digits, such as `\x1b` for the escape character. A
-  control character is one that a terminal acts on instead of showing. The line
-  is erased when the walk ends, and the next phase is drawn on it.
-
 - No line reaches the terminal's last column. A line too wide for the terminal
   first leaves out its rate, the words of a stalled phase, or what the phase is
   waiting for. It leaves out its bar next. It is cut short at the edge only when
@@ -1394,7 +1382,7 @@ All of the following, run in this directory, must pass:
     head -c 2000 /dev/urandom > "$d/a/unique.bin"     # same size, different content
     printf 'x' > "$d/tiny1"; printf 'x' > "$d/tiny2"  # 1-byte duplicates
     printf 'y' > "$d/tiny3"                           # 1-byte non-duplicate
-    : > "$d/empty1"; : > "$d/empty2"                  # empty files
+    : > "$d/empty1"; : > "$d/empty2"                  # empty duplicates
     # duplicate trees: t1 and t2 are identical; t3 differs by one filename
     mkdir -p "$d/t1/sub" "$d/t2/sub" "$d/t3/sub"
     head -c 3000 /dev/urandom > "$d/t1/f1"
@@ -1419,8 +1407,8 @@ All of the following, run in this directory, must pass:
       is the path that sorts first.
     - `t1/f1`, `t2/f1` and `t3/f1` form one group.
     - `t1/sub/f2`, `t2/sub/f2` and `t3/sub/f2renamed` form one group.
-    - `tiny1` and `tiny2` form a pair.
+    - `tiny1` and `tiny2` form a pair, and so do `empty1` and `empty2`.
-    - `unique.bin`, `tiny3`, `empty1` and `empty2` appear nowhere.
+    - `unique.bin` and `tiny3` appear nowhere.
     - The groups come in order of size, largest first.
 
     `trees` prints exactly one row, `$d/t1` as `first` and `$d/t2` as `dupe`.

TODO.md

@@ -29,17 +29,6 @@
 Newest first. An entry ends with its date and its issue or branch, where those
 are known.
 
-- On a terminal, the line below the `walk` display shows the path of the last
-  file the walk found, at each draw. Its control characters are written out, as
-  `\t` or `\x1b`, so that the terminal does not act on them. The line is erased
-  when the walk ends, and the next phase is drawn on it (2026-11-11,
-  https://git.eeqj.de/sneak/sfdupes/issues/160)
-
-- `report` and `trees` leave out empty files. `report` writes no group of size
-  0, and `trees` writes no group whose trees hold 0 bytes. An empty file still
-  counts toward a tree's digest (2026-11-11,
-  https://git.eeqj.de/sneak/sfdupes/issues/178)
-
 - The Go code moved out of the repository root. `cmd/sfdupes/main.go` is the
   command, and it calls `Main` in the package `internal/sfdupes`. `main.go`
   became `internal/sfdupes/cli.go`. Every other file kept its name. The install

go.mod

@@ -3,7 +3,6 @@ module sneak.berlin/go/sfdupes
 go 1.25.7
 
 require (
-	github.com/rivo/uniseg v0.4.7
 	github.com/schollz/progressbar/v3 v3.19.1
 	github.com/spf13/cobra v1.10.2
 	golang.org/x/sys v0.46.0
@@ -19,6 +18,7 @@ require (
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	modernc.org/libc v1.74.1 // indirect
 	modernc.org/mathutil v1.7.1 // indirect

internal/sfdupes/progress.go

@@ -9,10 +9,8 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode"
 	"unicode/utf8"
 
-	"github.com/rivo/uniseg"
 	"github.com/schollz/progressbar/v3"
 	"golang.org/x/term"
 )
@@ -118,8 +116,7 @@ type drawnCount struct {
 // terminal's width. A phase with an exact total ends the line with its
 // percent and bar, at the terminal's right edge, when the terminal has
 // room for them. It is drawn again every redrawInterval, so its elapsed
-// time keeps moving while the phase waits for its next item. Each draw
+// time keeps moving while the phase waits for its next item. When
-// of the walk shows the path it reached last on the line below. When
 // stderr is not a TTY it emits no ANSI redraws: it prints line as the
 // phase starts, every plainInterval while it runs, and as it ends.
 //
@@ -144,7 +141,6 @@ type progress struct {
 	estimate   bool
 	waitingFor string
 	stalledOn  string
-	path       string // the path the walk reached last; "" for none
 	count      int64
 	start      time.Time // when the phase started, for the elapsed time
 	rateStart  time.Time // when the count started, for the time remaining
@@ -243,73 +239,6 @@ func (p *progress) draw() {
 	// less the terminal's last column, which spinnerMargin keeps clear. A
 	// bar's cells take one column each but more than one byte.
 	p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
-
-	if p.path != "" {
-		// The library draws the path on the line below and returns to
-		// the phase's line. The path is escaped as warnings are, and its
-		// other control characters are written out, so that the terminal
-		// shows them instead of acting on them. It stays clear of the
-		// last column, so that no terminal wraps it.
-		shown := escapeControlChars(escapePath(p.path))
-		_ = p.bar.AddDetail(cutToColumns(shown, width-1))
-	}
-}
-
-// escapeControlChars returns s with each control character written as
-// \x and its code in two hex digits, as \x1b for the escape character.
-// The control characters are U+0000 to U+001F, U+007F and U+0080 to
-// U+009F, as unicode.IsControl reports them. Every other byte is kept
-// as it is.
-func escapeControlChars(s string) string {
-	var b strings.Builder
-
-	for rest := s; rest != ""; {
-		char, size := utf8.DecodeRuneInString(rest)
-		if unicode.IsControl(char) {
-			fmt.Fprintf(&b, `\x%02x`, char)
-		} else {
-			b.WriteString(rest[:size])
-		}
-
-		rest = rest[size:]
-	}
-
-	return b.String()
-}
-
-// cutToColumns returns s cut to at most columns terminal columns, at a
-// character boundary. A wide character, as in Chinese, takes two
-// columns.
-func cutToColumns(s string, columns int) string {
-	used, end, state := 0, 0, -1
-
-	for rest := s; rest != ""; {
-		char, next, width, nextState := uniseg.FirstGraphemeClusterInString(
-			rest, state)
-		if used+width > columns {
-			break
-		}
-
-		used += width
-		end += len(char)
-		rest, state = next, nextState
-	}
-
-	return s[:end]
-}
-
-// clear erases the display from the terminal: its line, and the path on
-// the line below when one is shown. p.mu must be held.
-func (p *progress) clear() {
-	if p.bar == nil {
-		return
-	}
-
-	if p.path != "" {
-		_ = p.bar.AddDetail("")
-	}
-
-	_ = p.bar.Clear()
 }
 
 // newBar has the library make what draw draws: its spinner, followed by
@@ -334,8 +263,6 @@ func (p *progress) newBar(width int) {
 		// The text carries the elapsed time and the time remaining.
 		progressbar.OptionSetElapsedTime(false),
 		progressbar.OptionSetPredictTime(false),
-		// The line below the phase's line, for the walk's path.
-		progressbar.OptionSetMaxDetailRow(1),
 	)
 	p.widest = 0
 }
@@ -387,19 +314,6 @@ func (p *progress) increment() {
 	p.lastItem = now
 }
 
-// setPath records the path the walk reached last, which a display on a
-// terminal shows at its next draw.
-func (p *progress) setPath(path string) {
-	if p == nil {
-		return
-	}
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	p.path = path
-}
-
 // warnf prints a one-line warning to stderr without corrupting the
 // display. The whole message is escaped like a report's path columns,
 // so a path holding a newline cannot split the warning.
@@ -413,9 +327,11 @@ func (p *progress) warnf(format string, args ...any) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	// The warning takes the display's line; the next draw shows the
+	if p.bar != nil {
-	// display again, on the line below.
+		// The warning takes the display's line; the next draw shows
-	p.clear()
+		// the display again, on the line below.
+		_ = p.bar.Clear()
+	}
 
 	fmt.Fprintln(os.Stderr, msg)
 }
@@ -476,7 +392,9 @@ func (w *progressWriter) Flush() error {
 		w.p.mu.Lock()
 		defer w.p.mu.Unlock()
 
-		w.p.clear()
+		if w.p.bar != nil {
+			_ = w.p.bar.Clear()
+		}
 	}
 
 	_, err := w.out.Write(w.buf)
@@ -496,13 +414,6 @@ func (p *progress) finish() {
 	close(p.stop)
 	<-p.done
 
-	// The walk's path is only for someone watching the walk. Its line is
-	// erased, and the next phase is drawn on it.
-	p.mu.Lock()
-	p.clear()
-	p.path = ""
-	p.mu.Unlock()
-
 	p.draw()
 
 	if p.width != nil {

internal/sfdupes/progress_test.go

@@ -6,7 +6,6 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"regexp"
 	"slices"
 	"strings"
 	"sync"
@@ -1139,157 +1138,3 @@ func TestWarningAfterLastItemKeepsBar(t *testing.T) {
 		t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
 	}
 }
-
-func TestCutToColumns(t *testing.T) {
-	t.Parallel()
-
-	cases := []struct {
-		s       string
-		columns int
-		want    string
-	}{
-		{"/srv/a", 10, "/srv/a"},
-		{"/srv/photos", 8, "/srv/pho"},
-		// Each of these two fullwidth letters takes two columns.
-		{"/ＡＢ/a", 4, "/Ａ"},
-		{"/ＡＢ/a", 5, "/ＡＢ"},
-		// An accent that combines with the letter before stays with it.
-		{"/café/a", 5, "/café"},
-		{"/srv", 0, ""},
-	}
-
-	for _, tc := range cases {
-		if got := cutToColumns(tc.s, tc.columns); got != tc.want {
-			t.Errorf("cutToColumns(%q, %d) = %q, want %q",
-				tc.s, tc.columns, got, tc.want)
-		}
-	}
-}
-
-func TestEscapeControlChars(t *testing.T) {
-	t.Parallel()
-
-	cases := map[string]string{
-		"/srv/music":    "/srv/music",
-		"/a\x1b[2Jb":    `/a\x1b[2Jb`,
-		"/a\x00\a\x7fb": `/a\x00\x07\x7fb`,
-		// U+009B takes two bytes in UTF-8.
-		"/a\u009b2Jb":     `/a\x9b2Jb`,
-		"/café/ＡＢ":        "/café/ＡＢ",
-		"/latin1-caf\xe9": "/latin1-caf\xe9",
-	}
-	for in, want := range cases {
-		if got := escapeControlChars(in); got != want {
-			t.Errorf("escapeControlChars(%q) = %q, want %q", in, got, want)
-		}
-	}
-}
-
-// pathLines returns the text of each line the library wrote below a
-// display's line. It writes one as an erase to the end of the line, a
-// carriage return, the text and a newline.
-func pathLines(out string) []string {
-	matches := regexp.MustCompile("\x1b\\[K\r([^\n]*)\n").
-		FindAllStringSubmatch(out, -1)
-	lines := make([]string, 0, len(matches))
-
-	for _, m := range matches {
-		lines = append(lines, m[1])
-	}
-
-	return lines
-}
-
-// TestTerminalWalkShowsPathBelow checks that the walk's display on a
-// terminal 30 columns wide shows the path it reached last on the line
-// below, cut to 29 columns. That line is erased before a warning is
-// printed, and again when the walk ends, so the next phase is drawn on
-// an empty line.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalWalkShowsPathBelow(t *testing.T) {
-	stderr := captureStderr(t)
-	p := startProgress(progressParams{
-		label: scanPhase("walk"), unit: scanUnit, total: -1,
-	}, func() int { return 30 }, testRedraw, time.Now)
-
-	const shown = "/srv/photos/2026/holiday/beac"
-
-	p.setPath("/srv/photos/2026/holiday/beach.jpg")
-	p.increment()
-	awaitOutput(t, stderr, shown, 1)
-
-	p.warnf("warning")
-
-	before, _, _ := strings.Cut(stderr(), "warning")
-	if lines := pathLines(before); lines[len(lines)-1] != "" {
-		t.Errorf("line below shows %q as the warning is printed, want it "+
-			"empty", lines[len(lines)-1])
-	}
-
-	// Drawn again below the warning.
-	awaitOutput(t, stderr, shown, strings.Count(before, shown)+1)
-	p.finish()
-
-	lines := pathLines(stderr())
-	if !slices.Contains(lines, shown) {
-		t.Errorf("lines below the display = %q, want %q among them",
-			lines, shown)
-	}
-
-	if last := lines[len(lines)-1]; last != "" {
-		t.Errorf("line below the display shows %q after the walk, want it "+
-			"empty", last)
-	}
-}
-
-// TestTerminalWalkEscapesControlChars checks that a walked path holding
-// escape characters, which would clear the screen and set the window
-// title, puts none on the line below the walk's display. The line shows
-// each control character written out, as \x1b.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalWalkEscapesControlChars(t *testing.T) {
-	stderr := captureStderr(t)
-	p := startProgress(progressParams{
-		label: scanPhase("walk"), unit: scanUnit, total: -1,
-	}, func() int { return 80 }, testRedraw, time.Now)
-
-	p.setPath("/srv/\x1b[2J\x1b]0;title\a/beach.jpg")
-	p.increment()
-	awaitOutput(t, stderr, "beach.jpg", 1)
-	p.finish()
-
-	lines := pathLines(stderr())
-	for _, line := range lines {
-		if strings.Contains(line, "\x1b") {
-			t.Errorf("line below the display = %q, want no escape "+
-				"character", line)
-		}
-	}
-
-	const shown = `/srv/\x1b[2J\x1b]0;title\x07/beach.jpg`
-	if !slices.Contains(lines, shown) {
-		t.Errorf("lines below the display = %q, want %q among them",
-			lines, shown)
-	}
-}
-
-// TestPlainWalkShowsNoPath checks that the walk's display prints no path
-// when stderr is not a terminal.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestPlainWalkShowsNoPath(t *testing.T) {
-	stderr := captureStderr(t)
-	p := startProgress(progressParams{
-		label: scanPhase("walk"), unit: scanUnit, total: -1,
-	}, nil, testRedraw, time.Now)
-
-	p.setPath("/srv/a")
-	p.increment()
-	p.finish()
-
-	if strings.Contains(stderr(), "/srv/a") {
-		t.Errorf("stderr = %q, want no path", stderr())
-	}
-}

internal/sfdupes/report.go

@@ -195,11 +195,10 @@ func (w *reportWriter) add(r scanRec) error {
 }
 
 // endGroup ends the paths read with w.sig: two or more are a duplicate
-// group, held with its paths sorted until writeGroups writes it. Empty
+// group, held with its paths sorted until writeGroups writes it.
-// files form no group, since removing one frees nothing.
 func (w *reportWriter) endGroup() {
-	if len(w.paths) < minGroupSize || w.sig.size == 0 {
+	if len(w.paths) < minGroupSize {
-		// The paths are dropped, and their slice reused.
+		// A path with no duplicate is dropped, and its slice reused.
 		w.paths = w.paths[:0]
 
 		return

internal/sfdupes/report_test.go

@@ -434,27 +434,6 @@ func TestDupeGroups(t *testing.T) {
 	}
 }
 
-// TestDupeGroupsLeavesOutEmptyFiles checks that empty files form no
-// duplicate group, while 1-byte files still do.
-func TestDupeGroupsLeavesOutEmptyFiles(t *testing.T) {
-	t.Parallel()
-
-	e, x := sha256.Sum256(nil), testHash("x")
-	recs := []scanRec{
-		{size: 0, head: e, tail: e, content: e, path: "/a/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/b/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/c/__init__.py"},
-		{size: 1, head: x, tail: x, content: x, path: "/tiny1"},
-		{size: 1, head: x, tail: x, content: x, path: "/tiny2"},
-	}
-
-	groups := dupeGroupsOf(t, recs)
-	if len(groups) != 1 || groups[0].size != 1 ||
-		!slices.Equal(groups[0].paths, []string{"/tiny1", "/tiny2"}) {
-		t.Fatalf("groups = %+v, want only /tiny1 /tiny2 of size 1", groups)
-	}
-}
-
 func TestDupeGroupsContentSeparates(t *testing.T) {
 	t.Parallel()
 

internal/sfdupes/scan.go

@@ -542,7 +542,6 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
 
 		s.st.walked++
 
-		prog.setPath(ev.rec.path)
 		prog.increment()
 
 		old, ok := s.existing[ev.rec.path]

internal/sfdupes/scan_test.go

@@ -1185,12 +1185,11 @@ func assertSmokeDupeGroups(t *testing.T, dir string, db *sql.DB) {
 	t.Helper()
 
 	groups := dupeGroups(t, db)
-	if len(groups) != 4 {
+	if len(groups) != 5 {
-		t.Fatalf("len(groups) = %d, want 4", len(groups))
+		t.Fatalf("len(groups) = %d, want 5", len(groups))
 	}
 
-	// The empty files empty1 and empty2 form no group.
+	wantSizes := []int64{3000, 2000, 100, 1, 0}
-	wantSizes := []int64{3000, 2000, 100, 1}
 	for i, g := range groups {
 		if g.size != wantSizes[i] {
 			t.Errorf("groups[%d].size = %d, want %d",
@@ -2368,8 +2367,8 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
 			report := runStdout(t, cmdReport)
 			if strings.Contains(report, "db.sqlite") ||
 				strings.Contains(report, hardLink) ||
-				!strings.Contains(report, filepath.Join(tree, "b")) {
+				!strings.Contains(report, filepath.Join(tree, "empty2")) {
-				t.Errorf("report = %q, want a and b and none of "+
+				t.Errorf("report = %q, want the empty files and none of "+
 					"the database's files", report)
 			}
 		})

internal/sfdupes/trees.go

@@ -301,9 +301,7 @@ func fileEntry(name string, r scanRec) string {
 // collectTreeGroups groups directories by digest, counting each one on
 // prog as it is grouped, and returns every maximal group with two or
 // more members, each group's members sorted by path, groups ordered by
-// tree size descending then by first path ascending. A group whose
+// tree size descending then by first path ascending.
-// trees hold 0 bytes is left out, since removing one frees nothing. The
-// groups it implies hold 0 bytes too, so leaving it out hides nothing.
 func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 	prog *progress,
 ) [][]*treeNode {
@@ -317,9 +315,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 	var dupes [][]*treeNode
 
 	for _, g := range groups {
-		// The members share a digest, so they share a total size.
+		if len(g) < minGroupSize || suppressed(g, super) {
-		if len(g) < minGroupSize || g[0].totalSize == 0 ||
-			suppressed(g, super) {
 			continue
 		}
 

internal/sfdupes/trees_test.go

@@ -2,7 +2,6 @@ package sfdupes //nolint:testpackage // checks the unexported treeNode results
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"database/sql"
 	"os"
 	"slices"
@@ -436,35 +435,6 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
 	}
 }
 
-// TestCollectTreeGroupsLeavesOutEmptyTrees checks that directories
-// holding only empty files form no group, and that an empty file still
-// counts toward the digest of a tree that holds other files.
-func TestCollectTreeGroupsLeavesOutEmptyTrees(t *testing.T) {
-	t.Parallel()
-
-	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"),
-		sha256.Sum256(nil)
-	recs := []scanRec{
-		{size: 0, head: e, tail: e, content: e, path: "/e/a/.gitkeep"},
-		{size: 0, head: e, tail: e, content: e, path: "/e/b/.gitkeep"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
-		{size: 0, head: e, tail: e, content: e, path: "/p/x1/.gitkeep"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
-		{size: 0, head: e, tail: e, content: e, path: "/p/x2/.gitkeep"},
-		// Without the empty file, /p/x3 is no copy of /p/x1.
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x3/f"},
-	}
-
-	super, dirs := treeOf(t, recs)
-
-	got := groupPaths(collectTreeGroups(dirs, super, nil))
-
-	want := [][]string{{"/p/x1", "/p/x2"}}
-	if !slices.EqualFunc(got, want, slices.Equal) {
-		t.Fatalf("groups = %v, want %v", got, want)
-	}
-}
-
 func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 	t.Parallel()