sneak/sfdupes
1
0
Fork 0
Code Issues 4 Pull Requests 3 Actions 7 Packages Projects Releases Wiki Activity

Skip removed paths silently and fix the progress line (closes #133)
check / check (push) Canceled after 0s
Details

Browse Source 
A path removed while the scan runs gets no warning and no skipped
count, in the walk or the hash phase. If the first of several hard
links is gone, the next is read. Any other error is still warned about.

On a terminal every display is the library's spinner, then the line. A
phase with an exact total ends the padded line with its percent and a
bar at a fixed column. sfdupes draws that bar: the library shows no
spinner beside its own.

The time remaining is labelled remain, not eta.

Judgement call: an operand removed after the scan starts is passed over
silently too.
Deviation: README's 80-column promise for stalled lines is narrowed to
its example.

Model: opus-5-5
This commit is contained in:
sneak
2026-10-26 04:06:11 +00:00
parent 68881f04f2
commit e80c6541d2
9 changed files with 522 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
progress_test.go
+142 -73
View File
@@ -117,24 +117,24 @@ func TestProgressLine(t *testing.T) {
label: scanPhase("load"), total: -1,
waitingFor: countingRecords,
},
want: "[1/4] load: counting the records, elapsed 10s, eta unknown",
want: "[1/4] load: counting the records, elapsed 10s, remain unknown",
}, {
p: &progress{
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
},
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
}, {
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, remain unknown",
}, {
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, remain 0s",
}, {
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: 100,
estimate: true, count: 20,
},
want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, remain 40s",
}, {
// Past the estimate the walk cannot tell how long it has left.
p: &progress{
@@ -142,13 +142,13 @@ func TestProgressLine(t *testing.T) {
estimate: true, count: 120,
},
want: "[2/4] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
"eta unknown",
"remain unknown",
}, {
// A first scan's walk has no estimate.
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
},
want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, remain unknown",
}, {
// The rate is in what the phase counts.
p: &progress{
@@ -156,7 +156,7 @@ func TestProgressLine(t *testing.T) {
count: 20,
},
want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
"eta 40s",
"remain 40s",
}}
for _, tc := range cases {
@@ -203,28 +203,28 @@ func TestLineFitsRoom(t *testing.T) {
percent bool
want string
}{{
p: hash, room: 57, percent: true,
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
p: hash, room: 60, percent: true,
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
}, {
p: hash, room: 56, percent: true,
want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40s",
p: hash, room: 59, percent: true,
want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40s",
}, {
p: hash, room: 45, percent: true,
want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40",
p: hash, room: 48, percent: true,
want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40",
}, {
// Before the library's bar, which shows the percent.
p: hash, room: 57, percent: false,
want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
// Before the bar, which shows the percent.
p: hash, room: 60, percent: false,
want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, remain 40s",
}, {
p: load, room: 40, percent: true,
want: "[1/4] load: elapsed 10s, eta unknown",
p: load, room: 43, percent: true,
want: "[1/4] load: elapsed 10s, remain unknown",
}, {
p: stalled, room: 65, percent: true,
p: stalled, room: 68, percent: true,
want: "[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
"eta 3m20s",
"remain 3m20s",
}, {
p: stalled, room: 64, percent: true,
want: "[3/4] hash: [20/100] 20%, elapsed 50s, eta 3m20s",
p: stalled, room: 67, percent: true,
want: "[3/4] hash: [20/100] 20%, elapsed 50s, remain 3m20s",
}}
for _, tc := range cases {
@@ -236,10 +236,10 @@ func TestLineFitsRoom(t *testing.T) {
}
// TestSetTotalStartsRate checks that setTotal starts a phase's rate,
// its ETA and its time with no record read from when the total becomes
// known, not from when the phase started waiting for it: after 20
// seconds of waiting for the total, the phase has not stalled 5 seconds
// after it, and has 10 seconds after it with no record read.
// its time remaining and its time with no record read from when the
// total becomes known, not from when the phase started waiting for it:
// after 20 seconds of waiting for the total, the phase has not stalled 5
// seconds after it, and has 10 seconds after it with no record read.
func TestSetTotalStartsRate(t *testing.T) {
t.Parallel()
@@ -255,14 +255,14 @@ func TestSetTotalStartsRate(t *testing.T) {
clock.Add(5 * time.Second)
want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, eta unknown"
want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, remain unknown"
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
clock.Add(5 * time.Second)
want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, eta unknown"
want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, remain unknown"
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
@@ -272,7 +272,7 @@ func TestSetTotalStartsRate(t *testing.T) {
p.increment()
}
want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, remain 40s"
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
@@ -316,7 +316,7 @@ func TestPlainProgressPrintsAtStartAndEnd(t *testing.T) {
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown"
if len(lines) != 2 || lines[0] != want ||
!strings.HasPrefix(lines[1], "[3/4] hash: [1000/1000] 100%, ") {
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
@@ -336,7 +336,7 @@ func TestPlainProgressPrintsWhileWaiting(t *testing.T) {
}, nil, testRedraw, time.Now)
awaitOutput(t, stderr,
"[1/4] load: counting the records, elapsed 0s, eta unknown\n", 3)
"[1/4] load: counting the records, elapsed 0s, remain unknown\n", 3)
p.finish()
}
@@ -366,23 +366,23 @@ func TestPlainProgressStalledProducer(t *testing.T) {
clock.Add(5 * time.Second)
awaitOutput(t, stderr,
"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, eta 20s\n", 3)
"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, remain 20s\n", 3)
clock.Add(time.Second)
awaitOutput(t, stderr,
"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, eta 24s\n", 3)
"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, remain 24s\n", 3)
clock.Add(4 * time.Second)
awaitOutput(t, stderr,
"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, eta 40s\n",
"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, remain 40s\n",
3)
}
// TestPlainProgressRateAfterStall draws a phase's plain line by hand, a
// little over plainInterval after the line before, as a late tick
// does, through a stall and a burst of files after it: the rate covers
// the files read since the line before, while the ETA comes from the
// phase's average.
// the files read since the line before, while the time remaining comes
// from the phase's average.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestPlainProgressRateAfterStall(t *testing.T) {
@@ -410,10 +410,10 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
p.finish()
want := []string{
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown",
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, eta unknown",
"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, eta unknown",
"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, eta 2m15s",
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown",
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, remain unknown",
"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, remain unknown",
"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, remain 2m15s",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
@@ -426,8 +426,8 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
// redrawInterval, as a terminal does, through a stall of stallTime and
// a burst of files after it: the rate covers the files read over the
// last rateWindow, and falls to 0 once the burst is older than that,
// while the ETA comes from the phase's average. Plain lines show each
// draw.
// while the time remaining comes from the phase's average. Plain lines
// show each draw.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestProgressRateAfterStallRedrawn(t *testing.T) {
@@ -457,8 +457,8 @@ func TestProgressRateAfterStallRedrawn(t *testing.T) {
redraw(rateWindow + redrawInterval)
p.finish()
want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, eta 4m45s\n" +
"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, eta 4m47s\n"
want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, remain 4m45s\n" +
"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, remain 4m47s\n"
if !strings.Contains(stderr(), want) {
t.Errorf("stderr = %q, want the lines %q", stderr(), want)
}
@@ -480,10 +480,10 @@ func lastFrame(out string) string {
}
// TestTerminalRedrawsWhileWaiting checks that the terminal display of a
// phase, spinner or bar, keeps being drawn while the phase waits for its
// next item, with the elapsed time as of now, and shows every item
// completed before the wait. stderr is not a terminal here, so the
// displays take it to be fallbackWidth wide.
// phase, with a bar or without, keeps being drawn while the phase waits
// for its next item, with the elapsed time as of now, and shows every
// item completed before the wait. The terminal is 120 columns wide, so
// the hash phase's display has room for its bar.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalRedrawsWhileWaiting(t *testing.T) {
@@ -506,7 +506,8 @@ func TestTerminalRedrawsWhileWaiting(t *testing.T) {
for _, tc := range cases {
before := len(stderr())
since := func() string { return stderr()[before:] }
p := startProgress(tc.params, terminalWidth, testRedraw, time.Now)
p := startProgress(tc.params, func() int { return 120 }, testRedraw,
time.Now)
for range 50 {
p.increment()
@@ -552,13 +553,13 @@ func drawLate(stderr func() string, params progressParams, width int,
return stderr()[before:]
}
// TestTerminalLinesKeepETA checks the line each display shows on an
// 80-column terminal at a large scan's counts and times, and the line
// of trees' read phase at the same counts: the rate is left out, so
// that the elapsed time and the ETA stay whole.
// TestTerminalLinesKeepTimeRemaining checks the line each display shows
// on an 80-column terminal at a large scan's counts and times, and the
// line of trees' read phase at the same counts: the rate is left out,
// so that the elapsed time and the time remaining stay whole.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalLinesKeepETA(t *testing.T) {
func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
stderr := captureStderr(t)
cases := []struct {
@@ -571,19 +572,19 @@ func TestTerminalLinesKeepETA(t *testing.T) {
estimate: true,
},
want: "[2/4] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
"eta unknown",
"remain unknown",
}, {
params: progressParams{
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
},
want: "[3/4] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
"remain 17h30m0s",
}, {
params: progressParams{
label: treesPhase("read"), unit: "records", total: 9876543,
},
want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
"remain 17h30m0s",
}}
for _, tc := range cases {
@@ -598,9 +599,10 @@ func TestTerminalLinesKeepETA(t *testing.T) {
// terminal once the phase has stalled, its last item done 42 seconds
// before. At an ordinary scan's counts and times, 12345 of 45678 items
// done four and a half minutes in, the line shows where the phase waits
// whole, beside the phase, progress, elapsed time and ETA. At a large
// scan's, 1234567 of 9876543 two and a half hours in, those four still
// show whole, as the words give way first where they do not fit.
// whole, beside the phase, progress, elapsed time and time remaining.
// At a large scan's, 1234567 of 9876543 two and a half hours in, those
// four still show whole, as the words give way first where they do not
// fit.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalStalledLines(t *testing.T) {
@@ -652,7 +654,7 @@ func TestTerminalStalledLines(t *testing.T) {
shown := drawStalled(params, 45678, 12345,
4*time.Minute+30*time.Second)
want := params.label + ": [12345/" + approx + "45678] 27%, stalled " +
params.stalledOn + ", elapsed 4m30s, eta 12m9s"
params.stalledOn + ", elapsed 4m30s, remain 12m9s"
if !strings.Contains(shown, want) {
t.Errorf("terminal shows %q, want %q", shown, want)
@@ -661,7 +663,7 @@ func TestTerminalStalledLines(t *testing.T) {
shown = drawStalled(params, 9876543, 1234567, 150*time.Minute)
for _, want := range []string{
params.label + ": [1234567/" + approx + "9876543] 12%, ",
", elapsed 2h30m0s, eta 17h30m0s",
", elapsed 2h30m0s, remain 17h30m0s",
} {
if !strings.Contains(shown, want) {
t.Errorf("terminal shows %q, want %q", shown, want)
@@ -706,13 +708,78 @@ func TestTerminalLinesFitWidth(t *testing.T) {
}
}
// TestTerminalLinesStartWithSpinner draws a display with no total and
// one with a bar, two and a half hours into the phase, on a terminal 120
// columns wide. Each line must start with the library's spinner in the
// first column, then a space and the phase's label.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalLinesStartWithSpinner(t *testing.T) {
stderr := captureStderr(t)
for _, params := range []progressParams{
{label: scanPhase("walk"), unit: scanUnit, total: -1},
{label: scanPhase("hash"), unit: scanUnit, total: 9876543},
} {
shown := lastFrame(drawLate(stderr, params, 120, 1234567))
spinner, rest, _ := strings.Cut(shown, " ")
if utf8.RuneCountInString(spinner) != 1 ||
!strings.HasPrefix(rest, params.label+": ") {
t.Errorf("terminal shows %q, want the spinner, a space and %q",
shown, params.label)
}
if params.total > 0 && !strings.Contains(shown, "% |") {
t.Errorf("terminal shows %q, want a bar", shown)
}
}
}
// TestTerminalBarStaysAtRightEdge draws the hash phase's display two
// and a half hours in, at two counts whose lines differ in length, on
// terminals 110 and 120 columns wide. Its bar must start the same number
// of columns from the terminal's right edge each time: a line that grows
// or shrinks, as its time remaining does, never moves the bar.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalBarStaysAtRightEdge(t *testing.T) {
stderr := captureStderr(t)
params := progressParams{
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
}
widths := []int{110, 120}
counts := []int64{1234567, 9876543}
fromEdge := make([]int, 0, len(widths)*len(counts))
for _, width := range widths {
for _, count := range counts {
shown := lastFrame(drawLate(stderr, params, width, count))
beforeBar, _, found := strings.Cut(shown, "% |")
if !found {
t.Fatalf("terminal %d wide shows %q, want a bar", width, shown)
}
fromEdge = append(fromEdge,
width-utf8.RuneCountInString(beforeBar))
}
}
if slices.Min(fromEdge) != slices.Max(fromEdge) {
t.Errorf("bar starts %v columns from the right edge, want the same "+
"every time", fromEdge)
}
}
// TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
// and a half hours into the phase, on a terminal 120 columns wide, then
// makes the terminal narrower than that line and draws it again a few
// times. Nothing written after that, the erase before each redraw
// included, may reach the narrower terminal's last column: it would
// wrap, and every redraw would leave a line behind. At these widths the
// spinner stays a spinner and the bar gives way to the spinner.
// wrap, and every redraw would leave a line behind. At the narrower
// width the hash phase's bar gives way.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
@@ -770,10 +837,10 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
}
}
// TestProgressWarningsOnOwnLines drives the terminal displays, spinner
// and bar, through a run of warnings with no items between them, as when
// the walk meets many unreadable paths, for many redraws: every warning
// must land on a line of its own, never inside a redraw.
// TestProgressWarningsOnOwnLines drives the terminal displays, with a
// bar and without, through a run of warnings with no items between
// them, as when the walk meets many unreadable paths, for many redraws:
// every warning must land on a line of its own, never inside a redraw.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestProgressWarningsOnOwnLines(t *testing.T) {
@@ -819,11 +886,12 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
}
}
// TestProgressWriterRowsOnOwnLines drives the terminal displays,
// spinner and bar, through a run of rows written through the display's
// TestProgressWriterRowsOnOwnLines drives the terminal displays, with a
// bar and without, through a run of rows written through the display's
// writer to the same terminal, as stdout is in a plain "sfdupes trees",
// each written out at once, for many redraws: the terminal must show
// every row whole, in order, on a line of its own.
// every row whole, in order, on a line of its own. The terminal is 120
// columns wide, so the write phase's display has room for its bar.
//
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
func TestProgressWriterRowsOnOwnLines(t *testing.T) {
@@ -840,7 +908,8 @@ func TestProgressWriterRowsOnOwnLines(t *testing.T) {
var want []string
for _, params := range cases {
p := startProgress(params, terminalWidth, testRedraw, time.Now)
p := startProgress(params, func() int { return 120 }, testRedraw,
time.Now)
out := p.writer(os.Stderr)
// No pause between rows: one is garbled only if a redraw lands