Show the progress rate over the last 5 seconds, and where a phase waits after 10 seconds with no item done (closes #125)

The rate was the phase's average, so a burst after a long wait read as
over 1,000 files a second. It now covers the items completed over the
last 5 seconds, or since the line before off a terminal; the ETA still
comes from the average. Once no item has completed for 10 seconds, the
rate gives way to "stalled" and where the phase waits, as "stalled
disk/db": disk, db, out (stdout) or cpu, or two of them. A line too
wide for the terminal loses its bar first, then its rate or those
words, so the phase, progress, elapsed time and ETA stay whole. The
content phase's last partial batch is now written under a display of
its own. The tests set the display's clock by hand.

Deviation: the stall words hold no time and name where the phase waits, not what it does.
Judgement call: trees' group phase, which works in memory, says "cpu".

Model: opus-5-5

progress.go

@@ -22,6 +22,15 @@ const plainInterval = 5 * time.Second
 // a TTY.
 const redrawInterval = 100 * time.Millisecond
 
+// rateWindow is the time the rate a display shows covers: the items
+// completed over the last rateWindow, which is the time since the line
+// before when stderr is not a TTY.
+const rateWindow = 5 * time.Second
+
+// stallTime is how long a phase goes with no item completed before its
+// line says it has stalled, and where it waits, in place of the rate.
+const stallTime = 10 * time.Second
+
 // spinnerType selects the progressbar library's spinner style.
 const spinnerType = 14
 
@@ -89,6 +98,14 @@ type progressParams struct {
 	total      int64  // the items the phase will complete; -1 when not known
 	estimate   bool   // total is an estimate, which the count may pass
 	waitingFor string // what the phase waits for until setTotal is called
+	stalledOn  string // where the phase waits, as "disk/db"; see stallTime
+}
+
+// drawnCount is the count of items completed as of one draw, and when
+// that draw was.
+type drawnCount struct {
+	at    time.Time
+	count int64
 }
 
 // progress renders one phase's progress on stderr. On a TTY the
@@ -107,9 +124,10 @@ type progressParams struct {
 // phase's helpers without rendering anything, and its writer writes
 // with no display to clear.
 type progress struct {
-	width func() int    // reads the terminal's width; nil when not a TTY
-	stop  chan struct{} // closed by finish to end the draws
-	done  chan struct{} // closed once the draws have ended
+	now   func() time.Time // reads the time
+	width func() int       // reads the terminal's width; nil when not a TTY
+	stop  chan struct{}    // closed by finish to end the draws
+	done  chan struct{}    // closed once the draws have ended
 
 	mu         sync.Mutex
 	bar        *progressbar.ProgressBar // what the library draws; nil until then
@@ -120,33 +138,42 @@ type progress struct {
 	total      int64
 	estimate   bool
 	waitingFor string
+	stalledOn  string
 	count      int64
 	start      time.Time // when the phase started, for the elapsed time
-	rateStart  time.Time // when the count started, for the rate
+	rateStart  time.Time // when the count started, for the ETA
+	lastItem   time.Time // when the last item was completed, or the count started
+	// drawn holds the count at each draw over the last rateWindow, oldest
+	// first; the rate counts from the first. It keeps the draw before the
+	// latest even when that is older, as the line before is when lines
+	// are plainInterval apart and a tick comes late.
+	drawn []drawnCount
 }
 
 // newProgress starts the display of one phase on stderr.
 func newProgress(params progressParams) *progress {
 	if stderrIsTTY() {
-		return startProgress(params, terminalWidth, redrawInterval)
+		return startProgress(params, terminalWidth, redrawInterval, time.Now)
 	}
 
-	return startProgress(params, nil, plainInterval)
+	return startProgress(params, nil, plainInterval, time.Now)
 }
 
 // startProgress shows the phase params describes at once, and again
 // every interval until finish: with the progressbar library on a
 // terminal, whose width each draw reads by calling width, or as a plain
-// line when width is nil.
+// line when width is nil. The display reads the time by calling now.
 func startProgress(params progressParams, width func() int,
-	interval time.Duration,
+	interval time.Duration, now func() time.Time,
 ) *progress {
-	now := time.Now()
+	start := now()
 	p := &progress{
-		width: width, stop: make(chan struct{}), done: make(chan struct{}),
+		now: now, width: width,
+		stop: make(chan struct{}), done: make(chan struct{}),
 		label: params.label, unit: params.unit, total: params.total,
 		estimate: params.estimate, waitingFor: params.waitingFor,
-		start: now, rateStart: now,
+		stalledOn: params.stalledOn, start: start, rateStart: start,
+		lastItem: start, drawn: []drawnCount{{at: start}},
 	}
 
 	// Show the zero state at once: the first item may take minutes,
@@ -173,32 +200,42 @@ func startProgress(params progressParams, width func() int,
 }
 
 // draw shows the phase's progress: on a TTY in place of the line
-// before, and otherwise as a plain line.
+// before, and otherwise as a plain line. It first adds the count to
+// drawn, then drops from drawn the draws older than rateWindow, except
+// the one before this draw.
 func (p *progress) draw() {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
+	now := p.now()
+
+	p.drawn = append(p.drawn, drawnCount{at: now, count: p.count})
+	for len(p.drawn) > 2 && now.Sub(p.drawn[0].at) > rateWindow {
+		p.drawn = p.drawn[1:]
+	}
+
 	if p.width == nil {
-		text, _ := p.line(math.MaxInt, true)
-		fmt.Fprintln(os.Stderr, text)
+		fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, true))
 
 		return
 	}
 
 	width := p.width()
 
-	// A phase with an exact total gets the library's bar when its line,
-	// without the percent the bar shows, fits whole beside the bar. A
-	// total of 0 leaves the bar nothing to fill.
+	// A phase with an exact total gets the library's bar when its whole
+	// line, without the percent the bar shows, fits beside the bar: the
+	// bar gives way before any part of the line. A total of 0 leaves the
+	// bar nothing to fill.
 	text, withBar := "", false
 	if p.total > 0 && !p.estimate {
-		text, withBar = p.line(width-barMargin, false)
+		text = p.line(now, math.MaxInt, false)
+		withBar = len(text) <= width-barMargin
 	}
 
 	margin := barMargin
 	if !withBar {
 		margin = spinnerMargin
-		text, _ = p.line(width-margin, true)
+		text = p.line(now, width-margin, true)
 	}
 
 	// The library makes a bar or a spinner, and before each draw erases
@@ -255,18 +292,22 @@ func (p *progress) newBar(withBar bool, width int) {
 }
 
 // setTotal gives a phase that started without a total its total, once
-// it is known. The rate, and so the ETA, count from then on: the time
-// spent waiting for the total says nothing about how fast items go.
+// it is known. The rate, the ETA and the time with no item completed
+// count from then on: the time spent waiting for the total says nothing
+// about how fast items go.
 func (p *progress) setTotal(total int64) {
 	if p == nil {
 		return
 	}
 
+	now := p.now()
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
 	p.total = total
-	p.rateStart = time.Now()
+	p.rateStart, p.lastItem = now, now
+	p.drawn = []drawnCount{{at: now, count: p.count}}
 }
 
 // increment records one completed item, which the display shows at its
@@ -276,10 +317,13 @@ func (p *progress) increment() {
 		return
 	}
 
+	now := p.now()
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
 	p.count++
+	p.lastItem = now
 }
 
 // warnf prints a one-line warning to stderr without corrupting the
@@ -391,50 +435,68 @@ func (p *progress) finish() {
 	}
 }
 
-// line formats the phase's progress in at most room columns, as
+// line formats the phase's progress as of now in at most room columns,
+// as
 //
 //	[3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
 //
 // where "files" is the phase's unit, without the percent when percent
 // is false, as before the library's bar, which shows its own. A phase
-// with no total shows the items done instead, or what it waits for.
+// with no total shows the items done instead, or what it waits for. The
+// rate is that since the first draw in drawn, and the ETA comes from
+// the rate since the count started. Once no item has been completed for
+// stallTime, the rate, which would read 0, gives way to the word
+// "stalled" and where the phase waits, as "stalled disk/db".
 //
 // The phase and its place, the progress, the elapsed time and the ETA
-// are kept whole. A line wider than room leaves out the rate, or what
-// the phase waits for; only if it is still wider is it cut at room, and
-// line reports that it does not fit. Every part is ASCII, so its length
-// is its width in columns. p.mu must be held.
-func (p *progress) line(room int, percent bool) (string, bool) {
-	elapsed := time.Since(p.start).Round(time.Second)
+// are kept whole. A line wider than room leaves out the rate, the words
+// of a stalled phase, or what the phase waits for; only if it is still
+// wider is it cut at room. Every part is ASCII, so its length is its
+// width in columns. p.mu must be held.
+func (p *progress) line(now time.Time, room int, percent bool) string {
+	elapsed := now.Sub(p.start).Round(time.Second)
 
 	// head and tail hold the parts kept whole; extra, between them, is
 	// the part left out.
 	var head, extra, tail string
 
 	if p.total < 0 && p.waitingFor != "" {
-		head, extra = p.label+": ", p.waitingFor
+		head, extra = p.label+": ", p.waitingFor+", "
 		tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
 	} else {
-		rate := 0.0
-		if s := time.Since(p.rateStart).Seconds(); s > 0 {
-			rate = float64(p.count) / s
-		}
+		since := p.drawn[0]
+		rate := perSecond(p.count-since.count, now.Sub(since.at))
+		average := perSecond(p.count, now.Sub(p.rateStart))
 
 		head = p.label + ": " + p.itemsDone(percent) + ", "
-		extra = fmt.Sprintf("%.0f %s/s", rate, p.unit)
-		tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(rate))
+		extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
+		tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(average))
+
+		if now.Sub(p.lastItem) >= stallTime {
+			extra = "stalled " + p.stalledOn + ", "
+		}
 	}
 
-	full, kept := head+extra+", "+tail, head+tail
+	full, kept := head+extra+tail, head+tail
 
 	switch {
 	case len(full) <= room:
-		return full, true
+		return full
 	case len(kept) <= room:
-		return kept, true
+		return kept
 	}
 
-	return kept[:max(room, 0)], false
+	return kept[:max(room, 0)]
+}
+
+// perSecond returns count items over d as items a second, or 0 when no
+// time has passed.
+func perSecond(count int64, d time.Duration) float64 {
+	if d <= 0 {
+		return 0
+	}
+
+	return float64(count) / d.Seconds()
 }
 
 // itemsDone formats the items done, as "[1234/5000] 25%", with "~"