Show the progress rate over the last 5 seconds, and what a phase is doing after 10 seconds with no item done (closes #125)

A display's rate was the phase's average since it started, so after a
long wait a burst of files read as a steady rate of over 1,000 files a
second. It now covers the items completed over the last 5 seconds, or
since the line before off a terminal, and falls to 0 while nothing
completes; the ETA still comes from the average. Once no item has
completed for 10 seconds, the line says so and what the phase is doing,
a phrase each phase gives where it is defined. A narrow terminal leaves
that out after the rate, keeping the phase, progress, elapsed time and
ETA whole. The tests set the display's clock by hand.

Judgement call: the line reads "no files finished", in the phase's unit.
Judgement call: trees' group phase says "grouping directories".

Model: opus-5-5

progress.go

@@ -22,6 +22,15 @@ const plainInterval = 5 * time.Second
 // a TTY.
 const redrawInterval = 100 * time.Millisecond
 
+// rateWindow is the time the rate a display shows covers: the items
+// completed over the last rateWindow, which is the time since the line
+// before when stderr is not a TTY.
+const rateWindow = 5 * time.Second
+
+// stallTime is how long a phase goes with no item completed before its
+// line says so, and what the phase is doing.
+const stallTime = 10 * time.Second
+
 // spinnerType selects the progressbar library's spinner style.
 const spinnerType = 14
 
@@ -59,6 +68,10 @@ const (
 	openingDatabase = "opening the database"
 )
 
+// readingDatabase is what the display of a phase that reads the records
+// says it is doing once no record has been read for stallTime.
+const readingDatabase = "reading the database"
+
 // stderrIsTTY reports whether stderr is attached to a terminal.
 func stderrIsTTY() bool {
 	return term.IsTerminal(int(os.Stderr.Fd()))
@@ -89,6 +102,14 @@ type progressParams struct {
 	total      int64  // the items the phase will complete; -1 when not known
 	estimate   bool   // total is an estimate, which the count may pass
 	waitingFor string // what the phase waits for until setTotal is called
+	doing      string // what the phase is doing, as "reading files"; see stallTime
+}
+
+// drawnCount is the count of items completed as of one draw, and when
+// that draw was.
+type drawnCount struct {
+	at    time.Time
+	count int64
 }
 
 // progress renders one phase's progress on stderr. On a TTY the
@@ -107,9 +128,10 @@ type progressParams struct {
 // phase's helpers without rendering anything, and its writer writes
 // with no display to clear.
 type progress struct {
-	width func() int    // reads the terminal's width; nil when not a TTY
-	stop  chan struct{} // closed by finish to end the draws
-	done  chan struct{} // closed once the draws have ended
+	now   func() time.Time // reads the time
+	width func() int       // reads the terminal's width; nil when not a TTY
+	stop  chan struct{}    // closed by finish to end the draws
+	done  chan struct{}    // closed once the draws have ended
 
 	mu         sync.Mutex
 	bar        *progressbar.ProgressBar // what the library draws; nil until then
@@ -120,33 +142,42 @@ type progress struct {
 	total      int64
 	estimate   bool
 	waitingFor string
+	doing      string
 	count      int64
 	start      time.Time // when the phase started, for the elapsed time
-	rateStart  time.Time // when the count started, for the rate
+	rateStart  time.Time // when the count started, for the ETA
+	lastItem   time.Time // when the last item was completed, or the count started
+	// drawn holds the count at each draw over the last rateWindow, oldest
+	// first; the rate counts from the first. It keeps the draw before the
+	// latest even when that is older, as the line before is when lines
+	// are plainInterval apart and a tick comes late.
+	drawn []drawnCount
 }
 
 // newProgress starts the display of one phase on stderr.
 func newProgress(params progressParams) *progress {
 	if stderrIsTTY() {
-		return startProgress(params, terminalWidth, redrawInterval)
+		return startProgress(params, terminalWidth, redrawInterval, time.Now)
 	}
 
-	return startProgress(params, nil, plainInterval)
+	return startProgress(params, nil, plainInterval, time.Now)
 }
 
 // startProgress shows the phase params describes at once, and again
 // every interval until finish: with the progressbar library on a
 // terminal, whose width each draw reads by calling width, or as a plain
-// line when width is nil.
+// line when width is nil. The display reads the time by calling now.
 func startProgress(params progressParams, width func() int,
-	interval time.Duration,
+	interval time.Duration, now func() time.Time,
 ) *progress {
-	now := time.Now()
+	start := now()
 	p := &progress{
-		width: width, stop: make(chan struct{}), done: make(chan struct{}),
+		now: now, width: width,
+		stop: make(chan struct{}), done: make(chan struct{}),
 		label: params.label, unit: params.unit, total: params.total,
 		estimate: params.estimate, waitingFor: params.waitingFor,
-		start: now, rateStart: now,
+		doing: params.doing, start: start, rateStart: start,
+		lastItem: start, drawn: []drawnCount{{at: start}},
 	}
 
 	// Show the zero state at once: the first item may take minutes,
@@ -173,13 +204,22 @@ func startProgress(params progressParams, width func() int,
 }
 
 // draw shows the phase's progress: on a TTY in place of the line
-// before, and otherwise as a plain line.
+// before, and otherwise as a plain line. It first adds the count to
+// drawn, then drops from drawn the draws older than rateWindow, except
+// the one before this draw.
 func (p *progress) draw() {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
+	now := p.now()
+
+	p.drawn = append(p.drawn, drawnCount{at: now, count: p.count})
+	for len(p.drawn) > 2 && now.Sub(p.drawn[0].at) > rateWindow {
+		p.drawn = p.drawn[1:]
+	}
+
 	if p.width == nil {
-		text, _ := p.line(math.MaxInt, true)
+		text, _ := p.line(now, math.MaxInt, true)
 		fmt.Fprintln(os.Stderr, text)
 
 		return
@@ -192,13 +232,13 @@ func (p *progress) draw() {
 	// total of 0 leaves the bar nothing to fill.
 	text, withBar := "", false
 	if p.total > 0 && !p.estimate {
-		text, withBar = p.line(width-barMargin, false)
+		text, withBar = p.line(now, width-barMargin, false)
 	}
 
 	margin := barMargin
 	if !withBar {
 		margin = spinnerMargin
-		text, _ = p.line(width-margin, true)
+		text, _ = p.line(now, width-margin, true)
 	}
 
 	// The library makes a bar or a spinner, and before each draw erases
@@ -255,18 +295,22 @@ func (p *progress) newBar(withBar bool, width int) {
 }
 
 // setTotal gives a phase that started without a total its total, once
-// it is known. The rate, and so the ETA, count from then on: the time
-// spent waiting for the total says nothing about how fast items go.
+// it is known. The rate, the ETA and the time with no item completed
+// count from then on: the time spent waiting for the total says nothing
+// about how fast items go.
 func (p *progress) setTotal(total int64) {
 	if p == nil {
 		return
 	}
 
+	now := p.now()
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
 	p.total = total
-	p.rateStart = time.Now()
+	p.rateStart, p.lastItem = now, now
+	p.drawn = []drawnCount{{at: now, count: p.count}}
 }
 
 // increment records one completed item, which the display shows at its
@@ -276,10 +320,13 @@ func (p *progress) increment() {
 		return
 	}
 
+	now := p.now()
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
 	p.count++
+	p.lastItem = now
 }
 
 // warnf prints a one-line warning to stderr without corrupting the
@@ -391,52 +438,77 @@ func (p *progress) finish() {
 	}
 }
 
-// line formats the phase's progress in at most room columns, as
+// line formats the phase's progress as of now in at most room columns,
+// as
 //
 //	[3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
 //
 // where "files" is the phase's unit, without the percent when percent
 // is false, as before the library's bar, which shows its own. A phase
-// with no total shows the items done instead, or what it waits for.
+// with no total shows the items done instead, or what it waits for. The
+// rate is that since the first draw in drawn, and the ETA comes from
+// the rate since the count started. Once no item has been completed for
+// stallTime, the line says so before the elapsed time, and what the
+// phase is doing, as "no files finished for 42s, reading files".
 //
 // The phase and its place, the progress, the elapsed time and the ETA
 // are kept whole. A line wider than room leaves out the rate, or what
-// the phase waits for; only if it is still wider is it cut at room, and
-// line reports that it does not fit. Every part is ASCII, so its length
-// is its width in columns. p.mu must be held.
-func (p *progress) line(room int, percent bool) (string, bool) {
-	elapsed := time.Since(p.start).Round(time.Second)
+// the phase waits for, then what it says of a stall; only if it is
+// still wider is it cut at room, and line reports that it does not fit.
+// Every part is ASCII, so its length is its width in columns. p.mu must
+// be held.
+func (p *progress) line(now time.Time, room int,
+	percent bool,
+) (string, bool) {
+	elapsed := now.Sub(p.start).Round(time.Second)
 
-	// head and tail hold the parts kept whole; extra, between them, is
-	// the part left out.
-	var head, extra, tail string
+	// head and tail hold the parts kept whole. extra, after head, is the
+	// part left out first, and stall, before tail, the part left out
+	// next.
+	var head, extra, stall, tail string
 
 	if p.total < 0 && p.waitingFor != "" {
-		head, extra = p.label+": ", p.waitingFor
+		head, extra = p.label+": ", p.waitingFor+", "
 		tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
 	} else {
-		rate := 0.0
-		if s := time.Since(p.rateStart).Seconds(); s > 0 {
-			rate = float64(p.count) / s
-		}
+		since := p.drawn[0]
+		rate := perSecond(p.count-since.count, now.Sub(since.at))
+		average := perSecond(p.count, now.Sub(p.rateStart))
 
 		head = p.label + ": " + p.itemsDone(percent) + ", "
-		extra = fmt.Sprintf("%.0f %s/s", rate, p.unit)
-		tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(rate))
+		extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
+
+		if idle := now.Sub(p.lastItem); idle >= stallTime {
+			stall = fmt.Sprintf("no %s finished for %s, %s, ",
+				p.unit, idle.Round(time.Second), p.doing)
+		}
+
+		tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(average))
 	}
 
-	full, kept := head+extra+", "+tail, head+tail
-
-	switch {
-	case len(full) <= room:
-		return full, true
-	case len(kept) <= room:
-		return kept, true
+	for _, text := range []string{
+		head + extra + stall + tail, head + stall + tail, head + tail,
+	} {
+		if len(text) <= room {
+			return text, true
+		}
 	}
 
+	kept := head + tail
+
 	return kept[:max(room, 0)], false
 }
 
+// perSecond returns count items over d as items a second, or 0 when no
+// time has passed.
+func perSecond(count int64, d time.Duration) float64 {
+	if d <= 0 {
+		return 0
+	}
+
+	return float64(count) / d.Seconds()
+}
+
 // itemsDone formats the items done, as "[1234/5000] 25%", with "~"
 // before a total that is an estimate and without the percent when
 // percent is false, or as "1234 files", in the phase's unit, when the