scan: show each phase as N of M, with progress, elapsed time and ETA, also while it waits (closes #112)

Every scan display now names its phase and its place among the five,
as "[2/5] walk", with its progress, rate, elapsed time and ETA. load
counts the records first; the walk estimates its total from the
records under the operands; the content phase adds up the sizes of the
matching groups and says it is searching until the first record
returns. Each display is drawn from its own goroutine: several times a
second on a terminal, every 5 seconds in plain mode.

On a terminal a phase with an exact total draws its line before the
library's bar when there is room, and any other a spinner before the
line. A line too wide drops its rate, then its bar; it is cut only when
the phase, progress, elapsed time and ETA alone do not fit.

Judgement call: load and the content checks draw the bar too, once their totals are known.
Judgement call: the bar is 10 columns wide, not the width the line leaves.

Model: opus-5-5

progress.go

@@ -2,122 +2,258 @@ package main
 
 import (
 	"fmt"
+	"math"
 	"os"
+	"slices"
+	"sync"
 	"time"
 
 	"github.com/schollz/progressbar/v3"
 	"golang.org/x/term"
 )
 
-// plainInterval is the minimum time between progress lines when stderr
-// is not a TTY.
+// plainInterval is the time between progress lines while a phase runs,
+// when stderr is not a TTY.
 const plainInterval = 5 * time.Second
 
-// barThrottle caps how often the TTY progress bar redraws.
-const barThrottle = 100 * time.Millisecond
+// redrawInterval is the time between redraws of the progress display on
+// a TTY.
+const redrawInterval = 100 * time.Millisecond
 
-// walkSpinnerType selects the progressbar library's spinner style used
-// for the walk pass, which has no known total.
-const walkSpinnerType = 14
+// spinnerType selects the progressbar library's spinner style.
+const spinnerType = 14
+
+// barWidth is the width of the progressbar library's bar between its
+// two ends, as in the example line in README "Progress".
+const barWidth = 10
 
 // percentScale converts a fraction to a percentage.
 const percentScale = 100
 
+// fallbackWidth is the width taken for a terminal whose width cannot be
+// read, as the progressbar library takes it.
+const fallbackWidth = 80
+
+// spinnerMargin is the columns of a spinner's line besides its text:
+// the spinner and the three spaces the library draws around the text,
+// and the terminal's last column, which every line stays clear of so
+// that no terminal wraps it.
+const spinnerMargin = 5
+
+// barMargin is the columns of a bar's line besides its text: the
+// percent and the bar the library draws after the text, the spaces
+// around them, and the terminal's last column.
+const barMargin = barWidth + 11
+
 // stderrIsTTY reports whether stderr is attached to a terminal.
 func stderrIsTTY() bool {
 	return term.IsTerminal(int(os.Stderr.Fd()))
 }
 
-// progress renders one scan pass's progress on stderr. On a TTY it
-// delegates to the progressbar library (spinner style when the total is
-// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
-// stderr is not a TTY it emits no ANSI redraws: it prints a plain
-// one-line update as the pass starts, then no more often than every
-// plainInterval.
-//
-// All methods must be called from the main goroutine only. On a TTY
-// the library also redraws a spinner from its own goroutine, several
-// times a second, so its count and elapsed time stay current while a
-// pass waits for its next item. A nil *progress is a valid
-// no-display receiver: every method is a no-op, so batched database
-// flushes during the streaming pass can reuse the update-pass helpers
-// without rendering anything.
-type progress struct {
-	label string
-	total int64 // -1 when unknown (walk pass)
-	bar   *progressbar.ProgressBar
-	count int64
-	start time.Time
-	last  time.Time
-}
-
-func newProgress(label string, total int64) *progress {
-	p := &progress{label: label, total: total, start: time.Now()}
-	if stderrIsTTY() {
-		p.bar = newBar(label, total)
-
-		return p
+// terminalWidth returns the width of the terminal on stderr, or
+// fallbackWidth, as the progressbar library does, when it cannot read
+// one.
+func terminalWidth() int {
+	width, _, err := term.GetSize(int(os.Stderr.Fd()))
+	if err != nil || width <= 0 {
+		return fallbackWidth
 	}
 
-	// Print the zero state at once: the first item may take minutes,
-	// and a pass must never look hung.
-	p.last = p.start
-	fmt.Fprintln(os.Stderr, p.plainLine())
+	return width
+}
+
+// phaseLabel names phase with its place among phases, as "[2/5] walk".
+func phaseLabel(phases []string, phase string) string {
+	return fmt.Sprintf("[%d/%d] %s",
+		slices.Index(phases, phase)+1, len(phases), phase)
+}
+
+// progressParams describe one phase's progress display as it starts.
+type progressParams struct {
+	label      string // the phase and its place, from phaseLabel
+	total      int64  // the items the phase will complete; -1 when not known
+	estimate   bool   // total is an estimate, which the count may pass
+	waitingFor string // what the phase waits for until setTotal is called
+}
+
+// progress renders one phase's progress on stderr. On a TTY the
+// progressbar library draws line, fitted to the terminal's width: after
+// the library's spinner, or, for a phase with an exact total, before the
+// library's bar when the terminal has room for the bar beside it. It is
+// drawn again every redrawInterval, so its elapsed time keeps moving
+// while the phase waits for its next item. When stderr is not a TTY it
+// emits no ANSI redraws: it prints line as the phase starts, every
+// plainInterval while it runs, and as it ends.
+//
+// The draws come from a goroutine of their own, so mu guards the fields
+// below it. Every method must be called from the main goroutine. A nil
+// *progress is a valid no-display receiver: every method is a no-op, so
+// batched database flushes during the hash phase can reuse the update
+// phase's helpers without rendering anything.
+type progress struct {
+	width func() int    // reads the terminal's width; nil when not a TTY
+	stop  chan struct{} // closed by finish to end the draws
+	done  chan struct{} // closed once the draws have ended
+
+	mu         sync.Mutex
+	bar        *progressbar.ProgressBar // what the library draws; nil until then
+	withBar    bool                     // bar is the library's bar, not its spinner
+	label      string
+	total      int64
+	estimate   bool
+	waitingFor string
+	count      int64
+	start      time.Time // when the phase started, for the elapsed time
+	rateStart  time.Time // when the count started, for the rate
+}
+
+// newProgress starts the display of one phase on stderr.
+func newProgress(params progressParams) *progress {
+	if stderrIsTTY() {
+		return startProgress(params, terminalWidth, redrawInterval)
+	}
+
+	return startProgress(params, nil, plainInterval)
+}
+
+// startProgress shows the phase params describes at once, and again
+// every interval until finish: with the progressbar library on a
+// terminal, whose width each draw reads by calling width, or as a plain
+// line when width is nil.
+func startProgress(params progressParams, width func() int,
+	interval time.Duration,
+) *progress {
+	now := time.Now()
+	p := &progress{
+		width: width, stop: make(chan struct{}), done: make(chan struct{}),
+		label: params.label, total: params.total, estimate: params.estimate,
+		waitingFor: params.waitingFor, start: now, rateStart: now,
+	}
+
+	// Show the zero state at once: the first item may take minutes,
+	// and a phase must never look hung.
+	p.draw()
+
+	go func() {
+		defer close(p.done)
+
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-p.stop:
+				return
+			case <-ticker.C:
+				p.draw()
+			}
+		}
+	}()
 
 	return p
 }
 
-// newBar builds the TTY display for newProgress.
-func newBar(label string, total int64) *progressbar.ProgressBar {
-	opts := []progressbar.Option{
-		progressbar.OptionSetWriter(os.Stderr),
-		progressbar.OptionSetDescription(label),
-		progressbar.OptionShowCount(),
-		progressbar.OptionShowIts(),
-		progressbar.OptionSetItsString("files"),
-		progressbar.OptionSetElapsedTime(true),
-		progressbar.OptionThrottle(barThrottle),
-		// Render at zero immediately: a phase must be visible the
-		// moment it starts, even before its first item completes.
-		progressbar.OptionSetRenderBlankState(true),
-	}
-	if total >= 0 {
-		opts = append(opts,
-			progressbar.OptionSetPredictTime(true),
-			progressbar.OptionShowElapsedTimeOnFinish(),
-		)
-	} else {
-		opts = append(opts,
-			progressbar.OptionSetPredictTime(false),
-			progressbar.OptionSpinnerType(walkSpinnerType),
-		)
+// draw shows the phase's progress: on a TTY in place of the line
+// before, and otherwise as a plain line.
+func (p *progress) draw() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.width == nil {
+		text, _ := p.line(math.MaxInt, true)
+		fmt.Fprintln(os.Stderr, text)
+
+		return
 	}
 
-	return progressbar.NewOptions64(total, opts...)
+	width := p.width()
+
+	// A phase with an exact total gets the library's bar when its line,
+	// without the percent the bar shows, fits whole beside the bar. A
+	// total of 0 leaves the bar nothing to fill.
+	text, withBar := "", false
+	if p.total > 0 && !p.estimate {
+		text, withBar = p.line(width-barMargin, false)
+	}
+
+	if !withBar {
+		text, _ = p.line(width-spinnerMargin, true)
+	}
+
+	// The library makes a bar or a spinner. A display that turns from
+	// one to the other, as its line grows or the terminal is resized,
+	// gets a new one.
+	if p.bar == nil || p.withBar != withBar {
+		p.newBar(withBar)
+	}
+
+	if withBar {
+		_ = p.bar.Set64(p.count)
+	}
+
+	p.bar.Describe(text)
 }
 
-// increment records one completed item and refreshes the display.
+// newBar has the library make what draw draws: a bar for the phase's
+// total when withBar is set, and a spinner when not. Either shows the
+// text draw gives it, and nothing of its own besides the spinner, or
+// the bar and its percent. p.mu must be held.
+func (p *progress) newBar(withBar bool) {
+	if p.bar != nil {
+		// Erase what was drawn before.
+		_ = p.bar.Clear()
+	}
+
+	total := int64(-1) // the library's spinner
+	if withBar {
+		total = p.total
+	}
+
+	p.bar = progressbar.NewOptions64(total,
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionSetWidth(barWidth),
+		progressbar.OptionSpinnerType(spinnerType),
+		// The spinner turns at each draw, not on a timer of the
+		// library's own.
+		progressbar.OptionSetSpinnerChangeInterval(0),
+		// The text carries the elapsed time and the ETA.
+		progressbar.OptionSetElapsedTime(false),
+		progressbar.OptionSetPredictTime(false),
+	)
+	p.withBar = withBar
+}
+
+// setTotal gives a phase that started without a total its total, once
+// it is known. The rate, and so the ETA, count from then on: the time
+// spent waiting for the total says nothing about how fast items go.
+func (p *progress) setTotal(total int64) {
+	if p == nil {
+		return
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.total = total
+	p.rateStart = time.Now()
+}
+
+// increment records one completed item, which the display shows at its
+// next draw.
 func (p *progress) increment() {
 	if p == nil {
 		return
 	}
 
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
 	p.count++
-	if p.bar != nil {
-		_ = p.bar.Add(1)
-
-		return
-	}
-
-	if time.Since(p.last) >= plainInterval {
-		p.last = time.Now()
-		fmt.Fprintln(os.Stderr, p.plainLine())
-	}
 }
 
-// warnf prints a one-line warning to stderr without corrupting the bar.
-// The whole message is escaped like a report's path columns, so a path
-// holding a newline cannot split the warning.
+// warnf prints a one-line warning to stderr without corrupting the
+// display. The whole message is escaped like a report's path columns,
+// so a path holding a newline cannot split the warning.
 func (p *progress) warnf(format string, args ...any) {
 	if p == nil {
 		return
@@ -125,58 +261,101 @@ func (p *progress) warnf(format string, args ...any) {
 
 	msg := escapePath(fmt.Sprintf(format, args...))
 
-	if p.bar != nil && p.total < 0 {
-		// The library also redraws a spinner from its own goroutine, so
-		// a direct write could land inside a redraw. The bar prints the
-		// warning itself, just before its next redraw.
-		_, _ = progressbar.Bprintln(p.bar, msg)
-
-		return
-	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
 
 	if p.bar != nil {
+		// The warning takes the display's line; the next draw shows
+		// the display again, on the line below.
 		_ = p.bar.Clear()
 	}
 
 	fmt.Fprintln(os.Stderr, msg)
-
-	// The library never draws a finished bar again, so the warning for
-	// a pass's last item would erase it for good: draw its last frame
-	// again below the warning.
-	if p.bar != nil && p.bar.IsFinished() {
-		fmt.Fprint(os.Stderr, p.bar.String())
-	}
 }
 
-// finish terminates the pass's display. A bar whose pass stopped short
-// of its total, as an interrupted one does, is left as last drawn; the
-// library's Finish would fill it up.
+// finish ends the phase's display, drawn once more as the phase ends.
+// A phase stopped short of its total, as an interrupted one is, shows
+// the count it stopped at rather than a full bar.
 func (p *progress) finish() {
 	if p == nil {
 		return
 	}
 
-	if p.bar != nil {
-		if p.total >= 0 && p.count < p.total {
-			_ = p.bar.Exit()
-		} else {
-			_ = p.bar.Finish()
-		}
+	close(p.stop)
+	<-p.done
 
+	p.draw()
+
+	if p.width != nil {
+		// The display stays on the TTY as drawn; what follows starts
+		// on a line of its own.
 		fmt.Fprintln(os.Stderr)
-
-		return
 	}
-
-	fmt.Fprintln(os.Stderr, p.plainLine())
 }
 
-// plainLine formats a non-TTY progress line.
-func (p *progress) plainLine() string {
-	elapsed := time.Since(p.start)
+// line formats the phase's progress in at most room columns, as
+//
+//	[3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
+//
+// without the percent when percent is false, as before the library's
+// bar, which shows its own. A phase with no total shows the items done
+// instead, or what it waits for.
+//
+// The phase and its place, the progress, the elapsed time and the ETA
+// are kept whole. A line wider than room leaves out the rate, or what
+// the phase waits for; only if it is still wider is it cut at room, and
+// line reports that it does not fit. Every part is ASCII, so its length
+// is its width in columns. p.mu must be held.
+func (p *progress) line(room int, percent bool) (string, bool) {
+	elapsed := time.Since(p.start).Round(time.Second)
+
+	// head and tail hold the parts kept whole; extra, between them, is
+	// the part left out.
+	var head, extra, tail string
+
+	if p.total < 0 && p.waitingFor != "" {
+		head, extra = p.label+": ", p.waitingFor
+		tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
+	} else {
+		rate := 0.0
+		if s := time.Since(p.rateStart).Seconds(); s > 0 {
+			rate = float64(p.count) / s
+		}
+
+		head = p.label + ": " + p.itemsDone(percent) + ", "
+		extra = fmt.Sprintf("%.0f files/s", rate)
+		tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(rate))
+	}
+
+	full, kept := head+extra+", "+tail, head+tail
+
+	switch {
+	case len(full) <= room:
+		return full, true
+	case len(kept) <= room:
+		return kept, true
+	}
+
+	return kept[:max(room, 0)], false
+}
+
+// itemsDone formats the items done, as "[1234/5000] 25%", with "~"
+// before a total that is an estimate and without the percent when
+// percent is false, or as "1234 files" when the phase has no total.
+// p.mu must be held.
+func (p *progress) itemsDone(percent bool) string {
 	if p.total < 0 {
-		return fmt.Sprintf("%s: %d files, elapsed %s",
-			p.label, p.count, elapsed.Round(time.Second))
+		return fmt.Sprintf("%d files", p.count)
+	}
+
+	approx := ""
+	if p.estimate {
+		approx = "~"
+	}
+
+	done := fmt.Sprintf("[%d/%s%d]", p.count, approx, p.total)
+	if !percent {
+		return done
 	}
 
 	pct := float64(percentScale)
@@ -184,19 +363,24 @@ func (p *progress) plainLine() string {
 		pct = percentScale * float64(p.count) / float64(p.total)
 	}
 
-	rate := 0.0
-	if s := elapsed.Seconds(); s > 0 {
-		rate = float64(p.count) / s
+	return fmt.Sprintf("%s %.0f%%", done, pct)
+}
+
+// eta returns the time the phase has left at rate items a second, or
+// "unknown": with no total, before the first item, and once the count
+// passes an estimate. p.mu must be held.
+func (p *progress) eta(rate float64) string {
+	switch {
+	case p.total < 0, p.estimate && p.count > p.total:
+		return "unknown"
+	case p.count >= p.total:
+		return "0s"
+	case rate <= 0:
+		return "unknown"
 	}
-
-	eta := "?"
 
-	if rate > 0 && p.count <= p.total {
-		remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
-		eta = remaining.Round(time.Second).String()
-	}
+	remaining := time.Duration(
+		float64(p.total-p.count) / rate * float64(time.Second))
 
-	return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
-		p.label, p.count, p.total, pct, rate,
-		elapsed.Round(time.Second), eta)
+	return remaining.Round(time.Second).String()
 }