scan: show each phase as N of M, with progress, elapsed time and ETA, also while it waits (closes #112)

Every scan display now starts with its phase and its place among the
five, as "[2/5] walk", and always shows the rate, elapsed time and
ETA. load counts the records before reading them, and the content
phase counts what its search finds before checking it, so both have
totals; until then the line says what the phase waits for. The walk's
total is estimated from the records under the operands, with the ETA
unknown past the estimate or without one. Each display draws from its
own goroutine under one mutex: several times a second on a terminal,
every 5 seconds in plain mode. scanPhase holds the phase list.

Judgement call: on a terminal the line is the plain line behind the library's spinner; the bar graphic is gone.
Judgement call: the content phase runs its search query twice, once to count its rows.

Model: opus-5-5

progress.go

@@ -3,22 +3,24 @@ package main
 import (
 	"fmt"
 	"os"
+	"slices"
+	"sync"
 	"time"
 
 	"github.com/schollz/progressbar/v3"
 	"golang.org/x/term"
 )
 
-// plainInterval is the minimum time between progress lines when stderr
-// is not a TTY.
+// plainInterval is the time between progress lines while a phase runs,
+// when stderr is not a TTY.
 const plainInterval = 5 * time.Second
 
-// barThrottle caps how often the TTY progress bar redraws.
-const barThrottle = 100 * time.Millisecond
+// redrawInterval is the time between redraws of the progress display on
+// a TTY.
+const redrawInterval = 100 * time.Millisecond
 
-// walkSpinnerType selects the progressbar library's spinner style used
-// for the walk pass, which has no known total.
-const walkSpinnerType = 14
+// spinnerType selects the progressbar library's spinner style.
+const spinnerType = 14
 
 // percentScale converts a fraction to a percentage.
 const percentScale = 100
@@ -28,96 +30,153 @@ func stderrIsTTY() bool {
 	return term.IsTerminal(int(os.Stderr.Fd()))
 }
 
-// progress renders one scan pass's progress on stderr. On a TTY it
-// delegates to the progressbar library (spinner style when the total is
-// unknown, full bar with count/percent/rate/elapsed/ETA otherwise). When
-// stderr is not a TTY it emits no ANSI redraws: it prints a plain
-// one-line update as the pass starts, then no more often than every
-// plainInterval.
-//
-// All methods must be called from the main goroutine only. On a TTY
-// the library also redraws a spinner from its own goroutine, several
-// times a second, so its count and elapsed time stay current while a
-// pass waits for its next item. A nil *progress is a valid
-// no-display receiver: every method is a no-op, so batched database
-// flushes during the streaming pass can reuse the update-pass helpers
-// without rendering anything.
-type progress struct {
-	label string
-	total int64 // -1 when unknown (walk pass)
-	bar   *progressbar.ProgressBar
-	count int64
-	start time.Time
-	last  time.Time
+// phaseLabel names phase with its place among phases, as "[2/5] walk".
+func phaseLabel(phases []string, phase string) string {
+	return fmt.Sprintf("[%d/%d] %s",
+		slices.Index(phases, phase)+1, len(phases), phase)
 }
 
-func newProgress(label string, total int64) *progress {
-	p := &progress{label: label, total: total, start: time.Now()}
-	if stderrIsTTY() {
-		p.bar = newBar(label, total)
+// progressParams describe one phase's progress display as it starts.
+type progressParams struct {
+	label      string // the phase and its place, from phaseLabel
+	total      int64  // the items the phase will complete; -1 when not known
+	estimate   bool   // total is an estimate, which the count may pass
+	waitingFor string // what the phase waits for until setTotal is called
+}
 
-		return p
+// progress renders one phase's progress on stderr as one line (see
+// line): the phase and its place among the phases, the items done out
+// of the total, the rate, the elapsed time and the estimated time
+// remaining. On a TTY the progressbar library draws the line behind a
+// spinner, again every redrawInterval. When stderr is not a TTY it
+// emits no ANSI redraws: it prints the line as the phase starts, every
+// plainInterval while it runs, and as it ends. Either way the display
+// shows the phase is alive while it waits for its next item.
+//
+// The redraws come from a goroutine of their own, so mu guards what
+// they read. Every method must be called from the main goroutine. A
+// nil *progress is a valid no-display receiver: every method is a
+// no-op, so batched database flushes during the hash phase can reuse
+// the update phase's helpers without rendering anything.
+type progress struct {
+	bar  *progressbar.ProgressBar // nil when stderr is not a TTY
+	stop chan struct{}            // closed by finish to end the redraws
+	done chan struct{}            // closed once the redraws have ended
+
+	mu         sync.Mutex
+	label      string
+	total      int64
+	estimate   bool
+	waitingFor string
+	count      int64
+	start      time.Time // when the phase started, for the elapsed time
+	rateStart  time.Time // when the count started, for the rate
+}
+
+// newProgress starts the display of one phase on stderr.
+func newProgress(params progressParams) *progress {
+	if stderrIsTTY() {
+		return startProgress(params, newSpinner(), redrawInterval)
 	}
 
-	// Print the zero state at once: the first item may take minutes,
-	// and a pass must never look hung.
-	p.last = p.start
-	fmt.Fprintln(os.Stderr, p.plainLine())
+	return startProgress(params, nil, plainInterval)
+}
+
+// startProgress shows the phase params describes at once, on bar, or as
+// a plain line when bar is nil, and again every interval until finish.
+func startProgress(params progressParams, bar *progressbar.ProgressBar,
+	interval time.Duration,
+) *progress {
+	now := time.Now()
+	p := &progress{
+		bar: bar, stop: make(chan struct{}), done: make(chan struct{}),
+		label: params.label, total: params.total, estimate: params.estimate,
+		waitingFor: params.waitingFor, start: now, rateStart: now,
+	}
+
+	// Show the zero state at once: the first item may take minutes,
+	// and a phase must never look hung.
+	p.draw()
+
+	go func() {
+		defer close(p.done)
+
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-p.stop:
+				return
+			case <-ticker.C:
+				p.draw()
+			}
+		}
+	}()
 
 	return p
 }
 
-// newBar builds the TTY display for newProgress.
-func newBar(label string, total int64) *progressbar.ProgressBar {
-	opts := []progressbar.Option{
+// newSpinner builds the TTY display for newProgress: the library's
+// spinner, followed by the text draw gives it.
+func newSpinner() *progressbar.ProgressBar {
+	return progressbar.NewOptions64(-1,
 		progressbar.OptionSetWriter(os.Stderr),
-		progressbar.OptionSetDescription(label),
-		progressbar.OptionShowCount(),
-		progressbar.OptionShowIts(),
-		progressbar.OptionSetItsString("files"),
-		progressbar.OptionSetElapsedTime(true),
-		progressbar.OptionThrottle(barThrottle),
-		// Render at zero immediately: a phase must be visible the
-		// moment it starts, even before its first item completes.
-		progressbar.OptionSetRenderBlankState(true),
-	}
-	if total >= 0 {
-		opts = append(opts,
-			progressbar.OptionSetPredictTime(true),
-			progressbar.OptionShowElapsedTimeOnFinish(),
-		)
-	} else {
-		opts = append(opts,
-			progressbar.OptionSetPredictTime(false),
-			progressbar.OptionSpinnerType(walkSpinnerType),
-		)
-	}
-
-	return progressbar.NewOptions64(total, opts...)
+		progressbar.OptionSpinnerType(spinnerType),
+		// The spinner turns at each draw, not on a timer of the
+		// library's own.
+		progressbar.OptionSetSpinnerChangeInterval(0),
+		// The text carries the elapsed time.
+		progressbar.OptionSetElapsedTime(false),
+	)
 }
 
-// increment records one completed item and refreshes the display.
+// draw shows the phase's line: on a TTY in place of the one before, and
+// otherwise as a plain line.
+func (p *progress) draw() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.bar == nil {
+		fmt.Fprintln(os.Stderr, p.line())
+
+		return
+	}
+
+	p.bar.Describe(p.line())
+}
+
+// setTotal gives a phase that started without a total its total, once
+// it is known. The rate, and so the ETA, count from then on: the time
+// spent waiting for the total says nothing about how fast items go.
+func (p *progress) setTotal(total int64) {
+	if p == nil {
+		return
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.total = total
+	p.rateStart = time.Now()
+}
+
+// increment records one completed item, which the display shows at its
+// next draw.
 func (p *progress) increment() {
 	if p == nil {
 		return
 	}
 
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
 	p.count++
-	if p.bar != nil {
-		_ = p.bar.Add(1)
-
-		return
-	}
-
-	if time.Since(p.last) >= plainInterval {
-		p.last = time.Now()
-		fmt.Fprintln(os.Stderr, p.plainLine())
-	}
 }
 
-// warnf prints a one-line warning to stderr without corrupting the bar.
-// The whole message is escaped like a report's path columns, so a path
-// holding a newline cannot split the warning.
+// warnf prints a one-line warning to stderr without corrupting the
+// display. The whole message is escaped like a report's path columns,
+// so a path holding a newline cannot split the warning.
 func (p *progress) warnf(format string, args ...any) {
 	if p == nil {
 		return
@@ -125,78 +184,98 @@ func (p *progress) warnf(format string, args ...any) {
 
 	msg := escapePath(fmt.Sprintf(format, args...))
 
-	if p.bar != nil && p.total < 0 {
-		// The library also redraws a spinner from its own goroutine, so
-		// a direct write could land inside a redraw. The bar prints the
-		// warning itself, just before its next redraw.
-		_, _ = progressbar.Bprintln(p.bar, msg)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.bar == nil {
+		fmt.Fprintln(os.Stderr, msg)
 
 		return
 	}
 
-	if p.bar != nil {
-		_ = p.bar.Clear()
-	}
-
-	fmt.Fprintln(os.Stderr, msg)
-
-	// The library never draws a finished bar again, so the warning for
-	// a pass's last item would erase it for good: draw its last frame
-	// again below the warning.
-	if p.bar != nil && p.bar.IsFinished() {
-		fmt.Fprint(os.Stderr, p.bar.String())
-	}
+	// A direct write could land inside the line on the TTY. The bar
+	// prints the warning on a line of its own, just before its next
+	// draw.
+	_, _ = progressbar.Bprintln(p.bar, msg)
 }
 
-// finish terminates the pass's display. A bar whose pass stopped short
-// of its total, as an interrupted one does, is left as last drawn; the
-// library's Finish would fill it up.
+// finish ends the phase's display with its line as the phase ends. A
+// phase stopped short of its total, as an interrupted one is, shows the
+// count it stopped at.
 func (p *progress) finish() {
 	if p == nil {
 		return
 	}
 
+	close(p.stop)
+	<-p.done
+
+	p.draw()
+
 	if p.bar != nil {
-		if p.total >= 0 && p.count < p.total {
-			_ = p.bar.Exit()
-		} else {
-			_ = p.bar.Finish()
-		}
+		// Finish draws the line once more, below any warning it holds,
+		// and leaves it on the TTY.
+		_ = p.bar.Finish()
 
 		fmt.Fprintln(os.Stderr)
-
-		return
 	}
-
-	fmt.Fprintln(os.Stderr, p.plainLine())
 }
 
-// plainLine formats a non-TTY progress line.
-func (p *progress) plainLine() string {
-	elapsed := time.Since(p.start)
-	if p.total < 0 {
-		return fmt.Sprintf("%s: %d files, elapsed %s",
-			p.label, p.count, elapsed.Round(time.Second))
-	}
+// line formats the phase's progress, as
+//
+//	[3/5] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
+//
+// with "~" before a total that is an estimate. A phase with no total
+// shows the items done instead, or what it waits for. p.mu must be
+// held.
+func (p *progress) line() string {
+	elapsed := time.Since(p.start).Round(time.Second)
 
-	pct := float64(percentScale)
-	if p.total > 0 {
-		pct = percentScale * float64(p.count) / float64(p.total)
+	if p.total < 0 && p.waitingFor != "" {
+		return fmt.Sprintf("%s: %s, elapsed %s, eta unknown",
+			p.label, p.waitingFor, elapsed)
 	}
 
 	rate := 0.0
-	if s := elapsed.Seconds(); s > 0 {
+	if s := time.Since(p.rateStart).Seconds(); s > 0 {
 		rate = float64(p.count) / s
 	}
 
-	eta := "?"
+	done := fmt.Sprintf("%d files", p.count)
 
-	if rate > 0 && p.count <= p.total {
-		remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
-		eta = remaining.Round(time.Second).String()
+	if p.total >= 0 {
+		approx := ""
+		if p.estimate {
+			approx = "~"
+		}
+
+		pct := float64(percentScale)
+		if p.total > 0 {
+			pct = percentScale * float64(p.count) / float64(p.total)
+		}
+
+		done = fmt.Sprintf("[%d/%s%d] %.0f%%", p.count, approx, p.total, pct)
 	}
 
-	return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
-		p.label, p.count, p.total, pct, rate,
-		elapsed.Round(time.Second), eta)
+	return fmt.Sprintf("%s: %s, %.0f files/s, elapsed %s, eta %s",
+		p.label, done, rate, elapsed, p.eta(rate))
+}
+
+// eta returns the time the phase has left at rate items a second, or
+// "unknown": with no total, before the first item, and once the count
+// passes an estimate. p.mu must be held.
+func (p *progress) eta(rate float64) string {
+	switch {
+	case p.total < 0, p.estimate && p.count > p.total:
+		return "unknown"
+	case p.count >= p.total:
+		return "0s"
+	case rate <= 0:
+		return "unknown"
+	}
+
+	remaining := time.Duration(
+		float64(p.total-p.count) / rate * float64(time.Second))
+
+	return remaining.Round(time.Second).String()
 }