scan: show each phase as N of M, with progress, elapsed time and ETA, also while it waits (closes #112)

Every scan display names its phase and its place among the five, as
"[2/5] walk", with its progress, elapsed time and ETA. load counts the
records first; the walk estimates its total from the records under the
operands; the content phase adds up the sizes of the matching groups
and says it is searching until the first record returns. Each display
redraws every 100 ms on a terminal and every 5 seconds in plain mode.

On a terminal no line reaches the last column: a line too wide drops
its rate, then its bar. After the terminal is made narrower than the
widest line drawn, the display makes a new library object, whose erase
fits.

Judgement call: load and the content checks draw the bar too, once their totals are known.
Judgement call: the bar is 10 columns wide, not the width the line leaves.

Model: opus-5-5

scan.go

@@ -84,18 +84,26 @@ type fileMeta struct {
 	hashed bool
 }
 
-// runScan implements the scan subcommand: four sequential phases —
-// walk (which stats each file as it is discovered), hash, update,
-// content — that synchronize the persistent database with the
-// filesystem state under the PATH operands. Only files whose size at
-// least one other file shares are ever hashed: a size-unique file
-// cannot be a duplicate. A file of headTailMin or more gets its content
-// hash only when its size, head, and tail match another file's. Flag
-// parsing and the at-least-one-operand check are done by cobra. Before
-// anything else, the scan names the database on stderr, saying whether
-// it is new. The scan holds the lock on the database for its whole
-// run, so a second scan fails before it walks the filesystem or opens
-// the database.
+// scanPhase names one phase of a scan with its place among all of them,
+// as "[2/5] walk", for the phase's progress display. The list here is
+// the one place the phases are counted.
+func scanPhase(name string) string {
+	return phaseLabel(
+		[]string{"load", "walk", "hash", "update", "content"}, name)
+}
+
+// runScan implements the scan subcommand: five sequential phases —
+// load (which reads the existing records), walk (which stats each file
+// as it is discovered), hash, update, content — that synchronize the
+// persistent database with the filesystem state under the PATH
+// operands. Only files whose size at least one other file shares are
+// ever hashed: a size-unique file cannot be a duplicate. A file of
+// headTailMin or more gets its content hash only when its size, head,
+// and tail match another file's. Flag parsing and the
+// at-least-one-operand check are done by cobra. Before anything else,
+// the scan names the database on stderr, saying whether it is new. The
+// scan holds the lock on the database for its whole run, so a second
+// scan fails before it walks the filesystem or opens the database.
 // Errors are returned rather than exiting, so that the deferred close —
 // which takes the database out of WAL mode — always runs, and the lock
 // is released after it. When ctx is cancelled, as by the SIGINT or
@@ -344,18 +352,18 @@ func syncScan(ctx context.Context, db *sql.DB, roots []string,
 }
 
 // runPhases synchronizes the database with the filesystem under roots
-// in four sequential phases: walk (enumerate and stat every file,
-// building a complete size census), hash (read only the new or
-// changed — or previously unhashed — files whose size at least one
-// other file shares, committing results in batches as they arrive),
-// update (record the size-unique files without reading them, and
-// delete the records the scan no longer verifies), and content (fill
-// in the content hash of every record of headTailMin or more whose
-// size, head, and tail match another record's). Records outside the
-// roots are never touched, except that the content phase fills in
-// their content hash. Operands the walk cannot start from are dropped
-// first, so the records beneath them are treated as if those operands
-// had not been given.
+// in five sequential phases: load (read the existing records), walk
+// (enumerate and stat every file, building a complete size census),
+// hash (read only the new or changed — or previously unhashed — files
+// whose size at least one other file shares, committing results in
+// batches as they arrive), update (record the size-unique files without
+// reading them, and delete the records the scan no longer verifies),
+// and content (fill in the content hash of every record of headTailMin
+// or more whose size, head, and tail match another record's). Records
+// outside the roots are never touched, except that the content phase
+// fills in their content hash. Operands the walk cannot start from are
+// dropped first, so the records beneath them are treated as if those
+// operands had not been given.
 func (s *scanState) runPhases(ctx context.Context, roots []string,
 	workers int, oneFS bool,
 ) error {
@@ -430,10 +438,21 @@ func (s *scanState) walkableRoots(roots []string) []string {
 // separately into the same database.
 func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 	// Indexing tens of millions of records takes real time; without a
-	// display the scan looks hung before the walk begins.
-	prog := newProgress("load", -1)
+	// display the scan looks hung before the walk begins. Counting them
+	// first gives the display its total.
+	prog := newProgress(progressParams{
+		label: scanPhase("load"), total: -1,
+		waitingFor: "counting the records",
+	})
 	defer prog.finish()
 
+	total, err := countRecords(ctx, s.db)
+	if err != nil {
+		return err
+	}
+
+	prog.setTotal(total)
+
 	s.existing = make(map[string]fileMeta)
 
 	return loadFileMeta(ctx, s.db,
@@ -465,7 +484,16 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 func (s *scanState) walkPhase(
 	events <-chan walkEvent, roots []string,
 ) ([]fileRec, []fileRec) {
-	prog := newProgress("walk", -1)
+	// The records the last scan left under the roots estimate how many
+	// files the walk will find. A first scan has none, and no estimate.
+	estimate := int64(len(s.existing))
+	if estimate == 0 {
+		estimate = -1
+	}
+
+	prog := newProgress(progressParams{
+		label: scanPhase("walk"), total: estimate, estimate: true,
+	})
 
 	var (
 		changed, unhashed []fileRec
@@ -636,10 +664,11 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
 }
 
 // readRuns reads runs with the worker pool, one read per inode run, in
-// the order given, under a progress display named label. The workers
-// compute each run's hashes with hash, and each result goes to record;
-// a run that fails to read is warned about and counted as skipped
-// instead. The total counts actual reads, so the bar shows a real ETA.
+// the order given, under the progress display of the phase named label.
+// The workers compute each run's hashes with hash, and each result goes
+// to record; a run that fails to read is warned about and counted as
+// skipped instead. The total counts actual reads, so the display shows
+// a real ETA.
 //
 // Returning early — a failed database write, or a cancelled scan — must
 // not strand the pool: the feeder would park forever on a full jobs
@@ -653,7 +682,9 @@ func (s *scanState) readRuns(ctx context.Context, workers int,
 	pool := startHashPool(ctx, runs, workers, hash)
 	defer pool.stop()
 
-	prog := newProgress(label, int64(len(runs)))
+	prog := newProgress(progressParams{
+		label: scanPhase(label), total: int64(len(runs)),
+	})
 	defer prog.finish()
 
 	for range runs {
@@ -740,7 +771,9 @@ func (s *scanState) updatePhase(ctx context.Context) error {
 	s.st.removed = len(deletes)
 
 	total := len(s.batch) + len(s.sentinels) + len(deletes)
-	prog := newProgress("update", int64(total))
+	prog := newProgress(progressParams{
+		label: scanPhase("update"), total: int64(total),
+	})
 
 	defer prog.finish()
 
@@ -821,16 +854,28 @@ func (s *scanState) contentPhase(ctx context.Context, workers int) error {
 func (s *scanState) contentCandidates(
 	ctx context.Context,
 ) ([]fileRec, map[string]scanRec, error) {
-	// The query and the checks take real time on a large database;
+	// The search and the checks take real time on a large database;
 	// without a display the scan looks hung before the reads begin.
-	prog := newProgress("content", -1)
+	// The display says it is searching until SQLite returns the first
+	// record. The checks start there, against a count of the records
+	// the search returns, and so do their rate and ETA.
+	prog := newProgress(progressParams{
+		label: scanPhase("content"), total: -1,
+		waitingFor: "searching the database",
+	})
 	defer prog.finish()
 
+	total, err := countContentCandidates(ctx, s.db)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	var (
-		toRead []fileRec
-		first  scanRec   // the current group's first record
-		passed int       // the current group's files that passed the check
-		unread []fileRec // those of them without a content hash
+		searching = true // no record has been returned yet
+		toRead    []fileRec
+		first     scanRec   // the current group's first record
+		passed    int       // the current group's files that passed the check
+		unread    []fileRec // those of them without a content hash
 	)
 
 	recs := make(map[string]scanRec)
@@ -849,7 +894,13 @@ func (s *scanState) contentCandidates(
 		passed, unread = 0, nil
 	}
 
-	err := loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) {
+	err = loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) {
+		if searching {
+			searching = false
+
+			prog.setTotal(total)
+		}
+
 		prog.increment()
 
 		if r.size != first.size || r.head != first.head || r.tail != first.tail {
@@ -880,6 +931,11 @@ func (s *scanState) contentCandidates(
 		return nil, nil, err
 	}
 
+	// A search that returned no record still ends showing its count.
+	if searching {
+		prog.setTotal(total)
+	}
+
 	endGroup()
 
 	return toRead, recs, nil