trees: show its phases as N of M, with progress, elapsed time and ETA (closes #113)

trees now runs in four phases, open, read, group and write, each under
the progress display scan uses, until the summary line. open covers a
wait on a busy database. read counts the records and reads them in one
read-only transaction, so the two agree during a scan; group counts the
directories, and write the groups. The rows reach stdout through a
writer. When stdout and stderr are the same file, it clears the display
and holds its lock while writing, so no row shares a line with the
display on a shared terminal; otherwise it writes without the lock, so
a slow reader of stdout cannot stop the display. Each rate names what
its phase counts.

Judgement call: opening the database is a phase of its own, so there are four phases, not the three the issue sketches.

Model: opus-5-5

trees.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"bufio"
 	"context"
 	"crypto/sha256"
+	"database/sql"
 	"fmt"
 	"io"
 	"os"
@@ -24,6 +24,13 @@ type treeNode struct {
 	totalSize int64
 }
 
+// treesPhase names one phase of trees with its place among all of them,
+// as "[2/4] read", for the phase's progress display. The list here is
+// the one place the phases are counted.
+func treesPhase(name string) string {
+	return phaseLabel([]string{"open", "read", "group", "write"}, name)
+}
+
 // runTrees implements the trees subcommand: it reads every record from
 // the database in path order, reconstructs the directory hierarchy from
 // the record paths, computes a Merkle-style digest per directory, and
@@ -31,6 +38,13 @@ type treeNode struct {
 // touches the scanned filesystem; its only I/O is the database, stdout,
 // and stderr. It names the database on stderr before it opens it. Any
 // database problem, including a missing database, is fatal.
+//
+// The work runs in four phases, each under a progress display of its
+// own on stderr: open (open the database), read (count the records,
+// then read them into the hierarchy), group (group the directories by
+// digest) and write (write the groups to stdout). Each display ends
+// before the next phase starts, and before an error is returned to be
+// printed.
 func runTrees(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -40,34 +54,99 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 	fmt.Fprintf(os.Stderr, "trees: database %s (%s)\n",
 		escapePath(dbPath), source)
 
+	// Opening can wait on a busy database for as long as the busy
+	// timeout.
+	prog := newProgress(progressParams{
+		label: treesPhase("open"), total: -1,
+		waitingFor: "opening the database",
+	})
 	db, err := openReportDatabase(ctx, dbPath)
+
+	prog.finish()
+
 	if err != nil {
 		return err
 	}
 
 	defer func() { _ = db.Close() }()
 
-	records := 0
-	tree := newTreeBuilder()
-
-	err = loadFileRows(ctx, db, func(r scanRec) {
-		records++
-
-		tree.add(r)
+	prog = newProgress(progressParams{
+		label: treesPhase("read"), unit: "records", total: -1,
+		waitingFor: "counting the records",
 	})
+	records, super, allDirs, err := readTree(ctx, db, prog)
+
+	prog.finish()
+
 	if err != nil {
 		return fmt.Errorf("database %s: %w", dbPath, err)
 	}
 
+	prog = newProgress(progressParams{
+		label: treesPhase("group"), unit: "directories",
+		total: int64(len(allDirs)),
+	})
+	dupes := collectTreeGroups(allDirs, super, prog)
+
+	prog.finish()
+
+	prog = newProgress(progressParams{
+		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
+	})
+	dupeTrees, reclaimable, err := writeTreeGroups(stdout, dupes, prog)
+
+	prog.finish()
+
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(os.Stderr,
+		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
+			"%s reclaimable\n",
+		records, len(dupes), dupeTrees, humanBytes(reclaimable))
+
+	return nil
+}
+
+// readTree gives prog the number of records in db as its total, then
+// reads every record in path order into the directory hierarchy,
+// counting each on prog. It returns the number of records read, the
+// super-root and every directory.
+func readTree(ctx context.Context, db *sql.DB,
+	prog *progress,
+) (int, *treeNode, []*treeNode, error) {
+	records := 0
+	tree := newTreeBuilder()
+
+	err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
+		records++
+
+		prog.increment()
+		tree.add(r)
+	})
+	if err != nil {
+		return 0, nil, nil, err
+	}
+
 	super, allDirs := tree.finish()
 
-	dupes := collectTreeGroups(allDirs, super)
+	return records, super, allDirs, nil
+}
 
-	out := bufio.NewWriterSize(stdout, ioBufSize)
+// writeTreeGroups writes dupes to stdout as TSV, a header line and then
+// a row for every member of each group but its first, counting each
+// group on prog once its rows are written. stdout may be the terminal
+// prog is drawn on, so the lines go through prog's writer. It returns
+// the number of rows (dupe trees) and the bytes they could reclaim.
+func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
+	prog *progress,
+) (int, int64, error) {
+	out := prog.writer(stdout)
 
-	_, err = fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
+	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
 	if err != nil {
-		return fmt.Errorf("write stdout: %w", err)
+		return 0, 0, fmt.Errorf("write stdout: %w", err)
 	}
 
 	dupeTrees := 0
@@ -81,25 +160,22 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 				escapePath(first.path), escapePath(n.path),
 				first.fileCount, first.totalSize)
 			if err != nil {
-				return fmt.Errorf("write stdout: %w", err)
+				return 0, 0, fmt.Errorf("write stdout: %w", err)
 			}
 
 			dupeTrees++
 			reclaimable += first.totalSize
 		}
+
+		prog.increment()
 	}
 
 	err = out.Flush()
 	if err != nil {
-		return fmt.Errorf("write stdout: %w", err)
+		return 0, 0, fmt.Errorf("write stdout: %w", err)
 	}
 
-	fmt.Fprintf(os.Stderr,
-		"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
-			"%s reclaimable\n",
-		records, len(dupes), dupeTrees, humanBytes(reclaimable))
-
-	return nil
+	return dupeTrees, reclaimable, nil
 }
 
 // treeBuilder reconstructs the directory hierarchy from records added
@@ -221,14 +297,18 @@ func fileEntry(name string, r scanRec) string {
 		"\x00" + r.head + "\x00" + r.tail + "\x00" + content
 }
 
-// collectTreeGroups groups directories by digest and returns every
-// maximal group with two or more members, each group's members sorted
-// by path, groups ordered by tree size descending then by first path
-// ascending.
-func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
+// collectTreeGroups groups directories by digest, counting each one on
+// prog as it is grouped, and returns every maximal group with two or
+// more members, each group's members sorted by path, groups ordered by
+// tree size descending then by first path ascending.
+func collectTreeGroups(allDirs []*treeNode, super *treeNode,
+	prog *progress,
+) [][]*treeNode {
 	groups := make(map[[sha256.Size]byte][]*treeNode)
 	for _, d := range allDirs {
 		groups[d.digest] = append(groups[d.digest], d)
+
+		prog.increment()
 	}
 
 	var dupes [][]*treeNode