report: show its phases as N of M, with progress, elapsed time and ETA (closes #114)

report now runs in two phases, open and read, under the progress
display scan and trees use, until the summary line. open covers a wait
on a busy database. read counts the records, then reads them in the
same read-only transaction in signature order, the files_signature
index read backwards, so SQLite sorts nothing and the read ends at
exactly its count. The duplicate groups are formed in Go from runs of
equal signatures; only those of the size being read are held, and they
are written, ordered by first path, before the next size. The rows go
through the display's stdout writer. stdout and the summary line are
unchanged.

Judgement call: writing is part of read, not a phase of its own, since each size's rows are written while the read goes on.

Model: opus-5-5

report.go

@@ -1,11 +1,12 @@
 package main
 
 import (
-	"bufio"
 	"context"
+	"database/sql"
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"strings"
 	"time"
 )
@@ -29,13 +30,25 @@ type scanRec struct {
 	path    string
 }
 
+// reportPhase names one phase of report with its place among all of
+// them, as "[2/2] read", for the phase's progress display. The list
+// here is the one place the phases are counted.
+func reportPhase(name string) string {
+	return phaseLabel([]string{"open", "read"}, name)
+}
+
 // runReport implements the report subcommand: it prints the file-level
-// duplicates report as TSV on stdout. SQLite groups and orders the
-// records, and each row is written as it is read, so no group is held
-// in memory. It never touches the scanned filesystem; its only I/O is
-// the database (with SQLite's temporary sort file), stdout, and stderr.
-// It names the database on stderr before it opens it. Any database
+// duplicates report as TSV on stdout. It never touches the scanned
+// filesystem; its only I/O is the database, stdout, and stderr. It
+// names the database on stderr before it opens it. Any database
 // problem, including a missing database, is fatal.
+//
+// The work runs in two phases, each under a progress display of its
+// own on stderr: open (open the database) and read (count the records,
+// then read them in signature order, writing each size's duplicate
+// groups to stdout once that size is read; see readReport). Each
+// display ends before the next phase starts, and before an error is
+// returned to be printed.
 func runReport(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -45,65 +58,194 @@ func runReport(ctx context.Context, stdout io.Writer) error {
 	fmt.Fprintf(os.Stderr, "report: database %s (%s)\n",
 		escapePath(dbPath), source)
 
+	// Opening can wait on a busy database for as long as the busy
+	// timeout.
+	prog := newProgress(progressParams{
+		label: reportPhase("open"), total: -1,
+		waitingFor: "opening the database",
+	})
 	db, err := openReportDatabase(ctx, dbPath)
+
+	prog.finish()
+
 	if err != nil {
 		return err
 	}
 
 	defer func() { _ = db.Close() }()
 
-	out := bufio.NewWriterSize(stdout, ioBufSize)
+	prog = newProgress(progressParams{
+		label: reportPhase("read"), unit: "records", total: -1,
+		waitingFor: countingRecords,
+	})
+	report, err := readReport(ctx, db, dbPath, stdout, prog)
 
-	_, err = fmt.Fprintln(out, "first\tdupe\tsize")
-	if err != nil {
-		return fmt.Errorf("write stdout: %w", err)
-	}
-
-	var (
-		groups, dupeFiles int
-		reclaimable       int64
-		writeErr          error
-	)
-
-	records, err := loadDupeRows(ctx, db,
-		func(first, path string, size int64) error {
-			// A group's first path is its first row; every other
-			// path is a dupe.
-			if path == first {
-				groups++
-
-				return nil
-			}
-
-			_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
-				escapePath(first), escapePath(path), size)
-			dupeFiles++
-			reclaimable += size
-
-			return writeErr
-		})
-
-	if writeErr != nil {
-		return fmt.Errorf("write stdout: %w", writeErr)
-	}
+	prog.finish()
 
 	if err != nil {
-		return fmt.Errorf("database %s: %w", dbPath, err)
-	}
-
-	err = out.Flush()
-	if err != nil {
-		return fmt.Errorf("write stdout: %w", err)
+		return err
 	}
 
 	fmt.Fprintf(os.Stderr,
 		"report: %d records read, %d duplicate groups, %d dupe files, "+
 			"%s reclaimable\n",
-		records, groups, dupeFiles, humanBytes(reclaimable))
+		report.records, report.dupeGroups, report.dupeFiles,
+		humanBytes(report.reclaimable))
 
 	return nil
 }
 
+// readReport gives prog the number of records in db, the database at
+// dbPath, as its total, then reads every record in signature order,
+// counting each on prog, and writes the report to stdout: the header
+// line, then the rows of each size's duplicate groups once that size is
+// read. stdout may be the terminal prog is drawn on, so the lines go
+// through prog's writer. It returns what it wrote, with the totals for
+// the summary line.
+func readReport(ctx context.Context, db *sql.DB, dbPath string,
+	stdout io.Writer, prog *progress,
+) (*reportWriter, error) {
+	out := prog.writer(stdout)
+	report := &reportWriter{out: out}
+
+	_, err := fmt.Fprintln(out, "first\tdupe\tsize")
+	if err != nil {
+		return nil, fmt.Errorf("write stdout: %w", err)
+	}
+
+	var writeErr error
+
+	err = loadSignatureRows(ctx, db, prog.setTotal, func(r scanRec) error {
+		prog.increment()
+
+		writeErr = report.add(r)
+
+		return writeErr
+	})
+
+	if writeErr != nil {
+		return nil, fmt.Errorf("write stdout: %w", writeErr)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("database %s: %w", dbPath, err)
+	}
+
+	err = report.finish()
+	if err == nil {
+		err = out.Flush()
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("write stdout: %w", err)
+	}
+
+	return report, nil
+}
+
+// reportWriter writes the report's rows to out from the records, added
+// in signature order, largest size first, as loadSignatureRows reads
+// them. The records sharing a signature come together, and so do the
+// records of one size, so it holds only the duplicate groups of the
+// size being read, and writes them, ordered by first path, once a
+// record of a smaller size comes. It counts what the summary line
+// reports.
+type reportWriter struct {
+	out io.Writer
+
+	sig    scanRec    // the signature being read; its path is not used
+	paths  []string   // the paths read so far with sig
+	groups [][]string // the duplicate groups of sig.size, each sorted
+
+	records, dupeGroups, dupeFiles int
+	reclaimable                    int64
+}
+
+// add takes the next record. One with the signature being read joins
+// its paths; any other ends them, and one of a smaller size first
+// writes the groups of the size before.
+func (w *reportWriter) add(r scanRec) error {
+	w.records++
+
+	// A record without a content hash has unknown content (README
+	// "Database"), so it is never a duplicate.
+	if r.content == "" {
+		return nil
+	}
+
+	if r.size == w.sig.size && r.head == w.sig.head &&
+		r.tail == w.sig.tail && r.content == w.sig.content {
+		w.paths = append(w.paths, r.path)
+
+		return nil
+	}
+
+	w.endGroup()
+
+	if r.size != w.sig.size {
+		err := w.writeGroups()
+		if err != nil {
+			return err
+		}
+	}
+
+	w.sig = r
+	w.paths = append(w.paths, r.path)
+
+	return nil
+}
+
+// endGroup ends the paths read with w.sig: two or more are a duplicate
+// group, held with its paths sorted until writeGroups writes it.
+func (w *reportWriter) endGroup() {
+	if len(w.paths) < minGroupSize {
+		// A path with no duplicate is dropped, and its slice reused.
+		w.paths = w.paths[:0]
+
+		return
+	}
+
+	slices.Sort(w.paths)
+	w.groups = append(w.groups, w.paths)
+	w.paths = nil
+}
+
+// writeGroups writes the duplicate groups held, all of size w.sig.size,
+// ordered by first path: for each group, a row for every path but the
+// first. No two groups share a first path, since no two records share
+// a path.
+func (w *reportWriter) writeGroups() error {
+	slices.SortFunc(w.groups, func(a, b []string) int {
+		return strings.Compare(a[0], b[0])
+	})
+
+	for _, g := range w.groups {
+		for _, path := range g[1:] {
+			_, err := fmt.Fprintf(w.out, "%s\t%s\t%d\n",
+				escapePath(g[0]), escapePath(path), w.sig.size)
+			if err != nil {
+				return err
+			}
+
+			w.dupeFiles++
+			w.reclaimable += w.sig.size
+		}
+
+		w.dupeGroups++
+	}
+
+	w.groups = nil
+
+	return nil
+}
+
+// finish writes the groups of the last size read.
+func (w *reportWriter) finish() error {
+	w.endGroup()
+
+	return w.writeGroups()
+}
+
 // escapePath returns a path as it is written in a report column (README
 // "Report output format"): a backslash, tab, newline or carriage return
 // becomes \\, \t, \n or \r, and every other byte is kept as it is.