trees: show its phases as N of M, with progress, elapsed time and ETA (closes #113)

trees now runs in four phases, open, read, group and write, each under
the progress display scan uses, from the line naming the database to
the summary line. open covers a wait on a busy database. read counts
the records first, as scan's load does, then reads them against that
count; group counts the directories, and write the groups. The display
gains a unit so that each rate names what its phase counts; every scan
phase passes files and prints as before. The tests that pinned trees'
stderr now allow the phase lines between the database line and the
summary.

Judgement call: opening the database is a phase of its own, so there are four phases, not the three the issue sketches.
Judgement call: the count and the read are separate queries, as in scan's load, so during a scan the read can end slightly past or short of its total.

Model: opus-5-5

README.md

@@ -801,8 +801,7 @@ Additional requirements:
   goroutine counts the items, and each display draws them from a goroutine of
   its own.
 - A warning printed during a phase always lands on a line of its own, never
-  inside the progress display. So does every line written to stdout when stdout
+  inside the progress display.
-  is the same terminal, as in a plain `sfdupes trees`.
 - A phase that stops short of its total, as an interrupted one does, is left
   showing the count it stopped at rather than filled up.
 - `report` mode needs no progress display, only its stderr summary.

TODO.md

@@ -36,9 +36,7 @@
   then reads them against that count; `group` counts the directories it groups,
   and `write` the groups it writes. A display's rate names what its phase
   counts: files, records, directories or groups. stdout and the summary line are
-  unchanged, and when stdout is the same terminal, none of its lines shares a
+  unchanged (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/113)
-  line with the display (2026-10-16,
-  https://git.eeqj.de/sneak/sfdupes/issues/113)
 
 - each `scan` phase's progress display names the phase and its place among the
   five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed

db.go

@@ -403,32 +403,10 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
 	return v, nil
 }
 
-// loadFileRows gives total the number of records, then streams every
+// loadFileRows streams every record to fn in path order: byte order,
-// record to fn in path order: byte order, which is the order of the
+// which is the order of the primary key, so SQLite does not sort.
-// primary key, so SQLite does not sort. The count and the records are
+func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
-// read in one transaction, so they agree while a scan is committing.
+	rows, err := db.QueryContext(ctx,
-func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
-	fn func(r scanRec),
-) error {
-	// Everything goes through tx: the report connection is the only
-	// one, so a query on db would wait for tx forever.
-	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
-	if err != nil {
-		return fmt.Errorf("read records: %w", err)
-	}
-
-	defer func() { _ = tx.Rollback() }()
-
-	var n int64
-
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
-	if err != nil {
-		return fmt.Errorf("count records: %w", err)
-	}
-
-	total(n)
-
-	rows, err := tx.QueryContext(ctx,
 		"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
 			"FROM files ORDER BY path")
 	if err != nil {

db_test.go

@@ -284,63 +284,12 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
 
 	defer func() { _ = reportDB.Close() }()
 
-	err = loadFileRows(t.Context(), reportDB, func(int64) {},
+	err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
-		func(scanRec) {})
 	if err != nil {
 		t.Fatalf("loadFileRows: %v", err)
 	}
 }
 
-// TestLoadFileRowsCountAgreesDuringScan has a scan commit records after
-// loadFileRows has counted the records and before it reads them, as
-// when trees runs during a scan: it must read exactly as many records
-// as it counted.
-func TestLoadFileRowsCountAgreesDuringScan(t *testing.T) {
-	t.Parallel()
-
-	path := testDBPath(t)
-
-	scanDB, err := openScanDatabase(t.Context(), path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = scanDB.Close() }()
-
-	recs := smokeTreeRecs()
-
-	err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	reportDB, err := openReportDatabase(t.Context(), path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() { _ = reportDB.Close() }()
-
-	var counted, read int64
-
-	err = loadFileRows(t.Context(), reportDB, func(n int64) {
-		counted = n
-
-		err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
-		if err != nil {
-			t.Error(err)
-		}
-	}, func(scanRec) { read++ })
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if counted != 3 || read != counted {
-		t.Errorf("counted %d records and read %d, want 3 and 3",
-			counted, read)
-	}
-}
-
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()
 

progress.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"math"
 	"os"
 	"slices"
@@ -92,8 +91,7 @@ type progressParams struct {
 // below it. Every method must be called from the main goroutine. A nil
 // *progress is a valid no-display receiver: every method is a no-op, so
 // batched database flushes during the hash phase can reuse the update
-// phase's helpers without rendering anything, and its writer writes
+// phase's helpers without rendering anything.
-// with no display to clear.
 type progress struct {
 	width func() int    // reads the terminal's width; nil when not a TTY
 	stop  chan struct{} // closed by finish to end the draws
@@ -292,56 +290,6 @@ func (p *progress) warnf(format string, args ...any) {
 	fmt.Fprintln(os.Stderr, msg)
 }
 
-// progressWriter writes a phase's output to out, which may be the
-// terminal the phase's display is drawn on, as stdout is in a plain
-// "sfdupes trees". It holds what is written to it and writes that to
-// out in pieces of about ioBufSize bytes, each under the display's
-// lock with the display cleared first, as warnf does for a warning: no
-// draw lands in the middle of a piece, and the next one shows the
-// display again, on the line below. Every write to it must end with a
-// newline, as a row written with one fmt.Fprintf does, so that each
-// piece ends a line. Flush writes what it holds once the last row is
-// written.
-type progressWriter struct {
-	p   *progress
-	out io.Writer
-	buf []byte
-}
-
-// writer returns a progressWriter writing to out around p's display.
-func (p *progress) writer(out io.Writer) *progressWriter {
-	return &progressWriter{p: p, out: out}
-}
-
-// Write holds b, and writes what it holds to out once that comes to
-// ioBufSize bytes.
-func (w *progressWriter) Write(b []byte) (int, error) {
-	w.buf = append(w.buf, b...)
-	if len(w.buf) < ioBufSize {
-		return len(b), nil
-	}
-
-	return len(b), w.Flush()
-}
-
-// Flush writes what w holds to out, with the display cleared first and
-// not drawn again until it is written.
-func (w *progressWriter) Flush() error {
-	if w.p != nil {
-		w.p.mu.Lock()
-		defer w.p.mu.Unlock()
-
-		if w.p.bar != nil {
-			_ = w.p.bar.Clear()
-		}
-	}
-
-	_, err := w.out.Write(w.buf)
-	w.buf = w.buf[:0]
-
-	return err
-}
-
 // finish ends the phase's display, drawn once more as the phase ends.
 // A phase stopped short of its total, as an interrupted one is, shows
 // the count it stopped at rather than a full bar.

progress_test.go

@@ -1,11 +1,9 @@
 package main
 
 import (
-	"fmt"
 	"math"
 	"os"
 	"path/filepath"
-	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -560,64 +558,6 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
 	}
 }
 
-// TestProgressWriterRowsOnOwnLines drives the terminal displays,
-// spinner and bar, through a run of rows written through the display's
-// writer to the same terminal, as stdout is in a plain "sfdupes trees",
-// each written out at once, for many redraws: the terminal must show
-// every row whole, in order, on a line of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestProgressWriterRowsOnOwnLines(t *testing.T) {
-	stderr := captureStderr(t)
-
-	cases := []progressParams{
-		{
-			label: treesPhase("open"), total: -1,
-			waitingFor: "opening the database",
-		},
-		{label: treesPhase("write"), unit: "groups", total: 10},
-	}
-
-	var want []string
-
-	for _, params := range cases {
-		p := startProgress(params, terminalWidth, testRedraw)
-		out := p.writer(os.Stderr)
-
-		// No pause between rows: one is garbled only if a redraw lands
-		// on its line.
-		for start := time.Now(); time.Since(start) < 50*testRedraw; {
-			row := fmt.Sprintf("row %d", len(want))
-			want = append(want, row)
-
-			_, err := fmt.Fprintln(out, row)
-			if err == nil {
-				err = out.Flush()
-			}
-
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-
-		p.finish()
-	}
-
-	// A terminal shows each line as the text after its last carriage
-	// return.
-	var shown []string
-
-	for line := range strings.SplitSeq(stderr(), "\n") {
-		if strings.Contains(line, "row ") {
-			shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
-		}
-	}
-
-	if !slices.Equal(shown, want) {
-		t.Errorf("terminal shows rows %q, want %q", shown, want)
-	}
-}
-
 // TestTerminalStoppedShortKeepsCount checks that the terminal display
 // of a phase that stops before its total, as an interrupted one does,
 // is left showing the count it stopped at rather than a full bar.

scan_test.go

@@ -1478,7 +1478,7 @@ func dbRecords(t *testing.T, db *sql.DB) []scanRec {
 
 	var recs []scanRec
 
-	err := loadFileRows(t.Context(), db, func(int64) {}, func(r scanRec) {
+	err := loadFileRows(t.Context(), db, func(r scanRec) {
 		recs = append(recs, r)
 	})
 	if err != nil {

trees.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"crypto/sha256"
 	"database/sql"
@@ -109,17 +110,24 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 	return nil
 }
 
-// readTree gives prog the number of records in db as its total, then
+// readTree counts the records in db and gives prog the count as its
-// reads every record in path order into the directory hierarchy,
+// total, then reads every record in path order into the directory
-// counting each on prog. It returns the number of records read, the
+// hierarchy, counting each on prog. It returns the number of records
-// super-root and every directory.
+// read, the super-root and every directory.
 func readTree(ctx context.Context, db *sql.DB,
 	prog *progress,
 ) (int, *treeNode, []*treeNode, error) {
+	total, err := countRecords(ctx, db)
+	if err != nil {
+		return 0, nil, nil, err
+	}
+
+	prog.setTotal(total)
+
 	records := 0
 	tree := newTreeBuilder()
 
-	err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
+	err = loadFileRows(ctx, db, func(r scanRec) {
 		records++
 
 		prog.increment()
@@ -136,13 +144,12 @@ func readTree(ctx context.Context, db *sql.DB,
 
 // writeTreeGroups writes dupes to stdout as TSV, a header line and then
 // a row for every member of each group but its first, counting each
-// group on prog once its rows are written. stdout may be the terminal
+// group on prog once its rows are written. It returns the number of
-// prog is drawn on, so the lines go through prog's writer. It returns
+// rows (dupe trees) and the bytes they could reclaim.
-// the number of rows (dupe trees) and the bytes they could reclaim.
 func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 	prog *progress,
 ) (int, int64, error) {
-	out := prog.writer(stdout)
+	out := bufio.NewWriterSize(stdout, ioBufSize)
 
 	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
 	if err != nil {

trees_test.go

@@ -7,7 +7,6 @@ import (
 	"slices"
 	"strings"
 	"testing"
-	"time"
 )
 
 // Signature hashes shared by the smoke-test records.
@@ -46,7 +45,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
 
 	tree := newTreeBuilder()
 
-	err := loadFileRows(t.Context(), db, func(int64) {}, tree.add)
+	err := loadFileRows(t.Context(), db, tree.add)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -265,44 +264,6 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
 	}
 }
 
-// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
-// terminal its write phase's display is drawn on, as a plain "sfdupes
-// trees" does: the terminal must show the header and the row whole,
-// each on a line of its own.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestWriteTreeGroupsOnTerminal(t *testing.T) {
-	stderr := captureStderr(t)
-	super, dirs := treeOf(t, smokeTreeRecs())
-	dupes := collectTreeGroups(dirs, super, nil)
-
-	// No timed draws: the display is drawn as it starts and as it ends.
-	p := startProgress(progressParams{
-		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
-	}, terminalWidth, time.Hour)
-
-	_, _, err := writeTreeGroups(os.Stderr, dupes, p)
-
-	p.finish()
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// A terminal shows each line as the text after its last carriage
-	// return.
-	var shown []string
-
-	for line := range strings.SplitSeq(stderr(), "\n") {
-		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
-	}
-
-	want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
-	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
-		t.Errorf("terminal shows lines %q, want %q first", shown, want)
-	}
-}
-
 func TestTreeDigests(t *testing.T) {
 	t.Parallel()