trees: show its phases as N of M, with progress, elapsed time and ETA (closes #113)

trees now runs in four phases, open, read, group and write, each under
the progress display scan uses, until the summary line. open covers a
wait on a busy database. read counts the records and reads them in one
read-only transaction, so the two agree during a scan; group counts the
directories, and write the groups. The rows reach stdout through a
writer that clears the display and holds its lock while writing, so no
row shares a line with the display on a shared terminal. Each rate
names what its phase counts.

Judgement call: opening the database is a phase of its own, so there are four phases, not the three the issue sketches.

Model: opus-5-5

README.md

@@ -801,7 +801,8 @@ Additional requirements:
   goroutine counts the items, and each display draws them from a goroutine of
   its own.
 - A warning printed during a phase always lands on a line of its own, never
-  inside the progress display.
+  inside the progress display. So does every line written to stdout when stdout
+  is the same terminal, as in a plain `sfdupes trees`.
 - A phase that stops short of its total, as an interrupted one does, is left
   showing the count it stopped at rather than filled up.
 - `report` mode needs no progress display, only its stderr summary.

TODO.md

@@ -36,7 +36,9 @@
   then reads them against that count; `group` counts the directories it groups,
   and `write` the groups it writes. A display's rate names what its phase
   counts: files, records, directories or groups. stdout and the summary line are
-  unchanged (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/113)
+  unchanged, and when stdout is the same terminal, none of its lines shares a
+  line with the display (2026-10-16,
+  https://git.eeqj.de/sneak/sfdupes/issues/113)
 
 - each `scan` phase's progress display names the phase and its place among the
   five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed

db.go

@@ -403,10 +403,32 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
 	return v, nil
 }
 
-// loadFileRows streams every record to fn in path order: byte order,
+// loadFileRows gives total the number of records, then streams every
-// which is the order of the primary key, so SQLite does not sort.
+// record to fn in path order: byte order, which is the order of the
-func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
+// primary key, so SQLite does not sort. The count and the records are
-	rows, err := db.QueryContext(ctx,
+// read in one transaction, so they agree while a scan is committing.
+func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
+	fn func(r scanRec),
+) error {
+	// Everything goes through tx: the report connection is the only
+	// one, so a query on db would wait for tx forever.
+	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
+	if err != nil {
+		return fmt.Errorf("read records: %w", err)
+	}
+
+	defer func() { _ = tx.Rollback() }()
+
+	var n int64
+
+	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	if err != nil {
+		return fmt.Errorf("count records: %w", err)
+	}
+
+	total(n)
+
+	rows, err := tx.QueryContext(ctx,
 		"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
 			"FROM files ORDER BY path")
 	if err != nil {

db_test.go

@@ -284,12 +284,63 @@ func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
 
 	defer func() { _ = reportDB.Close() }()
 
-	err = loadFileRows(t.Context(), reportDB, func(scanRec) {})
+	err = loadFileRows(t.Context(), reportDB, func(int64) {},
+		func(scanRec) {})
 	if err != nil {
 		t.Fatalf("loadFileRows: %v", err)
 	}
 }
 
+// TestLoadFileRowsCountAgreesDuringScan has a scan commit records after
+// loadFileRows has counted the records and before it reads them, as
+// when trees runs during a scan: it must read exactly as many records
+// as it counted.
+func TestLoadFileRowsCountAgreesDuringScan(t *testing.T) {
+	t.Parallel()
+
+	path := testDBPath(t)
+
+	scanDB, err := openScanDatabase(t.Context(), path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() { _ = scanDB.Close() }()
+
+	recs := smokeTreeRecs()
+
+	err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportDB, err := openReportDatabase(t.Context(), path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() { _ = reportDB.Close() }()
+
+	var counted, read int64
+
+	err = loadFileRows(t.Context(), reportDB, func(n int64) {
+		counted = n
+
+		err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
+		if err != nil {
+			t.Error(err)
+		}
+	}, func(scanRec) { read++ })
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if counted != 3 || read != counted {
+		t.Errorf("counted %d records and read %d, want 3 and 3",
+			counted, read)
+	}
+}
+
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()
 

progress.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"math"
 	"os"
 	"slices"
@@ -91,7 +92,8 @@ type progressParams struct {
 // below it. Every method must be called from the main goroutine. A nil
 // *progress is a valid no-display receiver: every method is a no-op, so
 // batched database flushes during the hash phase can reuse the update
-// phase's helpers without rendering anything.
+// phase's helpers without rendering anything, and its writer writes
+// with no display to clear.
 type progress struct {
 	width func() int    // reads the terminal's width; nil when not a TTY
 	stop  chan struct{} // closed by finish to end the draws
@@ -290,6 +292,56 @@ func (p *progress) warnf(format string, args ...any) {
 	fmt.Fprintln(os.Stderr, msg)
 }
 
+// progressWriter writes a phase's output to out, which may be the
+// terminal the phase's display is drawn on, as stdout is in a plain
+// "sfdupes trees". It holds what is written to it and writes that to
+// out in pieces of about ioBufSize bytes, each under the display's
+// lock with the display cleared first, as warnf does for a warning: no
+// draw lands in the middle of a piece, and the next one shows the
+// display again, on the line below. Every write to it must end with a
+// newline, as a row written with one fmt.Fprintf does, so that each
+// piece ends a line. Flush writes what it holds once the last row is
+// written.
+type progressWriter struct {
+	p   *progress
+	out io.Writer
+	buf []byte
+}
+
+// writer returns a progressWriter writing to out around p's display.
+func (p *progress) writer(out io.Writer) *progressWriter {
+	return &progressWriter{p: p, out: out}
+}
+
+// Write holds b, and writes what it holds to out once that comes to
+// ioBufSize bytes.
+func (w *progressWriter) Write(b []byte) (int, error) {
+	w.buf = append(w.buf, b...)
+	if len(w.buf) < ioBufSize {
+		return len(b), nil
+	}
+
+	return len(b), w.Flush()
+}
+
+// Flush writes what w holds to out, with the display cleared first and
+// not drawn again until it is written.
+func (w *progressWriter) Flush() error {
+	if w.p != nil {
+		w.p.mu.Lock()
+		defer w.p.mu.Unlock()
+
+		if w.p.bar != nil {
+			_ = w.p.bar.Clear()
+		}
+	}
+
+	_, err := w.out.Write(w.buf)
+	w.buf = w.buf[:0]
+
+	return err
+}
+
 // finish ends the phase's display, drawn once more as the phase ends.
 // A phase stopped short of its total, as an interrupted one is, shows
 // the count it stopped at rather than a full bar.

progress_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"fmt"
 	"math"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -558,6 +560,64 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
 	}
 }
 
+// TestProgressWriterRowsOnOwnLines drives the terminal displays,
+// spinner and bar, through a run of rows written through the display's
+// writer to the same terminal, as stdout is in a plain "sfdupes trees",
+// each written out at once, for many redraws: the terminal must show
+// every row whole, in order, on a line of its own.
+//
+//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
+func TestProgressWriterRowsOnOwnLines(t *testing.T) {
+	stderr := captureStderr(t)
+
+	cases := []progressParams{
+		{
+			label: treesPhase("open"), total: -1,
+			waitingFor: "opening the database",
+		},
+		{label: treesPhase("write"), unit: "groups", total: 10},
+	}
+
+	var want []string
+
+	for _, params := range cases {
+		p := startProgress(params, terminalWidth, testRedraw)
+		out := p.writer(os.Stderr)
+
+		// No pause between rows: one is garbled only if a redraw lands
+		// on its line.
+		for start := time.Now(); time.Since(start) < 50*testRedraw; {
+			row := fmt.Sprintf("row %d", len(want))
+			want = append(want, row)
+
+			_, err := fmt.Fprintln(out, row)
+			if err == nil {
+				err = out.Flush()
+			}
+
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		p.finish()
+	}
+
+	// A terminal shows each line as the text after its last carriage
+	// return.
+	var shown []string
+
+	for line := range strings.SplitSeq(stderr(), "\n") {
+		if strings.Contains(line, "row ") {
+			shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
+		}
+	}
+
+	if !slices.Equal(shown, want) {
+		t.Errorf("terminal shows rows %q, want %q", shown, want)
+	}
+}
+
 // TestTerminalStoppedShortKeepsCount checks that the terminal display
 // of a phase that stops before its total, as an interrupted one does,
 // is left showing the count it stopped at rather than a full bar.

scan_test.go

@@ -1478,7 +1478,7 @@ func dbRecords(t *testing.T, db *sql.DB) []scanRec {
 
 	var recs []scanRec
 
-	err := loadFileRows(t.Context(), db, func(r scanRec) {
+	err := loadFileRows(t.Context(), db, func(int64) {}, func(r scanRec) {
 		recs = append(recs, r)
 	})
 	if err != nil {

trees.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bufio"
 	"context"
 	"crypto/sha256"
 	"database/sql"
@@ -110,24 +109,17 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
 	return nil
 }
 
-// readTree counts the records in db and gives prog the count as its
+// readTree gives prog the number of records in db as its total, then
-// total, then reads every record in path order into the directory
+// reads every record in path order into the directory hierarchy,
-// hierarchy, counting each on prog. It returns the number of records
+// counting each on prog. It returns the number of records read, the
-// read, the super-root and every directory.
+// super-root and every directory.
 func readTree(ctx context.Context, db *sql.DB,
 	prog *progress,
 ) (int, *treeNode, []*treeNode, error) {
-	total, err := countRecords(ctx, db)
-	if err != nil {
-		return 0, nil, nil, err
-	}
-
-	prog.setTotal(total)
-
 	records := 0
 	tree := newTreeBuilder()
 
-	err = loadFileRows(ctx, db, func(r scanRec) {
+	err := loadFileRows(ctx, db, prog.setTotal, func(r scanRec) {
 		records++
 
 		prog.increment()
@@ -144,12 +136,13 @@ func readTree(ctx context.Context, db *sql.DB,
 
 // writeTreeGroups writes dupes to stdout as TSV, a header line and then
 // a row for every member of each group but its first, counting each
-// group on prog once its rows are written. It returns the number of
+// group on prog once its rows are written. stdout may be the terminal
-// rows (dupe trees) and the bytes they could reclaim.
+// prog is drawn on, so the lines go through prog's writer. It returns
+// the number of rows (dupe trees) and the bytes they could reclaim.
 func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 	prog *progress,
 ) (int, int64, error) {
-	out := bufio.NewWriterSize(stdout, ioBufSize)
+	out := prog.writer(stdout)
 
 	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
 	if err != nil {

trees_test.go

@@ -7,6 +7,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 // Signature hashes shared by the smoke-test records.
@@ -45,7 +46,7 @@ func dbTree(t *testing.T, db *sql.DB) (*treeNode, []*treeNode) {
 
 	tree := newTreeBuilder()
 
-	err := loadFileRows(t.Context(), db, tree.add)
+	err := loadFileRows(t.Context(), db, func(int64) {}, tree.add)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -264,6 +265,44 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
 	}
 }
 
+// TestWriteTreeGroupsOnTerminal writes the smoke-test report to the
+// terminal its write phase's display is drawn on, as a plain "sfdupes
+// trees" does: the terminal must show the header and the row whole,
+// each on a line of its own.
+//
+//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
+func TestWriteTreeGroupsOnTerminal(t *testing.T) {
+	stderr := captureStderr(t)
+	super, dirs := treeOf(t, smokeTreeRecs())
+	dupes := collectTreeGroups(dirs, super, nil)
+
+	// No timed draws: the display is drawn as it starts and as it ends.
+	p := startProgress(progressParams{
+		label: treesPhase("write"), unit: "groups", total: int64(len(dupes)),
+	}, terminalWidth, time.Hour)
+
+	_, _, err := writeTreeGroups(os.Stderr, dupes, p)
+
+	p.finish()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A terminal shows each line as the text after its last carriage
+	// return.
+	var shown []string
+
+	for line := range strings.SplitSeq(stderr(), "\n") {
+		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
+	}
+
+	want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
+	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
+		t.Errorf("terminal shows lines %q, want %q first", shown, want)
+	}
+}
+
 func TestTreeDigests(t *testing.T) {
 	t.Parallel()