Stream report and trees instead of loading every record (closes #14)

report now has SQLite group the records and put the rows in report
order, helped by a new files_signature index on (size, head, tail,
content), and writes each row as it reads it. trees reads the records
in path order, where all the paths under a directory come together, so
it computes each directory's digest as soon as the stream leaves it and
keeps only its path, parent, digest and totals. Output is unchanged.

The tests that called the removed in-memory grouping functions now group
records stored in a database. New tests check that both commands give
the same output whatever order the records were inserted in, and that a
stdout failure partway through a long report is reported as one.

Model: opus-5-5

report.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"slices"
 	"strings"
 )
 
@@ -29,50 +28,21 @@ type scanRec struct {
 	path    string
 }
 
-// loadRecords opens the database and reads every file record for the
-// report and trees subcommands. Any database problem — including a
-// missing database — is fatal. The error is returned rather than
-// exiting, so that the deferred close always runs; the database is
-// closed before the caller formats its output, so it stays closed even
-// if that output fails.
-func loadRecords(ctx context.Context) ([]scanRec, error) {
+// runReport implements the report subcommand: it prints the file-level
+// duplicates report as TSV on stdout. SQLite groups and orders the
+// records, and each row is written as it is read, so no group is held
+// in memory. It never touches the scanned filesystem; its only I/O is
+// the database (with SQLite's temporary sort file), stdout, and stderr.
+// Any database problem, including a missing database, is fatal.
+func runReport(ctx context.Context, stdout io.Writer) error {
 	dbPath := databasePath()
 
 	db, err := openReportDatabase(ctx, dbPath)
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() { _ = db.Close() }()
-
-	recs, err := loadFileRows(ctx, db)
-	if err != nil {
-		return nil, fmt.Errorf("database %s: %w", dbPath, err)
-	}
-
-	return recs, nil
-}
-
-// dupeGroup is one set of candidate-duplicate files: identical size,
-// head hash, tail hash, and content hash. paths is sorted
-// lexicographically; the first entry is the group's "first", the rest
-// are dupes.
-type dupeGroup struct {
-	size  int64
-	paths []string
-}
-
-// runReport implements the report subcommand: it reads every record
-// from the database and prints the file-level duplicates report as TSV
-// on stdout. It never touches the scanned filesystem; its only I/O is
-// the database, stdout, and stderr.
-func runReport(ctx context.Context, stdout io.Writer) error {
-	recs, err := loadRecords(ctx)
 	if err != nil {
 		return err
 	}
 
-	dupes := collectDupeGroups(recs)
+	defer func() { _ = db.Close() }()
 
 	out := bufio.NewWriterSize(stdout, ioBufSize)
 
@@ -81,21 +51,36 @@ func runReport(ctx context.Context, stdout io.Writer) error {
 		return fmt.Errorf("write stdout: %w", err)
 	}
 
-	dupeFiles := 0
+	var (
+		groups, dupeFiles int
+		reclaimable       int64
+		writeErr          error
+	)
 
-	var reclaimable int64
+	records, err := loadDupeRows(ctx, db,
+		func(first, path string, size int64) error {
+			// A group's first path is its first row; every other
+			// path is a dupe.
+			if path == first {
+				groups++
 
-	for _, g := range dupes {
-		for _, p := range g.paths[1:] {
-			_, err = fmt.Fprintf(out, "%s\t%s\t%d\n",
-				escapePath(g.paths[0]), escapePath(p), g.size)
-			if err != nil {
-				return fmt.Errorf("write stdout: %w", err)
+				return nil
 			}
 
+			_, writeErr = fmt.Fprintf(out, "%s\t%s\t%d\n",
+				escapePath(first), escapePath(path), size)
 			dupeFiles++
-			reclaimable += g.size
-		}
+			reclaimable += size
+
+			return writeErr
+		})
+
+	if writeErr != nil {
+		return fmt.Errorf("write stdout: %w", writeErr)
+	}
+
+	if err != nil {
+		return fmt.Errorf("database %s: %w", dbPath, err)
 	}
 
 	err = out.Flush()
@@ -106,57 +91,11 @@ func runReport(ctx context.Context, stdout io.Writer) error {
 	fmt.Fprintf(os.Stderr,
 		"report: %d records read, %d duplicate groups, %d dupe files, "+
 			"%s reclaimable\n",
-		len(recs), len(dupes), dupeFiles, humanBytes(reclaimable))
+		records, groups, dupeFiles, humanBytes(reclaimable))
 
 	return nil
 }
 
-// collectDupeGroups groups records by signature and returns every group
-// with two or more paths, each group's paths sorted lexicographically,
-// groups ordered by size descending then by first path ascending.
-func collectDupeGroups(recs []scanRec) []dupeGroup {
-	groups := make(map[fileSig][]string)
-
-	for _, r := range recs {
-		// A record without a content hash has unknown content and is
-		// never reported as a duplicate (README "Database").
-		if r.content == "" {
-			continue
-		}
-
-		k := fileSig{
-			size: r.size, head: r.head, tail: r.tail, content: r.content,
-		}
-		groups[k] = append(groups[k], r.path)
-	}
-
-	var dupes []dupeGroup
-
-	for k, paths := range groups {
-		if len(paths) < minGroupSize {
-			continue
-		}
-
-		slices.Sort(paths)
-		dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
-	}
-
-	// Biggest reclaimable space first; ties broken by first path.
-	slices.SortFunc(dupes, func(a, b dupeGroup) int {
-		if a.size != b.size {
-			if a.size > b.size {
-				return -1
-			}
-
-			return 1
-		}
-
-		return strings.Compare(a.paths[0], b.paths[0])
-	})
-
-	return dupes
-}
-
 // escapePath returns a path as it is written in a report column (README
 // "Report output format"): a backslash, tab, newline or carriage return
 // becomes \\, \t, \n or \r, and every other byte is kept as it is.