Keep the record count and total size in a totals table (closes #146 )

info counted the records and added up their sizes with a query that reads every record, which takes minutes on a large database. The schema now has a totals table of one row, kept current by three triggers on files, in the transaction of each insert, delete and size update. info, scan's load phase and the read phases of report and trees read that row instead of counting over files. scan writes files as before. Model: opus-5-5
2026-10-27 16:53:53 +00:00
10 changed files with 1155 additions and 1086 deletions
@@ -29,13 +29,11 @@

 # Completed Steps

- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
-  it. It leads with what the program is and how to use it. A reference section
-  covers every setting and every subcommand. Development and testing come last.
-  Every statement was checked against the code on `next`. Five comments in
-  `main_test.go` cite "Settings", where their rules now live. One comment each
-  in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
-  now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
+- the database keeps the number of records and the sum of their sizes in the one
+  row of a `totals` table, which three triggers on `files` keep current. `info`
+  reads that row, and so do `scan`'s `load` phase and the `read` phases of
+  `report` and `trees` for their totals; no count reads every record any more
+  (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/146)

 - an operand at or inside a directory named `.zfs` is never scanned, whether
  given alone or under another operand. `scan` warns about it, counts it as
@@ -69,6 +69,49 @@ const createIndexSQL = `
 CREATE INDEX files_signature ON files (size, head, tail, content)
 `

+// createTotalsSQL creates the totals table, which holds one row: the
+// number of records and the sizes of all records added up. Counting
+// them over files would read every record. The three triggers below
+// keep the row current, in the transaction of each change to files.
+const createTotalsSQL = `
+CREATE TABLE totals (
+    records INTEGER NOT NULL,
+    size    INTEGER NOT NULL
+)
+`
+
+// insertTotalsSQL adds the one row of totals, for an empty files table.
+const insertTotalsSQL = `
+INSERT INTO totals (records, size) VALUES (0, 0)
+`
+
+// totalsAfterInsertSQL counts each inserted record into totals.
+const totalsAfterInsertSQL = `
+CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
+    UPDATE totals SET records = records + 1, size = size + NEW.size;
+END
+`
+
+// totalsAfterDeleteSQL takes each deleted record out of totals.
+const totalsAfterDeleteSQL = `
+CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
+    UPDATE totals SET records = records - 1, size = size - OLD.size;
+END
+`
+
+// totalsAfterUpdateSQL replaces a record's old size in totals with its
+// new one. scan's upsert of an existing record is such an update.
+const totalsAfterUpdateSQL = `
+CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
+    UPDATE totals SET size = size + NEW.size - OLD.size;
+END
+`
+
+// recordCountSQL reads the number of records from totals.
+const recordCountSQL = `
+SELECT records FROM totals
+`
+
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
@@ -426,14 +469,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {

 	defer func() { _ = tx.Rollback() }()

-	_, err = tx.ExecContext(ctx, createTableSQL)
-	if err != nil {
-		return fmt.Errorf("create schema: %w", err)
-	}
-
-	_, err = tx.ExecContext(ctx, createIndexSQL)
-	if err != nil {
-		return fmt.Errorf("create schema: %w", err)
+	for _, stmt := range []string{
+		createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
+		totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
+	} {
+		_, err = tx.ExecContext(ctx, stmt)
+		if err != nil {
+			return fmt.Errorf("create schema: %w", err)
+		}
 	}

 	_, err = tx.ExecContext(ctx,
@@ -480,7 +523,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),

 	var n int64

-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -556,7 +599,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,

 	var n int64

-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -622,7 +665,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
 func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	var n int64

-	err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return 0, fmt.Errorf("count records: %w", err)
 	}
@@ -630,12 +673,10 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	return n, nil
 }

-// databaseCountsSQL counts the records and adds up their sizes, in one
-// pass over the table. A SUM over no rows is NULL, so on an empty table
-// the SUM falls back to 0.
+// databaseCountsSQL reads the number of records and the sizes of all
+// records added up from totals.
 const databaseCountsSQL = `
-SELECT COUNT(*), COALESCE(SUM(size), 0)
-FROM files
+SELECT records, size FROM totals
 `

 // databaseCounts is what info reads from the database: its schema
@@ -535,17 +535,12 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
 	}
 }

-// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
-// from the files_signature index alone, with nothing to sort: the
-// first record comes at once, so report's read phase counts each
-// record as it comes rather than waiting on SQLite for all of them.
-func TestSignatureRowsNeedNoSort(t *testing.T) {
-	t.Parallel()
+// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
+// PLAN for query.
+func queryPlan(t *testing.T, db *sql.DB, query string) []string {
+	t.Helper()

-	db := openTestDB(t)
-
-	rows, err := db.QueryContext(t.Context(),
-		"EXPLAIN QUERY PLAN "+signatureRowsSQL)
+	rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -573,12 +568,164 @@ func TestSignatureRowsNeedNoSort(t *testing.T) {
 		t.Fatal(err)
 	}

+	return plan
+}
+
+// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
+// from the files_signature index alone, with nothing to sort: the
+// first record comes at once, so report's read phase counts each
+// record as it comes rather than waiting on SQLite for all of them.
+func TestSignatureRowsNeedNoSort(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	plan := queryPlan(t, db, signatureRowsSQL)
+
 	want := []string{"SCAN files USING COVERING INDEX files_signature"}
 	if !slices.Equal(plan, want) {
 		t.Errorf("query plan = %q, want %q", plan, want)
 	}
 }

+// TestCountsReadOnlyTotals checks that SQLite answers each query that
+// counts the records from the one row of totals. A plan that reads
+// files or its index reads every record.
+func TestCountsReadOnlyTotals(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	for _, query := range []string{recordCountSQL, databaseCountsSQL} {
+		plan := queryPlan(t, db, query)
+
+		want := []string{"SCAN totals"}
+		if !slices.Equal(plan, want) {
+			t.Errorf("query plan of %q = %q, want %q", query, plan, want)
+		}
+	}
+}
+
+// TestCountsComeFromTotals sets the row of totals to numbers that files
+// does not hold, then checks that scan's load phase, the read phases of
+// trees and report, and info each report the numbers from totals.
+func TestCountsComeFromTotals(t *testing.T) {
+	t.Parallel()
+
+	// files is empty, so a count over files gives 0 records of 0 bytes.
+	const records, size = 7, 9000
+
+	db := openTestDB(t)
+
+	_, err := db.ExecContext(t.Context(),
+		"UPDATE totals SET records = ?, size = ?", records, size)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	scanTotal, err := countRecords(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var treesTotal, reportTotal int64
+
+	err = loadFileRows(t.Context(), db,
+		func(n int64) { treesTotal = n }, func(scanRec) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = loadSignatureRows(t.Context(), db,
+		func(n int64) { reportTotal = n },
+		func(scanRec) error { return nil })
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	info, err := readDatabaseCounts(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for name, got := range map[string]int64{
+		"scan's load":   scanTotal,
+		"trees' read":   treesTotal,
+		"report's read": reportTotal,
+		"info":          info.records,
+	} {
+		if got != records {
+			t.Errorf("%s: %d records, want %d from totals",
+				name, got, records)
+		}
+	}
+
+	if info.size != size {
+		t.Errorf("info: size %d, want %d from totals", info.size, size)
+	}
+}
+
+// assertTotalsMatchFiles fails unless the row of totals holds the
+// number of records and the sizes of all records added up, as counted
+// over files.
+func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
+	t.Helper()
+
+	got, err := readDatabaseCounts(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var records, size int64
+
+	err = db.QueryRowContext(t.Context(),
+		"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
+		Scan(&records, &size)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got.records != records || got.size != size {
+		t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
+			got.records, got.size, records, size)
+	}
+}
+
+// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
+// change size, and rescans it after two files are removed. After each
+// scan the row of totals must agree with files.
+func TestTotalsMatchFiles(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	syncTree(t, db, dir)
+	assertTotalsMatchFiles(t, db)
+
+	writeFile(t, dir, "a/one.bin", pattern(10, 2500))
+	writeFile(t, dir, "t1/f1", pattern(30, 1000))
+
+	if st := syncTree(t, db, dir); st.updated != 2 {
+		t.Fatalf("rescan stats = %+v, want 2 updated", st)
+	}
+
+	assertTotalsMatchFiles(t, db)
+
+	for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
+		err := os.Remove(filepath.Join(dir, name))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if st := syncTree(t, db, dir); st.removed != 2 {
+		t.Fatalf("rescan stats = %+v, want 2 removed", st)
+	}
+
+	assertTotalsMatchFiles(t, db)
+}
+
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()

@@ -17,9 +17,7 @@ import (
 func TestDefaultExcludes(t *testing.T) {
 	t.Parallel()

-	// README "Rules for the walk": the Linux and macOS lists, and none
-	// on any other system. The Windows list exists in the code only: the
-	// binary does not build for Windows, so the README leaves it out.
+	// README "Rules for the walk": each system's list.
 	cases := map[string][]string{
 		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
 		goosDarwin: {
@@ -26,11 +26,11 @@ func infoPhase(name string) string {
 // it. Any database problem, including a missing database, is fatal.
 //
 // The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and count (count the records,
-// with one query). Each display ends before the next phase starts, and
-// before an error is returned to be printed. The details are written
-// once the last display has ended, so no display is drawn while they
-// are written.
+// own on stderr: open (open the database) and count (read the number of
+// records and their total size from the totals table). Each display
+// ends before the next phase starts, and before an error is returned
+// to be printed. The details are written once the last display has
+// ended, so no display is drawn while they are written.
 func runInfo(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {

 	defer func() { _ = db.Close() }()

-	// One query counts everything, so the phase never has a total.
+	// One row holds both counts, so the phase never has a total.
 	prog = newProgress(progressParams{
 		label: infoPhase("count"), total: -1,
 		waitingFor: countingRecords,
@@ -180,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
 }

 // brokenDatabase writes a database that opens cleanly and passes the
-// schema-version check but has no files table, so the first query
-// fails with the database already open: a fatal error on a path that
-// owns an open database. It closes the database the way scan does.
+// schema-version check but has no tables, so the first query fails
+// with the database already open: a fatal error on a path that owns an
+// open database. It closes the database the way scan does.
 func brokenDatabase(t *testing.T) string {
 	t.Helper()

@@ -265,8 +265,9 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
 			assertFatalOutput(t, stderr.String(), stdout())

 			// Proof that the failure happened after the open: only a
-			// query against the opened database can report this.
-			if !strings.Contains(stderr.String(), "no such table: files") {
+			// query against the opened database can report this. Each
+			// subcommand's first query reads totals.
+			if !strings.Contains(stderr.String(), "no such table: totals") {
 				t.Errorf("stderr = %q, want the failure to come from a "+
 					"query on the open database", stderr.String())
 			}
@@ -393,7 +394,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
 }

 func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
-	// README §Settings: the path names the file exactly. With link a
+	// README §Database: the path names the file exactly. With link a
 	// symlink to the directory real/sub, link/../db.sqlite names
 	// real/db.sqlite, not db.sqlite beside link. Every subcommand must
 	// use that file, relative or absolute, and its first line must name
@@ -648,7 +649,7 @@ func TestRunUsageErrors(t *testing.T) {
 }

 func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
-	// README §Settings: --workers below 1 is a usage error reported in
+	// README §scan mode: --workers below 1 is a usage error reported in
 	// one line on stderr, before the scan opens the database.
 	for _, workers := range []string{"0", "-1"} {
 		t.Run(workers, func(t *testing.T) {
@@ -686,8 +687,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
 func TestRunHelp(t *testing.T) {
 	t.Parallel()

-	// README §Settings: help goes to stderr, exits 0, and leaves stdout
-	// empty.
+	// README §Subcommands: help goes to stderr, exits 0, and leaves
+	// stdout empty.
 	cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}

 	for _, args := range cases {
@@ -713,8 +714,8 @@ func TestRunHelp(t *testing.T) {
 func TestRunVersion(t *testing.T) {
 	t.Parallel()

-	// README §Settings: the version is one line on stdout, with nothing
-	// on stderr, and exits 0.
+	// README §Subcommands: the version is one line on stdout, with
+	// nothing on stderr, and exits 0.
 	info, _ := debug.ReadBuildInfo()

 	for _, arg := range []string{"--version", "-v"} {
@@ -773,8 +774,8 @@ func TestBuildVersion(t *testing.T) {
 func TestRunVersionWriteFailureIsFatal(t *testing.T) {
 	t.Parallel()

-	// README §Error handling: a failed write to stdout is a fatal error,
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// README §Error handling: a stdout write failure exits 1, reported
+	// in one line on stderr.
 	var stderr bytes.Buffer

 	code := run([]string{"--version"}, failingWriter{}, &stderr)
@@ -1234,7 +1235,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 }

 func TestRunDatabasePathUsedAsGiven(t *testing.T) {
-	// README §Settings: the path names the database file exactly. In
+	// README §Database: the path names the database file exactly. In
 	// SQLite's connection string an unescaped ? or # would end the file
 	// name and % would start an escape, and a path starting with //
 	// could be read as a host name. %25 is a valid escape, so unescaped
@@ -1279,8 +1280,9 @@ func holdScanLock(t *testing.T, path string) {
 }

 func TestRunSecondScanFails(t *testing.T) {
-	// README §Database: a second scan that finds the lock held fails
-	// at once, naming the lock file, without opening the database.
+	// README §Database: while one scan holds the lock, a second scan
+	// fails at once, naming the lock file, without opening the
+	// database.
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)

@@ -1339,7 +1341,7 @@ func assertSecondScanFails(t *testing.T, path string) {
 }

 func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
-	// README §Database: a path through a symlink names the same lock
+	// README §Database: every path to one database names the same lock
 	// file, so while a scan holds it through one path, a scan through
 	// another fails, naming the lock file beside the database's real
 	// path.
@@ -1444,8 +1446,8 @@ func TestRunReportsDuringScan(t *testing.T) {
 }

 func TestRunStdoutClosedIsFatal(t *testing.T) {
-	// README §Error handling: a failed write to stdout is a fatal error,
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// README §Error handling: a stdout write failure exits 1, reported
+	// in one line on stderr.
 	for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
 		t.Run(name, func(t *testing.T) {
 			t.Setenv(databaseEnv, testDBPath(t))
@@ -455,8 +455,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
 // included, so its display counts against the number of records.
 func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 	// Indexing tens of millions of records takes real time; without a
-	// display the scan looks hung before the walk begins. Counting them
-	// first gives the display its total.
+	// display the scan looks hung before the walk begins. The number of
+	// records, read first, gives the display its total.
 	prog := newProgress(progressParams{
 		label: scanPhase("load"), unit: scanUnit, total: -1,
 		waitingFor: countingRecords, stalledOn: "db",
@@ -1064,9 +1064,8 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
 	}
 }

-// buildSmokeTree recreates the tree of the test under README
-// "Definition of done" with deterministic content and returns the tree
-// root.
+// buildSmokeTree recreates the README smoke-test filesystem layout
+// with deterministic content and returns the tree root.
 func buildSmokeTree(t *testing.T) string {
 	t.Helper()

@@ -15,9 +15,8 @@ import (
 const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
 	"remain unknown"

-// smokeTreeRecs mirrors the tree of the test under README "Definition
-// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
-// by one filename.
+// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
+// /d/t2 are identical, /d/t3 differs from them only by one filename.
 func smokeTreeRecs() []scanRec {
 	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
 	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")