Keep the record count and total size in a totals table (closes #146)

info counted the records and added up their sizes with a query that
reads every record, which takes minutes on a large database. The
schema now has a totals table of one row, kept current by three
triggers on files, in the transaction of each insert, delete and size
update. info, scan's load phase and the read phases of report and
trees read that row instead of counting over files. scan writes files
with the same statements as before. Each of those writes now fires a
trigger and is slower, so scan keeps SQLite's temporary data in
memory, which wins back part of that time.

Model: opus-5-5

db.go

@@ -69,6 +69,49 @@ const createIndexSQL = `
 CREATE INDEX files_signature ON files (size, head, tail, content)
 `
 
+// createTotalsSQL creates the totals table, which holds one row: the
+// number of records and the sizes of all records added up. Counting
+// them over files would read every record. The three triggers below
+// keep the row current, in the transaction of each change to files.
+const createTotalsSQL = `
+CREATE TABLE totals (
+    records INTEGER NOT NULL,
+    size    INTEGER NOT NULL
+)
+`
+
+// insertTotalsSQL adds the one row of totals, for an empty files table.
+const insertTotalsSQL = `
+INSERT INTO totals (records, size) VALUES (0, 0)
+`
+
+// totalsAfterInsertSQL counts each inserted record into totals.
+const totalsAfterInsertSQL = `
+CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
+    UPDATE totals SET records = records + 1, size = size + NEW.size;
+END
+`
+
+// totalsAfterDeleteSQL takes each deleted record out of totals.
+const totalsAfterDeleteSQL = `
+CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
+    UPDATE totals SET records = records - 1, size = size - OLD.size;
+END
+`
+
+// totalsAfterUpdateSQL replaces a record's old size in totals with its
+// new one. scan's upsert of an existing record is such an update.
+const totalsAfterUpdateSQL = `
+CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
+    UPDATE totals SET size = size + NEW.size - OLD.size;
+END
+`
+
+// recordCountSQL reads the number of records from totals.
+const recordCountSQL = `
+SELECT records FROM totals
+`
+
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
@@ -150,9 +193,14 @@ func createDatabaseDirectory(path string) error {
 // scanParams are the connection parameters for scan: read-write, with
 // WAL journaling and a busy timeout, so a report can run while a cron
 // scan is in progress. closeScanDatabase leaves WAL mode again.
+//
+// Each write to files fires a trigger, so SQLite keeps a statement
+// journal for it. temp_store(MEMORY) keeps that journal in memory,
+// which costs far less than a temporary file.
 const scanParams = "_pragma=busy_timeout(10000)" +
 	"&_pragma=journal_mode(WAL)" +
-	"&_pragma=synchronous(NORMAL)"
+	"&_pragma=synchronous(NORMAL)" +
+	"&_pragma=temp_store(MEMORY)"
 
 // reportParams are the connection parameters for report, trees and
 // info: read-only, with the same busy timeout. They set no journal
@@ -426,14 +474,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
 
 	defer func() { _ = tx.Rollback() }()
 
-	_, err = tx.ExecContext(ctx, createTableSQL)
-	if err != nil {
-		return fmt.Errorf("create schema: %w", err)
-	}
-
-	_, err = tx.ExecContext(ctx, createIndexSQL)
-	if err != nil {
-		return fmt.Errorf("create schema: %w", err)
+	for _, stmt := range []string{
+		createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
+		totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
+	} {
+		_, err = tx.ExecContext(ctx, stmt)
+		if err != nil {
+			return fmt.Errorf("create schema: %w", err)
+		}
 	}
 
 	_, err = tx.ExecContext(ctx,
@@ -480,7 +528,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 
 	var n int64
 
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -556,7 +604,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 
 	var n int64
 
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -622,7 +670,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
 func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	var n int64
 
-	err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return 0, fmt.Errorf("count records: %w", err)
 	}
@@ -630,12 +678,10 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	return n, nil
 }
 
-// databaseCountsSQL counts the records and adds up their sizes, in one
-// pass over the table. A SUM over no rows is NULL, so on an empty table
-// the SUM falls back to 0.
+// databaseCountsSQL reads the number of records and the sizes of all
+// records added up from totals.
 const databaseCountsSQL = `
-SELECT COUNT(*), COALESCE(SUM(size), 0)
-FROM files
+SELECT records, size FROM totals
 `
 
 // databaseCounts is what info reads from the database: its schema