diff --git a/README.md b/README.md index 6a4b5c3..f702a06 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,20 @@ All four subcommands operate on a single SQLite database file: content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples ) WITHOUT ROWID; CREATE INDEX files_signature ON files (size, head, tail, content); + CREATE TABLE totals ( + records INTEGER NOT NULL, -- number of records in files + size INTEGER NOT NULL -- sizes of all records in files added up + ); + INSERT INTO totals (records, size) VALUES (0, 0); + CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN + UPDATE totals SET records = records + 1, size = size + NEW.size; + END; + CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN + UPDATE totals SET records = records - 1, size = size - OLD.size; + END; + CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN + UPDATE totals SET size = size + NEW.size - OLD.size; + END; ``` Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed @@ -406,6 +420,13 @@ All four subcommands operate on a single SQLite database file: `files_signature` index lets `report` read the records in signature order without sorting the table. + `totals` holds one row: the number of records in `files` and the sum of + their sizes. Its three triggers keep that row current. Each runs in the + transaction of the change to `files` that fires it, so the row and the + records always agree. `info` and the progress displays read the number of + records from this row. Counting them in `files` would read every record, + which takes minutes on a database of tens of millions. + ### Duplicate detection Two files are duplicates when they have the same size and the same three hashes. @@ -688,14 +709,14 @@ Processing: database state. `report` runs in two phases, each with its own progress display on stderr (see -"Progress"): `open` opens the database; `read` counts the records, then reads -them in signature order, largest size first. That is the order of the -`files_signature` index read backwards, so SQLite sorts nothing and returns the -first record at once. The records sharing a signature come together, and so do -the records of one size, so `report` holds only the duplicate groups of the size -it is reading, and writes them, ordered by `first` path, once it reaches a -smaller size. Its memory holds at most the duplicate paths of one size at a -time, never every record. +"Progress"): `open` opens the database; `read` gets the number of records from +`totals`, then reads the records in signature order, largest size first. That is +the order of the `files_signature` index read backwards, so SQLite sorts nothing +and returns the first record at once. The records sharing a signature come +together, and so do the records of one size, so `report` holds only the +duplicate groups of the size it is reading, and writes them, ordered by `first` +path, once it reaches a smaller size. Its memory holds at most the duplicate +paths of one size at a time, never every record. #### Report output format @@ -769,10 +790,10 @@ Processing: ascending. Output must be fully deterministic for a given input. `trees` runs in four phases, each with its own progress display on stderr (see -"Progress"): `open` opens the database; `read` counts the records, then reads -them in path order, building the hierarchy and computing the digests as it goes; -`group` groups the directories by digest and picks the maximal groups; `write` -writes the groups to stdout. +"Progress"): `open` opens the database; `read` gets the number of records from +`totals`, then reads the records in path order, building the hierarchy and +computing the digests as it goes; `group` groups the directories by digest and +picks the maximal groups; `write` writes the groups to stdout. #### Trees output format @@ -823,11 +844,10 @@ size of recorded files 150000000000000 (136.4 TiB) of all records, in bytes, then in human units. `info` runs in two phases, each with its own progress display on stderr (see -"Progress"): `open` opens the database; `count` reads the schema version and -counts the records with one query, a single pass over the table, both in one -read-only transaction, so the numbers agree with each other while a scan is -committing. `info` writes its lines once the `count` display has ended, so on a -terminal they come after the displays. +"Progress"): `open` opens the database; `count` reads the schema version and the +row of `totals` (see "Database") in one read-only transaction, so the numbers +agree with each other while a scan is committing. `info` writes its lines once +the `count` display has ended, so on a terminal they come after the displays. ### Progress @@ -843,8 +863,9 @@ progress, its elapsed time and its time remaining. A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`, `[4/4] update`): -- `load` counts the records first, then reads them against that count. While the - count runs, the display says it is counting the records. +- `load` gets the number of records from `totals` first, then reads the records + against that number. Until it has the number, the display says it is counting + the records. - `walk` has no exact total. Its estimate is the number of records the last scan left under the operands, known once `load` is done and shown with a `~`. The time remaining is unknown once the walk passes the estimate, and when there @@ -857,8 +878,9 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`, - `open` has no total. While it opens the database, waiting on a busy one included, the display says it is opening the database. -- `read` counts the records first, then reads them against that count. While the - count runs, the display says it is counting the records. +- `read` gets the number of records from `totals` first, then reads the records + against that number. Until it has the number, the display says it is counting + the records. - `group` has the directories `read` found as its total, and `write` the duplicate-tree groups. @@ -866,17 +888,16 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`, - `open` has no total. While it opens the database, waiting on a busy one included, the display says it is opening the database. -- `read` counts the records first, then reads them against that count, writing - the rows to stdout as it goes. While the count runs, the display says it is - counting the records. +- `read` gets the number of records from `totals` first, then reads the records + against that number, writing the rows to stdout as it goes. Until it has the + number, the display says it is counting the records. `info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"): - `open` has no total. While it opens the database, waiting on a busy one included, the display says it is opening the database. -- `count` has no total either, because one query counts everything: the display - says it is counting the records until the query returns, and its time - remaining is unknown. +- `count` has no total either, because it reads one row: the display says it is + counting the records until it has the row, and its time remaining is unknown. Each display is one line holding: diff --git a/TODO.md b/TODO.md index def2438..59631a1 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ # Completed Steps +- the database keeps the number of records and the sum of their sizes in the one + row of a `totals` table, which three triggers on `files` keep current. `info` + reads that row, and so do `scan`'s `load` phase and the `read` phases of + `report` and `trees` for their totals; no count reads every record any more. A + database made before this has no `totals` table and must be removed and + rescanned (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/146) + - an operand at or inside a directory named `.zfs` is never scanned, whether given alone or under another operand. `scan` warns about it, counts it as skipped and does not walk it (2026-10-26, diff --git a/db.go b/db.go index c7835b4..8ae7c0f 100644 --- a/db.go +++ b/db.go @@ -69,6 +69,49 @@ const createIndexSQL = ` CREATE INDEX files_signature ON files (size, head, tail, content) ` +// createTotalsSQL creates the totals table, which holds one row: the +// number of records and the sizes of all records added up. Counting +// them over files would read every record. The three triggers below +// keep the row current, in the transaction of each change to files. +const createTotalsSQL = ` +CREATE TABLE totals ( + records INTEGER NOT NULL, + size INTEGER NOT NULL +) +` + +// insertTotalsSQL adds the one row of totals, for an empty files table. +const insertTotalsSQL = ` +INSERT INTO totals (records, size) VALUES (0, 0) +` + +// totalsAfterInsertSQL counts each inserted record into totals. +const totalsAfterInsertSQL = ` +CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN + UPDATE totals SET records = records + 1, size = size + NEW.size; +END +` + +// totalsAfterDeleteSQL takes each deleted record out of totals. +const totalsAfterDeleteSQL = ` +CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN + UPDATE totals SET records = records - 1, size = size - OLD.size; +END +` + +// totalsAfterUpdateSQL replaces a record's old size in totals with its +// new one. scan's upsert of an existing record is such an update. +const totalsAfterUpdateSQL = ` +CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN + UPDATE totals SET size = size + NEW.size - OLD.size; +END +` + +// recordCountSQL reads the number of records from totals. +const recordCountSQL = ` +SELECT records FROM totals +` + // upsertSQL inserts one file record, replacing any existing record for // the same path. const upsertSQL = ` @@ -426,14 +469,14 @@ func createSchema(ctx context.Context, db *sql.DB) error { defer func() { _ = tx.Rollback() }() - _, err = tx.ExecContext(ctx, createTableSQL) - if err != nil { - return fmt.Errorf("create schema: %w", err) - } - - _, err = tx.ExecContext(ctx, createIndexSQL) - if err != nil { - return fmt.Errorf("create schema: %w", err) + for _, stmt := range []string{ + createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL, + totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL, + } { + _, err = tx.ExecContext(ctx, stmt) + if err != nil { + return fmt.Errorf("create schema: %w", err) + } } _, err = tx.ExecContext(ctx, @@ -480,7 +523,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64), var n int64 - err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n) + err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n) if err != nil { return fmt.Errorf("count records: %w", err) } @@ -556,7 +599,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB, var n int64 - err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n) + err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n) if err != nil { return fmt.Errorf("count records: %w", err) } @@ -622,7 +665,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error { func countRecords(ctx context.Context, db *sql.DB) (int64, error) { var n int64 - err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n) + err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n) if err != nil { return 0, fmt.Errorf("count records: %w", err) } @@ -630,12 +673,10 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) { return n, nil } -// databaseCountsSQL counts the records and adds up their sizes, in one -// pass over the table. A SUM over no rows is NULL, so on an empty table -// the SUM falls back to 0. +// databaseCountsSQL reads the number of records and the sizes of all +// records added up from totals. const databaseCountsSQL = ` -SELECT COUNT(*), COALESCE(SUM(size), 0) -FROM files +SELECT records, size FROM totals ` // databaseCounts is what info reads from the database: its schema diff --git a/db_test.go b/db_test.go index 9268048..0f0aa64 100644 --- a/db_test.go +++ b/db_test.go @@ -535,17 +535,12 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) { } } -// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL -// from the files_signature index alone, with nothing to sort: the -// first record comes at once, so report's read phase counts each -// record as it comes rather than waiting on SQLite for all of them. -func TestSignatureRowsNeedNoSort(t *testing.T) { - t.Parallel() +// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY +// PLAN for query. +func queryPlan(t *testing.T, db *sql.DB, query string) []string { + t.Helper() - db := openTestDB(t) - - rows, err := db.QueryContext(t.Context(), - "EXPLAIN QUERY PLAN "+signatureRowsSQL) + rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query) if err != nil { t.Fatal(err) } @@ -573,12 +568,105 @@ func TestSignatureRowsNeedNoSort(t *testing.T) { t.Fatal(err) } + return plan +} + +// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL +// from the files_signature index alone, with nothing to sort: the +// first record comes at once, so report's read phase counts each +// record as it comes rather than waiting on SQLite for all of them. +func TestSignatureRowsNeedNoSort(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + plan := queryPlan(t, db, signatureRowsSQL) + want := []string{"SCAN files USING COVERING INDEX files_signature"} if !slices.Equal(plan, want) { t.Errorf("query plan = %q, want %q", plan, want) } } +// TestCountsReadOnlyTotals checks that SQLite answers each query that +// counts the records from the one row of totals. A plan that reads +// files or its index reads every record. +func TestCountsReadOnlyTotals(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + for _, query := range []string{recordCountSQL, databaseCountsSQL} { + plan := queryPlan(t, db, query) + + want := []string{"SCAN totals"} + if !slices.Equal(plan, want) { + t.Errorf("query plan of %q = %q, want %q", query, plan, want) + } + } +} + +// assertTotalsMatchFiles fails unless the row of totals holds the +// number of records and the sizes of all records added up, as counted +// over files. +func assertTotalsMatchFiles(t *testing.T, db *sql.DB) { + t.Helper() + + got, err := readDatabaseCounts(t.Context(), db) + if err != nil { + t.Fatal(err) + } + + var records, size int64 + + err = db.QueryRowContext(t.Context(), + "SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files"). + Scan(&records, &size) + if err != nil { + t.Fatal(err) + } + + if got.records != records || got.size != size { + t.Errorf("totals = %d records of %d bytes, files holds %d of %d", + got.records, got.size, records, size) + } +} + +// TestTotalsMatchFiles scans the smoke tree, rescans it after two files +// change size, and rescans it after two files are removed. After each +// scan the row of totals must agree with files. +func TestTotalsMatchFiles(t *testing.T) { + t.Parallel() + + dir := buildSmokeTree(t) + db := openTestDB(t) + + syncTree(t, db, dir) + assertTotalsMatchFiles(t, db) + + writeFile(t, dir, "a/one.bin", pattern(10, 2500)) + writeFile(t, dir, "t1/f1", pattern(30, 1000)) + + if st := syncTree(t, db, dir); st.updated != 2 { + t.Fatalf("rescan stats = %+v, want 2 updated", st) + } + + assertTotalsMatchFiles(t, db) + + for _, name := range []string{"a/unique.bin", "t2/sub/f2"} { + err := os.Remove(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + } + + if st := syncTree(t, db, dir); st.removed != 2 { + t.Fatalf("rescan stats = %+v, want 2 removed", st) + } + + assertTotalsMatchFiles(t, db) +} + func TestApplyChangesRoundTrip(t *testing.T) { t.Parallel() diff --git a/info.go b/info.go index 3ec91ff..c30cac8 100644 --- a/info.go +++ b/info.go @@ -26,11 +26,11 @@ func infoPhase(name string) string { // it. Any database problem, including a missing database, is fatal. // // The work runs in two phases, each under a progress display of its -// own on stderr: open (open the database) and count (count the records, -// with one query). Each display ends before the next phase starts, and -// before an error is returned to be printed. The details are written -// once the last display has ended, so no display is drawn while they -// are written. +// own on stderr: open (open the database) and count (read the number of +// records and their total size from the totals table). Each display +// ends before the next phase starts, and before an error is returned +// to be printed. The details are written once the last display has +// ended, so no display is drawn while they are written. func runInfo(ctx context.Context, stdout io.Writer) error { dbPath, source, err := databasePath() if err != nil { @@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error { defer func() { _ = db.Close() }() - // One query counts everything, so the phase never has a total. + // One row holds both counts, so the phase never has a total. prog = newProgress(progressParams{ label: infoPhase("count"), total: -1, waitingFor: countingRecords, diff --git a/main_test.go b/main_test.go index cca359a..cdb385c 100644 --- a/main_test.go +++ b/main_test.go @@ -180,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string { } // brokenDatabase writes a database that opens cleanly and passes the -// schema-version check but has no files table, so the first query -// fails with the database already open: a fatal error on a path that -// owns an open database. It closes the database the way scan does. +// schema-version check but has no tables, so the first query fails +// with the database already open: a fatal error on a path that owns an +// open database. It closes the database the way scan does. func brokenDatabase(t *testing.T) string { t.Helper() @@ -265,8 +265,9 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) { assertFatalOutput(t, stderr.String(), stdout()) // Proof that the failure happened after the open: only a - // query against the opened database can report this. - if !strings.Contains(stderr.String(), "no such table: files") { + // query against the opened database can report this. Each + // subcommand's first query reads totals. + if !strings.Contains(stderr.String(), "no such table: totals") { t.Errorf("stderr = %q, want the failure to come from a "+ "query on the open database", stderr.String()) } diff --git a/scan.go b/scan.go index a16dd84..ac49c7e 100644 --- a/scan.go +++ b/scan.go @@ -455,8 +455,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string { // included, so its display counts against the number of records. func (s *scanState) loadIndex(ctx context.Context, roots []string) error { // Indexing tens of millions of records takes real time; without a - // display the scan looks hung before the walk begins. Counting them - // first gives the display its total. + // display the scan looks hung before the walk begins. The number of + // records, read first, gives the display its total. prog := newProgress(progressParams{ label: scanPhase("load"), unit: scanUnit, total: -1, waitingFor: countingRecords, stalledOn: "db",