Record each file's size on disk and print it in trees (closes #147)

The files table gets a disk_size column: the file's allocated blocks,
st_blocks × 512, from the lstat the walk already does. scan writes it
with every record it writes. When a rescan finds a file with its size
and mtime unchanged but a different disk_size, it updates disk_size
alone, without reading the file. trees adds disk_size up per directory
and prints it as a column after size. On a compressed or sparse file it
is less than size, so two copies of one tree can differ; each row shows
the dupe copy's.

Judgement calls: a row's disk_size is the dupe copy's, not the first's.
A file whose record gets only a new disk_size counts as unchanged.

Model: opus-5-5

TODO.md

@@ -29,13 +29,20 @@
 
 # Completed Steps
 
-- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
-  it. It leads with what the program is and how to use it. A reference section
-  covers every setting and every subcommand. Development and testing come last.
-  Every statement was checked against the code on `next`. Five comments in
-  `main_test.go` cite "Settings", where their rules now live. One comment each
-  in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
-  now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
+- each record holds `disk_size`, the bytes the file takes on disk: its allocated
+  blocks, `st_blocks` × 512, from the lstat the walk already does. `trees`
+  prints a `disk_size` column after `size`, the sum over the `dupe` copy's
+  files; on a compressed or sparse file it differs from `size`. A rescan updates
+  the `disk_size` of an unchanged file whose size on disk changed, without
+  reading the file (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/147)
+
+- the database keeps the number of records and the sum of their sizes in the one
+  row of a `totals` table, which three triggers on `files` keep current. `info`
+  reads that row, and so do `scan`'s `load` phase and the `read` phases of
+  `report` and `trees` for their totals; no count reads every record any more.
+  The triggers make each write to `files` slower. `scan` keeps SQLite's
+  temporary data in memory, which wins back part of that time (2026-10-27,
+  https://git.eeqj.de/sneak/sfdupes/issues/146)
 
 - an operand at or inside a directory named `.zfs` is never scanned, whether
   given alone or under another operand. `scan` warns about it, counts it as

db.go

@@ -47,14 +47,15 @@ const dbFilePerm = 0o644
 const lockFilePerm = 0o600
 
 // createTableSQL is the schema applied to a fresh database. Paths are
-// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
-// holds whole Unix seconds and mtime_nsec the nanoseconds within that
-// second. head, tail and content each hold the raw 32 bytes of a
-// SHA-256.
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+// disk_size holds the bytes the file takes on disk. mtime holds whole
+// Unix seconds and mtime_nsec the nanoseconds within that second. head,
+// tail and content each hold the raw 32 bytes of a SHA-256.
 const createTableSQL = `
 CREATE TABLE files (
     path       BLOB PRIMARY KEY,
     size       INTEGER NOT NULL,
+    disk_size  INTEGER NOT NULL,
     mtime      INTEGER NOT NULL,
     mtime_nsec INTEGER NOT NULL,
     head       BLOB NOT NULL,
@@ -69,14 +70,59 @@ const createIndexSQL = `
 CREATE INDEX files_signature ON files (size, head, tail, content)
 `
 
+// createTotalsSQL creates the totals table, which holds one row: the
+// number of records and the sizes of all records added up. Counting
+// them over files would read every record. The three triggers below
+// keep the row current, in the transaction of each change to files.
+const createTotalsSQL = `
+CREATE TABLE totals (
+    records INTEGER NOT NULL,
+    size    INTEGER NOT NULL
+)
+`
+
+// insertTotalsSQL adds the one row of totals, for an empty files table.
+const insertTotalsSQL = `
+INSERT INTO totals (records, size) VALUES (0, 0)
+`
+
+// totalsAfterInsertSQL counts each inserted record into totals.
+const totalsAfterInsertSQL = `
+CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
+    UPDATE totals SET records = records + 1, size = size + NEW.size;
+END
+`
+
+// totalsAfterDeleteSQL takes each deleted record out of totals.
+const totalsAfterDeleteSQL = `
+CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
+    UPDATE totals SET records = records - 1, size = size - OLD.size;
+END
+`
+
+// totalsAfterUpdateSQL replaces a record's old size in totals with its
+// new one. scan's upsert of an existing record is such an update.
+const totalsAfterUpdateSQL = `
+CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
+    UPDATE totals SET size = size + NEW.size - OLD.size;
+END
+`
+
+// recordCountSQL reads the number of records from totals.
+const recordCountSQL = `
+SELECT records FROM totals
+`
+
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
-INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
-VALUES (?, ?, ?, ?, ?, ?, ?)
+INSERT INTO files (
+    path, size, disk_size, mtime, mtime_nsec, head, tail, content
+)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT (path) DO UPDATE SET
-    size = excluded.size, mtime = excluded.mtime,
-    mtime_nsec = excluded.mtime_nsec,
+    size = excluded.size, disk_size = excluded.disk_size,
+    mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
     head = excluded.head, tail = excluded.tail,
     content = excluded.content
 `
@@ -150,9 +196,14 @@ func createDatabaseDirectory(path string) error {
 // scanParams are the connection parameters for scan: read-write, with
 // WAL journaling and a busy timeout, so a report can run while a cron
 // scan is in progress. closeScanDatabase leaves WAL mode again.
+//
+// Each write to files fires a trigger, so SQLite keeps a statement
+// journal for it. temp_store(MEMORY) keeps that journal in memory,
+// which costs far less than a temporary file.
 const scanParams = "_pragma=busy_timeout(10000)" +
 	"&_pragma=journal_mode(WAL)" +
-	"&_pragma=synchronous(NORMAL)"
+	"&_pragma=synchronous(NORMAL)" +
+	"&_pragma=temp_store(MEMORY)"
 
 // reportParams are the connection parameters for report, trees and
 // info: read-only, with the same busy timeout. They set no journal
@@ -426,14 +477,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
 
 	defer func() { _ = tx.Rollback() }()
 
-	_, err = tx.ExecContext(ctx, createTableSQL)
-	if err != nil {
-		return fmt.Errorf("create schema: %w", err)
-	}
-
-	_, err = tx.ExecContext(ctx, createIndexSQL)
-	if err != nil {
-		return fmt.Errorf("create schema: %w", err)
+	for _, stmt := range []string{
+		createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
+		totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
+	} {
+		_, err = tx.ExecContext(ctx, stmt)
+		if err != nil {
+			return fmt.Errorf("create schema: %w", err)
+		}
 	}
 
 	_, err = tx.ExecContext(ctx,
@@ -480,7 +531,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 
 	var n int64
 
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -488,8 +539,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 	total(n)
 
 	rows, err := tx.QueryContext(ctx,
-		"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
-			"FROM files ORDER BY path")
+		"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
+			"content FROM files ORDER BY path")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -503,7 +554,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 			r                         scanRec
 		)
 
-		err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
+		err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
+			&head, &tail, &content)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
@@ -538,10 +590,11 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
 `
 
 // loadSignatureRows gives total the number of records, then streams
-// every record's signature and path to fn, without its mtime, in the
-// order of signatureRowsSQL. The count and the records are read in one
-// transaction, so they agree while a scan is committing. An error from
-// fn stops the reading and is returned as it is.
+// every record's signature and path to fn, without its mtime or its
+// disk_size, in the order of signatureRowsSQL. The count and the
+// records are read in one transaction, so they agree while a scan is
+// committing. An error from fn stops the reading and is returned as it
+// is.
 func loadSignatureRows(ctx context.Context, db *sql.DB,
 	total func(n int64), fn func(r scanRec) error,
 ) error {
@@ -556,7 +609,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 
 	var n int64
 
-	err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return fmt.Errorf("count records: %w", err)
 	}
@@ -622,7 +675,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
 func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	var n int64
 
-	err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
+	err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
 	if err != nil {
 		return 0, fmt.Errorf("count records: %w", err)
 	}
@@ -630,12 +683,10 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	return n, nil
 }
 
-// databaseCountsSQL counts the records and adds up their sizes, in one
-// pass over the table. A SUM over no rows is NULL, so on an empty table
-// the SUM falls back to 0.
+// databaseCountsSQL reads the number of records and the sizes of all
+// records added up from totals.
 const databaseCountsSQL = `
-SELECT COUNT(*), COALESCE(SUM(size), 0)
-FROM files
+SELECT records, size FROM totals
 `
 
 // databaseCounts is what info reads from the database: its schema
@@ -678,15 +729,15 @@ func readDatabaseCounts(ctx context.Context,
 	return c, nil
 }
 
-// loadFileMeta streams every record's path, size and mtime to fn. Scan
-// change detection needs no hash values, and skipping the hash columns
-// keeps the scan's in-memory index small on multi-million-file
-// databases.
+// loadFileMeta streams every record's path, size, disk_size and mtime
+// to fn. Scan change detection needs no hash values, and skipping the
+// hash columns keeps the scan's in-memory index small on
+// multi-million-file databases.
 func loadFileMeta(ctx context.Context, db *sql.DB,
-	fn func(path string, size int64, mtime time.Time),
+	fn func(path string, m fileMeta),
 ) error {
 	rows, err := db.QueryContext(ctx,
-		"SELECT path, size, mtime, mtime_nsec FROM files")
+		"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -695,16 +746,19 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 
 	for rows.Next() {
 		var (
-			path            []byte
-			size, sec, nsec int64
+			path      []byte
+			m         fileMeta
+			sec, nsec int64
 		)
 
-		err = rows.Scan(&path, &size, &sec, &nsec)
+		err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
-		fn(string(path), size, time.Unix(sec, nsec))
+		m.mtime = time.Unix(sec, nsec)
+
+		fn(string(path), m)
 	}
 
 	err = rows.Err()
@@ -722,21 +776,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 // WAL small and let concurrent reports observe progress.
 const updateBatchSize = 10000
 
-// applyChanges writes one scan's database changes — upserts for new and
-// changed files, deletes for vanished ones — in batched transactions.
-// Progress is rendered on prog (one increment per change).
+// applyChanges writes upserts for new and changed files and deletes for
+// vanished ones, in batched transactions. Progress is rendered on prog
+// (one increment per change).
 func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
 	deletes []string, prog *progress,
 ) error {
 	for batch := range slices.Chunk(upserts, updateBatchSize) {
-		err := applyBatch(ctx, db, batch, nil, prog)
+		err := applyBatch(ctx, db, batch, nil, nil, prog)
 		if err != nil {
 			return err
 		}
 	}
 
 	for batch := range slices.Chunk(deletes, updateBatchSize) {
-		err := applyBatch(ctx, db, nil, batch, prog)
+		err := applyBatch(ctx, db, nil, nil, batch, prog)
 		if err != nil {
 			return err
 		}
@@ -745,10 +799,26 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
 	return nil
 }
 
-// applyBatch commits one batch of upserts and deletes in a single
-// transaction.
+// applyDiskSizes writes the disk_size of each of recs into its record,
+// in batched transactions, and leaves the rest of the record alone.
+// Progress is rendered on prog (one increment per record).
+func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
+	prog *progress,
+) error {
+	for batch := range slices.Chunk(recs, updateBatchSize) {
+		err := applyBatch(ctx, db, nil, batch, nil, prog)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// applyBatch commits one batch of upserts, disk_size updates and
+// deletes in a single transaction.
 func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
-	deletes []string, prog *progress,
+	diskSizes []fileRec, deletes []string, prog *progress,
 ) error {
 	tx, err := db.BeginTx(ctx, nil)
 	if err != nil {
@@ -762,6 +832,11 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
 		return err
 	}
 
+	err = execDiskSizes(ctx, tx, diskSizes, prog)
+	if err != nil {
+		return err
+	}
+
 	err = execDeletes(ctx, tx, deletes, prog)
 	if err != nil {
 		return err
@@ -787,7 +862,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 	defer func() { _ = st.Close() }()
 
 	for _, r := range upserts {
-		_, err = st.ExecContext(ctx, []byte(r.path), r.size,
+		_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
 			r.mtime.Unix(), r.mtime.Nanosecond(),
 			r.head[:], r.tail[:], r.content[:])
 		if err != nil {
@@ -800,6 +875,31 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 	return nil
 }
 
+// execDiskSizes sets the disk_size of the record of each of recs. It
+// does not touch size, so the totals triggers do not fire.
+func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
+	prog *progress,
+) error {
+	st, err := tx.PrepareContext(ctx,
+		"UPDATE files SET disk_size = ? WHERE path = ?")
+	if err != nil {
+		return fmt.Errorf("prepare disk_size update: %w", err)
+	}
+
+	defer func() { _ = st.Close() }()
+
+	for _, r := range recs {
+		_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
+		if err != nil {
+			return fmt.Errorf("update disk_size of %s: %w", r.path, err)
+		}
+
+		prog.increment()
+	}
+
+	return nil
+}
+
 // execDeletes removes the records for paths no longer present.
 func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
 	prog *progress,

db_test.go

@@ -535,17 +535,12 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
 	}
 }
 
-// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
-// from the files_signature index alone, with nothing to sort: the
-// first record comes at once, so report's read phase counts each
-// record as it comes rather than waiting on SQLite for all of them.
-func TestSignatureRowsNeedNoSort(t *testing.T) {
-	t.Parallel()
+// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
+// PLAN for query.
+func queryPlan(t *testing.T, db *sql.DB, query string) []string {
+	t.Helper()
 
-	db := openTestDB(t)
-
-	rows, err := db.QueryContext(t.Context(),
-		"EXPLAIN QUERY PLAN "+signatureRowsSQL)
+	rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -573,12 +568,164 @@ func TestSignatureRowsNeedNoSort(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	return plan
+}
+
+// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
+// from the files_signature index alone, with nothing to sort: the
+// first record comes at once, so report's read phase counts each
+// record as it comes rather than waiting on SQLite for all of them.
+func TestSignatureRowsNeedNoSort(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	plan := queryPlan(t, db, signatureRowsSQL)
+
 	want := []string{"SCAN files USING COVERING INDEX files_signature"}
 	if !slices.Equal(plan, want) {
 		t.Errorf("query plan = %q, want %q", plan, want)
 	}
 }
 
+// TestCountsReadOnlyTotals checks that SQLite answers the queries for
+// the number of records and their total size from the one row of
+// totals. A plan that reads files or its index reads every record.
+func TestCountsReadOnlyTotals(t *testing.T) {
+	t.Parallel()
+
+	db := openTestDB(t)
+
+	for _, query := range []string{recordCountSQL, databaseCountsSQL} {
+		plan := queryPlan(t, db, query)
+
+		want := []string{"SCAN totals"}
+		if !slices.Equal(plan, want) {
+			t.Errorf("query plan of %q = %q, want %q", query, plan, want)
+		}
+	}
+}
+
+// TestCountsComeFromTotals sets the row of totals to numbers that files
+// does not hold, then checks that scan's load phase, the read phases of
+// trees and report, and info each report the numbers from totals.
+func TestCountsComeFromTotals(t *testing.T) {
+	t.Parallel()
+
+	// files is empty, so a count over files gives 0 records of 0 bytes.
+	const records, size = 7, 9000
+
+	db := openTestDB(t)
+
+	_, err := db.ExecContext(t.Context(),
+		"UPDATE totals SET records = ?, size = ?", records, size)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	scanTotal, err := countRecords(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var treesTotal, reportTotal int64
+
+	err = loadFileRows(t.Context(), db,
+		func(n int64) { treesTotal = n }, func(scanRec) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = loadSignatureRows(t.Context(), db,
+		func(n int64) { reportTotal = n },
+		func(scanRec) error { return nil })
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	info, err := readDatabaseCounts(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for name, got := range map[string]int64{
+		"scan's load":   scanTotal,
+		"trees' read":   treesTotal,
+		"report's read": reportTotal,
+		"info":          info.records,
+	} {
+		if got != records {
+			t.Errorf("%s: %d records, want %d from totals",
+				name, got, records)
+		}
+	}
+
+	if info.size != size {
+		t.Errorf("info: size %d, want %d from totals", info.size, size)
+	}
+}
+
+// assertTotalsMatchFiles fails unless the row of totals holds the
+// number of records and the sizes of all records added up, as counted
+// over files.
+func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
+	t.Helper()
+
+	got, err := readDatabaseCounts(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var records, size int64
+
+	err = db.QueryRowContext(t.Context(),
+		"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
+		Scan(&records, &size)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got.records != records || got.size != size {
+		t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
+			got.records, got.size, records, size)
+	}
+}
+
+// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
+// change size, and rescans it after two files are removed. After each
+// scan the row of totals must agree with files.
+func TestTotalsMatchFiles(t *testing.T) {
+	t.Parallel()
+
+	dir := buildSmokeTree(t)
+	db := openTestDB(t)
+
+	syncTree(t, db, dir)
+	assertTotalsMatchFiles(t, db)
+
+	writeFile(t, dir, "a/one.bin", pattern(10, 2500))
+	writeFile(t, dir, "t1/f1", pattern(30, 1000))
+
+	if st := syncTree(t, db, dir); st.updated != 2 {
+		t.Fatalf("rescan stats = %+v, want 2 updated", st)
+	}
+
+	assertTotalsMatchFiles(t, db)
+
+	for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
+		err := os.Remove(filepath.Join(dir, name))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if st := syncTree(t, db, dir); st.removed != 2 {
+		t.Fatalf("rescan stats = %+v, want 2 removed", st)
+	}
+
+	assertTotalsMatchFiles(t, db)
+}
+
 func TestApplyChangesRoundTrip(t *testing.T) {
 	t.Parallel()
 

exclude_test.go

@@ -17,9 +17,7 @@ import (
 func TestDefaultExcludes(t *testing.T) {
 	t.Parallel()
 
-	// README "Rules for the walk": the Linux and macOS lists, and none
-	// on any other system. The Windows list exists in the code only: the
-	// binary does not build for Windows, so the README leaves it out.
+	// README "Rules for the walk": each system's list.
 	cases := map[string][]string{
 		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
 		goosDarwin: {

info.go

@@ -26,11 +26,11 @@ func infoPhase(name string) string {
 // it. Any database problem, including a missing database, is fatal.
 //
 // The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and count (count the records,
-// with one query). Each display ends before the next phase starts, and
-// before an error is returned to be printed. The details are written
-// once the last display has ended, so no display is drawn while they
-// are written.
+// own on stderr: open (open the database) and count (read the number of
+// records and their total size from the totals table). Each display
+// ends before the next phase starts, and before an error is returned
+// to be printed. The details are written once the last display has
+// ended, so no display is drawn while they are written.
 func runInfo(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {
 
 	defer func() { _ = db.Close() }()
 
-	// One query counts everything, so the phase never has a total.
+	// One row holds both counts, so the phase never has a total.
 	prog = newProgress(progressParams{
 		label: infoPhase("count"), total: -1,
 		waitingFor: countingRecords,

main_test.go

@@ -180,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
 }
 
 // brokenDatabase writes a database that opens cleanly and passes the
-// schema-version check but has no files table, so the first query
-// fails with the database already open: a fatal error on a path that
-// owns an open database. It closes the database the way scan does.
+// schema-version check but has no tables, so the first query fails
+// with the database already open: a fatal error on a path that owns an
+// open database. It closes the database the way scan does.
 func brokenDatabase(t *testing.T) string {
 	t.Helper()
 
@@ -265,8 +265,9 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
 			assertFatalOutput(t, stderr.String(), stdout())
 
 			// Proof that the failure happened after the open: only a
-			// query against the opened database can report this.
-			if !strings.Contains(stderr.String(), "no such table: files") {
+			// query against the opened database can report this. Each
+			// subcommand's first query reads totals.
+			if !strings.Contains(stderr.String(), "no such table: totals") {
 				t.Errorf("stderr = %q, want the failure to come from a "+
 					"query on the open database", stderr.String())
 			}
@@ -393,7 +394,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
 }
 
 func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
-	// README §Settings: the path names the file exactly. With link a
+	// README §Database: the path names the file exactly. With link a
 	// symlink to the directory real/sub, link/../db.sqlite names
 	// real/db.sqlite, not db.sqlite beside link. Every subcommand must
 	// use that file, relative or absolute, and its first line must name
@@ -648,7 +649,7 @@ func TestRunUsageErrors(t *testing.T) {
 }
 
 func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
-	// README §Settings: --workers below 1 is a usage error reported in
+	// README §scan mode: --workers below 1 is a usage error reported in
 	// one line on stderr, before the scan opens the database.
 	for _, workers := range []string{"0", "-1"} {
 		t.Run(workers, func(t *testing.T) {
@@ -686,8 +687,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
 func TestRunHelp(t *testing.T) {
 	t.Parallel()
 
-	// README §Settings: help goes to stderr, exits 0, and leaves stdout
-	// empty.
+	// README §Subcommands: help goes to stderr, exits 0, and leaves
+	// stdout empty.
 	cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
 
 	for _, args := range cases {
@@ -713,8 +714,8 @@ func TestRunHelp(t *testing.T) {
 func TestRunVersion(t *testing.T) {
 	t.Parallel()
 
-	// README §Settings: the version is one line on stdout, with nothing
-	// on stderr, and exits 0.
+	// README §Subcommands: the version is one line on stdout, with
+	// nothing on stderr, and exits 0.
 	info, _ := debug.ReadBuildInfo()
 
 	for _, arg := range []string{"--version", "-v"} {
@@ -773,8 +774,8 @@ func TestBuildVersion(t *testing.T) {
 func TestRunVersionWriteFailureIsFatal(t *testing.T) {
 	t.Parallel()
 
-	// README §Error handling: a failed write to stdout is a fatal error,
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// README §Error handling: a stdout write failure exits 1, reported
+	// in one line on stderr.
 	var stderr bytes.Buffer
 
 	code := run([]string{"--version"}, failingWriter{}, &stderr)
@@ -817,6 +818,23 @@ func scanFixture(t *testing.T) []string {
 	return dupes
 }
 
+// storedDiskSize returns the disk_size of the record for path in the
+// database at dbPath. A test of a trees run compares with this rather
+// than with a new lstat, which on ZFS can differ from the scan's (see
+// diskSizeOf).
+func storedDiskSize(t *testing.T, dbPath, path string) int64 {
+	t.Helper()
+
+	db, err := openReportDatabase(t.Context(), dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() { _ = db.Close() }()
+
+	return recordByPath(t, dbRecords(t, db), path).diskSize
+}
+
 // scanOK runs scan over operands, fails the test unless it exits 0 with
 // nothing on stdout, and returns everything it printed to stderr.
 func scanOK(t *testing.T, operands ...string) string {
@@ -1140,8 +1158,9 @@ func TestRunTreesSucceeds(t *testing.T) {
 
 	// The two directories holding the duplicate pair are duplicate
 	// trees of each other.
-	want := "first\tdupe\tfiles\tsize\n" +
-		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
+	want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
+		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
+		strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
 	if got := stdout.String(); got != want {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
@@ -1175,9 +1194,10 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
 	cases := map[string]string{
 		cmdReport: "first\tdupe\tsize\n" +
 			dupes[0] + "\t" + dupes[1] + "\t300\n",
-		cmdTrees: "first\tdupe\tfiles\tsize\n" +
+		cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
 			filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
-			"\t1\t300\n",
+			"\t1\t300\t" +
+			strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
 	}
 
 	for name, want := range cases {
@@ -1212,8 +1232,11 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 		t.Errorf("report stdout = %q, want %q", got, want)
 	}
 
-	want = "first\tdupe\tfiles\tsize\n" +
-		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
+	diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
+
+	want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
+		filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
+		strconv.FormatInt(diskSize, 10) + "\n"
 	if got := runStdout(t, cmdTrees); got != want {
 		t.Errorf("trees stdout = %q, want %q", got, want)
 	}
@@ -1234,7 +1257,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 }
 
 func TestRunDatabasePathUsedAsGiven(t *testing.T) {
-	// README §Settings: the path names the database file exactly. In
+	// README §Database: the path names the database file exactly. In
 	// SQLite's connection string an unescaped ? or # would end the file
 	// name and % would start an escape, and a path starting with //
 	// could be read as a host name. %25 is a valid escape, so unescaped
@@ -1279,8 +1302,9 @@ func holdScanLock(t *testing.T, path string) {
 }
 
 func TestRunSecondScanFails(t *testing.T) {
-	// README §Database: a second scan that finds the lock held fails
-	// at once, naming the lock file, without opening the database.
+	// README §Database: while one scan holds the lock, a second scan
+	// fails at once, naming the lock file, without opening the
+	// database.
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
 
@@ -1339,7 +1363,7 @@ func assertSecondScanFails(t *testing.T, path string) {
 }
 
 func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
-	// README §Database: a path through a symlink names the same lock
+	// README §Database: every path to one database names the same lock
 	// file, so while a scan holds it through one path, a scan through
 	// another fails, naming the lock file beside the database's real
 	// path.
@@ -1444,8 +1468,8 @@ func TestRunReportsDuringScan(t *testing.T) {
 }
 
 func TestRunStdoutClosedIsFatal(t *testing.T) {
-	// README §Error handling: a failed write to stdout is a fatal error,
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// README §Error handling: a stdout write failure exits 1, reported
+	// in one line on stderr.
 	for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
 		t.Run(name, func(t *testing.T) {
 			t.Setenv(databaseEnv, testDBPath(t))

progress.go

@@ -60,7 +60,7 @@ const fallbackWidth = 80
 const spinnerMargin = 5
 
 // countingRecords is what the display of a phase that reads the records
-// says while it counts them, before it has its total.
+// says until it has the number of records, its total.
 const countingRecords = "counting the records"
 
 // openPhase is the name of the phase in which report, trees and info

report.go

@@ -21,14 +21,16 @@ const minGroupSize = 2
 
 // scanRec is one file record from the database. The signature (size,
 // head, tail, content) is the duplicate key; mtime is informational
-// only and used by scan for change detection.
+// only and used by scan for change detection. diskSize is the bytes the
+// file takes on disk, which trees adds up.
 type scanRec struct {
-	size    int64
-	mtime   time.Time
-	head    [sha256.Size]byte
-	tail    [sha256.Size]byte
-	content [sha256.Size]byte
-	path    string
+	size     int64
+	diskSize int64
+	mtime    time.Time
+	head     [sha256.Size]byte
+	tail     [sha256.Size]byte
+	content  [sha256.Size]byte
+	path     string
 }
 
 // reportPhase names one phase of report with its place among all of
@@ -45,11 +47,11 @@ func reportPhase(name string) string {
 // problem, including a missing database, is fatal.
 //
 // The work runs in two phases, each under a progress display of its
-// own on stderr: open (open the database) and read (count the records,
-// then read them in signature order, writing each size's duplicate
-// groups to stdout once that size is read; see readReport). Each
-// display ends before the next phase starts, and before an error is
-// returned to be printed.
+// own on stderr: open (open the database) and read (read the number of
+// records from the totals table, then read the records in signature
+// order, writing each size's duplicate groups to stdout once that size
+// is read; see readReport). Each display ends before the next phase
+// starts, and before an error is returned to be printed.
 func runReport(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {

report_test.go

@@ -38,8 +38,8 @@ func awkwardPairRecs() []scanRec {
 	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 
 	return []scanRec{
-		{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
-		{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
+		{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"},
+		{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
 	}
 }
 

scan.go

@@ -52,23 +52,26 @@ const workQueueDepth = 1024
 // has already printed its line, so run prints nothing more.
 var errInterrupted = errors.New("scan interrupted")
 
-// fileRec carries one statted file between the scan phases. dev and
-// ino identify the underlying inode so hard-linked paths can share
-// one read; both are zero when the platform exposes no inode.
+// fileRec carries one statted file between the scan phases. diskSize is
+// the bytes the file takes on disk. dev and ino identify the underlying
+// inode so hard-linked paths can share one read; both are zero when the
+// platform exposes no inode.
 type fileRec struct {
-	path  string
-	size  int64
-	mtime time.Time
-	dev   uint64
-	ino   uint64
+	path     string
+	size     int64
+	diskSize int64
+	mtime    time.Time
+	dev      uint64
+	ino      uint64
 }
 
 // fileMeta is the in-memory index entry for one existing database
-// record: just enough for change detection. Hashes stay on disk; at
+// record: what the walk compares the file with. Hashes stay on disk; at
 // tens of millions of records they would dominate the scan's memory.
 type fileMeta struct {
-	size  int64
-	mtime time.Time
+	size     int64
+	diskSize int64
+	mtime    time.Time
 }
 
 // scanUnit is what every scan phase counts, as its progress display
@@ -339,11 +342,12 @@ type scanStats struct {
 // whatever remains after the walk and hash phases is deleted by the
 // update phase.
 type scanState struct {
-	db       *sql.DB
-	existing map[string]fileMeta
-	toHash   []fileRec // new and changed files: must be read
-	batch    []scanRec
-	st       scanStats
+	db        *sql.DB
+	existing  map[string]fileMeta
+	toHash    []fileRec // new and changed files: must be read
+	diskSizes []fileRec // unchanged files whose disk_size changed
+	batch     []scanRec
+	st        scanStats
 }
 
 // syncScan synchronizes the database with the filesystem under roots;
@@ -455,8 +459,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
 // included, so its display counts against the number of records.
 func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 	// Indexing tens of millions of records takes real time; without a
-	// display the scan looks hung before the walk begins. Counting them
-	// first gives the display its total.
+	// display the scan looks hung before the walk begins. The number of
+	// records, read first, gives the display its total.
 	prog := newProgress(progressParams{
 		label: scanPhase("load"), unit: scanUnit, total: -1,
 		waitingFor: countingRecords, stalledOn: "db",
@@ -472,24 +476,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 
 	s.existing = make(map[string]fileMeta)
 
-	return loadFileMeta(ctx, s.db,
-		func(path string, size int64, mtime time.Time) {
-			prog.increment()
+	return loadFileMeta(ctx, s.db, func(path string, m fileMeta) {
+		prog.increment()
 
-			if underAnyRoot(path, roots) {
-				s.existing[path] = fileMeta{size: size, mtime: mtime}
-			}
-		})
+		if underAnyRoot(path, roots) {
+			s.existing[path] = m
+		}
+	})
 }
 
 // walkPhase drains the walk and queues every new or changed file for
-// the hash phase. An unchanged file needs nothing more, and its record
-// counts as verified. A record at or below a path the walk left out,
-// because -x stopped at it, it is a default exclude, or it is not on a
-// persistent filesystem, counts as outside the roots: it is not
-// deleted. That does not hold below a root at or inside that path,
-// since that root's own walk verifies the records below it; nor is a
-// line printed for such a root.
+// the hash phase. An unchanged file is not read, and its record counts
+// as verified. When its disk_size differs from the record's, it is
+// queued for the update phase, which writes the new disk_size. A record
+// at or below a path the walk left out, because -x stopped at it, it is
+// a default exclude, or it is not on a persistent filesystem, counts as
+// outside the roots: it is not deleted. That does not hold below a root
+// at or inside that path, since that root's own walk verifies the
+// records below it; nor is a line printed for such a root.
 func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
 	// The records the last scan left under the roots estimate how many
 	// files the walk will find. A first scan has none, and no estimate.
@@ -547,6 +551,13 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
 		delete(s.existing, ev.rec.path)
 
 		s.st.unchanged++
+
+		// The space a file takes on disk can change while its size and
+		// mtime stay the same. ZFS, for one, counts a new file's blocks
+		// only seconds after the write.
+		if old.diskSize != ev.rec.diskSize {
+			s.diskSizes = append(s.diskSizes, ev.rec)
+		}
 	}
 
 	prog.finish()
@@ -695,12 +706,13 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
 		s.resolve(rec.path)
 
 		s.batch = append(s.batch, scanRec{
-			size:    rec.size,
-			mtime:   rec.mtime,
-			head:    r.head,
-			tail:    r.tail,
-			content: r.content,
-			path:    rec.path,
+			size:     rec.size,
+			diskSize: rec.diskSize,
+			mtime:    rec.mtime,
+			head:     r.head,
+			tail:     r.tail,
+			content:  r.content,
+			path:     rec.path,
 		})
 	}
 
@@ -716,7 +728,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
 		return nil
 	}
 
-	err := applyBatch(ctx, s.db, s.batch, nil, nil)
+	err := applyBatch(ctx, s.db, s.batch, nil, nil, nil)
 	if err != nil {
 		return err
 	}
@@ -727,9 +739,10 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
 }
 
 // updatePhase writes the scan's tail under one progress display: the
-// final partial batch of hashed records, and deletions for every record
-// the scan did not verify (vanished files, plus paths that failed to
-// stat or hash).
+// final partial batch of hashed records, the new disk_size of every
+// unchanged file whose disk_size changed, and deletions for every
+// record the scan did not verify (vanished files, plus paths that
+// failed to stat or hash).
 func (s *scanState) updatePhase(ctx context.Context) error {
 	deletes := make([]string, 0, len(s.existing))
 	for path := range s.existing {
@@ -741,7 +754,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
 
 	s.st.removed = len(deletes)
 
-	total := len(s.batch) + len(deletes)
+	total := len(s.batch) + len(s.diskSizes) + len(deletes)
 	prog := newProgress(progressParams{
 		label: scanPhase("update"), unit: scanUnit, total: int64(total),
 		stalledOn: "db",
@@ -756,6 +769,11 @@ func (s *scanState) updatePhase(ctx context.Context) error {
 
 	s.batch = nil
 
+	err = applyDiskSizes(ctx, s.db, s.diskSizes, prog)
+	if err != nil {
+		return err
+	}
+
 	return applyChanges(ctx, s.db, nil, deletes, prog)
 }
 
@@ -961,11 +979,12 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
 
 	dev, ino := inodeOfInfo(fi)
 	rec := fileRec{
-		path:  root,
-		size:  fi.Size(),
-		mtime: fi.ModTime(),
-		dev:   dev,
-		ino:   ino,
+		path:     root,
+		size:     fi.Size(),
+		diskSize: diskSizeOfInfo(fi),
+		mtime:    fi.ModTime(),
+		dev:      dev,
+		ino:      ino,
 	}
 
 	if !isDatabaseFile(rec, rules) {
@@ -1145,11 +1164,12 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 
 	dev, ino := inodeOfInfo(info)
 	rec := fileRec{
-		path:  p,
-		size:  info.Size(),
-		mtime: info.ModTime(),
-		dev:   dev,
-		ino:   ino,
+		path:     p,
+		size:     info.Size(),
+		diskSize: diskSizeOfInfo(info),
+		mtime:    info.ModTime(),
+		dev:      dev,
+		ino:      ino,
 	}
 
 	if !isDatabaseFile(rec, rules) {
@@ -1241,6 +1261,23 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
 	return statDev(st), st.Ino
 }
 
+// statBlockSize is the unit of st_blocks in an lstat result on Linux and
+// macOS, whatever the filesystem's own block size.
+const statBlockSize = 512
+
+// diskSizeOfInfo returns the bytes a file takes on disk: its allocated
+// blocks, st_blocks × 512. On a compressed or sparse file it is less
+// than the file's size. It is 0 when the platform exposes no block
+// count.
+func diskSizeOfInfo(fi fs.FileInfo) int64 {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0
+	}
+
+	return st.Blocks * statBlockSize
+}
+
 // hashResult carries the hashes computed for one inode run (or the
 // error that prevented computing them) from the pool's workers to the
 // hash phase. Its run leaves out the paths found removed.

scan_test.go

@@ -42,6 +42,24 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
 	return p
 }
 
+// diskSizeOf returns the bytes the file at path takes on disk, from its
+// own lstat: st_blocks counts 512-byte units. ZFS counts a new file's
+// blocks only seconds after the write, and fsync does not hasten that.
+// A test therefore calls unix.Sync, which waits for it, before the scan
+// whose disk_size it compares with this.
+func diskSizeOf(t *testing.T, path string) int64 {
+	t.Helper()
+
+	var st syscall.Stat_t
+
+	err := syscall.Lstat(path, &st)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return st.Blocks * 512
+}
+
 // pattern returns n bytes of deterministic content seeded by tag.
 func pattern(tag byte, n int) []byte {
 	data := make([]byte, n)
@@ -1064,9 +1082,8 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
 	}
 }
 
-// buildSmokeTree recreates the tree of the test under README
-// "Definition of done" with deterministic content and returns the tree
-// root.
+// buildSmokeTree recreates the README smoke-test filesystem layout
+// with deterministic content and returns the tree root.
 func buildSmokeTree(t *testing.T) string {
 	t.Helper()
 
@@ -1229,9 +1246,9 @@ func TestScanPipeline(t *testing.T) {
 // TestScanProgressNamesEachPhase checks the plain progress lines of a
 // first scan, and of a rescan of one operand holding two of the
 // database's records. Each phase's first line names it with its place
-// among the four: load counts the records before reading them, and the
-// walk estimates its total from the records under its operand, or says
-// its time remaining is unknown when there are none.
+// among the four: load gets the number of records before reading them,
+// and the walk estimates its total from the records under its operand,
+// or says its time remaining is unknown when there are none.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestScanProgressNamesEachPhase(t *testing.T) {
@@ -1417,6 +1434,116 @@ func TestScanStoresRawHashes(t *testing.T) {
 	}
 }
 
+// TestScanStoresDiskSize scans a sparse file, which takes less space on
+// disk than its length, once through its directory and once as the
+// operand itself. Each scan stores the disk_size of the file's lstat.
+func TestScanStoresDiskSize(t *testing.T) {
+	t.Parallel()
+
+	const length = 1024 * 1024
+
+	p := sparseFile(t, t.TempDir(), "sparse", length)
+	pokeAt(t, p, 0, []byte("x"))
+
+	unix.Sync() // see diskSizeOf
+
+	want := diskSizeOf(t, p)
+	if want == 0 || want >= length {
+		t.Fatalf("the file takes %d bytes on disk, want more than none "+
+			"and fewer than its length %d", want, length)
+	}
+
+	for _, operand := range []string{filepath.Dir(p), p} {
+		db := openTestDB(t)
+		syncTree(t, db, operand)
+
+		got := recordByPath(t, dbRecords(t, db), p).diskSize
+		if got != want {
+			t.Errorf("scan of %s: disk_size = %d, want %d",
+				operand, got, want)
+		}
+	}
+}
+
+// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned
+// sparse file and moves its mtime on. The rescan hashes the file again
+// and rewrites its record with the new disk_size.
+func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := sparseFile(t, dir, "sparse", 1024*1024)
+	pokeAt(t, p, 0, []byte("x"))
+
+	syncTree(t, db, dir)
+
+	before := recordByPath(t, dbRecords(t, db), p).diskSize
+
+	pokeAt(t, p, 512*1024, pattern(7, 64*1024))
+
+	// The write may leave the mtime the scan recorded; a later one makes
+	// the rescan read the file again.
+	later := time.Now().Add(time.Hour)
+
+	err := os.Chtimes(p, later, later)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	unix.Sync() // see diskSizeOf
+
+	want := diskSizeOf(t, p)
+	if want == before {
+		t.Fatalf("the file still takes %d bytes on disk after the write "+
+			"into its hole", want)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{walked: 1, updated: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 updated", st)
+	}
+
+	got := recordByPath(t, dbRecords(t, db), p).diskSize
+	if got != want {
+		t.Errorf("disk_size = %d, want %d", got, want)
+	}
+}
+
+// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a
+// disk_size the file does not have, as when ZFS had not yet counted the
+// file's blocks at the last scan. The rescan writes the file's own
+// disk_size into the record without reading the file again.
+func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	p := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	syncTree(t, db, dir)
+
+	// lstat counts 512-byte units, so no file takes 1 byte on disk.
+	_, err := db.ExecContext(t.Context(),
+		"UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	unix.Sync() // see diskSizeOf
+
+	// A file read again would count as updated.
+	st := syncTree(t, db, dir)
+	if st != (scanStats{walked: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
+	}
+
+	got := recordByPath(t, dbRecords(t, db), p).diskSize
+	if want := diskSizeOf(t, p); got != want {
+		t.Errorf("disk_size = %d, want %d", got, want)
+	}
+}
+
 // TestSyncScanSameSecondRewrite rewrites a file in place at the same
 // size with an mtime later in the same second as the recorded one: the
 // next scan must notice the change and re-hash the file. The operand is

trees.go

@@ -22,6 +22,10 @@ type treeNode struct {
 	digest    [sha256.Size]byte
 	fileCount int64
 	totalSize int64
+	// diskSize is the bytes the directory's files take on disk. Two
+	// copies of one tree have the same totalSize, but their diskSize
+	// can differ.
+	diskSize int64
 }
 
 // treesPhase names one phase of trees with its place among all of them,
@@ -40,11 +44,11 @@ func treesPhase(name string) string {
 // database problem, including a missing database, is fatal.
 //
 // The work runs in four phases, each under a progress display of its
-// own on stderr: open (open the database), read (count the records,
-// then read them into the hierarchy), group (group the directories by
-// digest) and write (write the groups to stdout). Each display ends
-// before the next phase starts, and before an error is returned to be
-// printed.
+// own on stderr: open (open the database), read (read the number of
+// records from the totals table, then read the records into the
+// hierarchy), group (group the directories by digest) and write (write
+// the groups to stdout). Each display ends before the next phase
+// starts, and before an error is returned to be printed.
 func runTrees(ctx context.Context, stdout io.Writer) error {
 	dbPath, source, err := databasePath()
 	if err != nil {
@@ -139,15 +143,16 @@ func readTree(ctx context.Context, db *sql.DB,
 
 // writeTreeGroups writes dupes to stdout as TSV, a header line and then
 // a row for every member of each group but its first, counting each
-// group on prog once its rows are written. stdout may be the terminal
-// prog is drawn on, so the lines go through prog's writer. It returns
-// the number of rows (dupe trees) and the bytes they could reclaim.
+// group on prog once its rows are written. The disk_size column is the
+// dupe's own. stdout may be the terminal prog is drawn on, so the lines
+// go through prog's writer. It returns the number of rows (dupe trees)
+// and the bytes they could reclaim.
 func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 	prog *progress,
 ) (int, int64, error) {
 	out := prog.writer(stdout)
 
-	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
+	_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
 	if err != nil {
 		return 0, 0, fmt.Errorf("write stdout: %w", err)
 	}
@@ -159,9 +164,9 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
 	for _, g := range dupes {
 		first := g[0]
 		for _, n := range g[1:] {
-			_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
+			_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n",
 				escapePath(first.path), escapePath(n.path),
-				first.fileCount, first.totalSize)
+				first.fileCount, first.totalSize, n.diskSize)
 			if err != nil {
 				return 0, 0, fmt.Errorf("write stdout: %w", err)
 			}
@@ -232,6 +237,7 @@ func (b *treeBuilder) add(r scanRec) {
 	dir.entries = append(dir.entries, fileEntry(name, r))
 	dir.fileCount++
 	dir.totalSize += r.size
+	dir.diskSize += r.diskSize
 }
 
 // openDir starts the directory called name inside the innermost open
@@ -270,6 +276,7 @@ func (b *treeBuilder) closeTo(n int) {
 			"d\x00"+name+"\x00"+string(dir.digest[:]))
 		dir.parent.fileCount += dir.fileCount
 		dir.parent.totalSize += dir.totalSize
+		dir.parent.diskSize += dir.diskSize
 		b.dirs = append(b.dirs, dir)
 	}
 }

trees_test.go

@@ -15,21 +15,26 @@ import (
 const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
 	"remain unknown"
 
-// smokeTreeRecs mirrors the tree of the test under README "Definition
-// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
-// by one filename.
+// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
+// /d/t2 are identical, /d/t3 differs from them only by one filename.
+// Each file takes one 4 KiB block on disk.
 func smokeTreeRecs() []scanRec {
 	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
 	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
 
 	return []scanRec{
-		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
-		{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
-		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
-		{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
-		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
-		{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
-			path: "/d/t3/sub/f2renamed"},
+		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
+			content: f1Content, path: "/d/t1/f1"},
+		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
+			content: f2Content, path: "/d/t1/sub/f2"},
+		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
+			content: f1Content, path: "/d/t2/f1"},
+		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
+			content: f2Content, path: "/d/t2/sub/f2"},
+		{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
+			content: f1Content, path: "/d/t3/f1"},
+		{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
+			content: f2Content, path: "/d/t3/sub/f2renamed"},
 	}
 }
 
@@ -199,13 +204,33 @@ func TestRunTreesEscapesPaths(t *testing.T) {
 			code, exitOK, stderr.String())
 	}
 
-	want := "first\tdupe\tfiles\tsize\n" +
-		`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n"
+	want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
+		`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\t4096\n"
 	if got := stdout.String(); got != want {
 		t.Errorf("stdout = %q, want %q", got, want)
 	}
 }
 
+// TestRunTreesDiskSize runs trees on two copies of a tree whose files
+// take different space on disk. The row's disk_size is the sum over the
+// dupe copy's files, those in its subdirectory included.
+func TestRunTreesDiskSize(t *testing.T) {
+	f, g := testHash("f"), testHash("g")
+	recs := []scanRec{
+		{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
+		{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
+		{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
+		{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
+	}
+
+	t.Setenv(databaseEnv, seedDatabase(t, recs))
+
+	want := "first\tdupe\tfiles\tsize\tdisk_size\n/a\t/b\t2\t30\t1536\n"
+	if got := runStdout(t, cmdTrees); got != want {
+		t.Errorf("stdout = %q, want %q", got, want)
+	}
+}
+
 // TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
 // records with stderr not a terminal, and checks every line it prints
 // there. After the line naming the database, each phase prints a line
@@ -227,7 +252,8 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
 			code, exitOK, stderr())
 	}
 
-	report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
+	report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
+		"/d/t1\t/d/t2\t2\t3100\t8192\n"
 	if got := stdout.String(); got != report {
 		t.Errorf("stdout = %q, want %q", got, report)
 	}
@@ -292,7 +318,9 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
 		shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
 	}
 
-	want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
+	want := []string{
+		"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
+	}
 	if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
 		t.Errorf("terminal shows lines %q, want %q first", shown, want)
 	}