From 4ae7274bc05375dfd97643b88c6a3388e9987789 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 27 Oct 2026 18:08:27 +0000 Subject: [PATCH] Record each file's size on disk and print it in trees (closes #147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The files table gets a disk_size column: the file's allocated blocks, st_blocks × 512, from the lstat the walk already does. scan writes it with every record it writes. When a rescan finds a file with its size and mtime unchanged but a different disk_size, it updates disk_size alone, without reading the file. trees adds disk_size up per directory and prints it as a column after size. On a compressed or sparse file it is less than size, so two copies of one tree can differ; each row shows the dupe copy's. Judgement calls: a row's disk_size is the dupe copy's, not the first's. A file whose record gets only a new disk_size counts as unchanged. Model: opus-5-5 --- README.md | 77 +++++++++++++++++---------- TODO.md | 7 +++ db.go | 122 +++++++++++++++++++++++++++++++------------ main_test.go | 34 +++++++++--- report.go | 16 +++--- report_test.go | 4 +- scan.go | 139 +++++++++++++++++++++++++++++++------------------ scan_test.go | 128 +++++++++++++++++++++++++++++++++++++++++++++ trees.go | 19 ++++--- trees_test.go | 51 ++++++++++++++---- 10 files changed, 452 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index 7dc8fb3..e16414b 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,14 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite" ``` `scan` walks one or more filesystem trees and maintains one database record per -regular file (path, size, mtime, head hash, tail hash, content hash). The -database persists between runs. A rescan hashes only the files that are new or -changed, and removes the records of files that no longer exist. `report` reads -the database and prints the file-level duplicates report. `trees` reads the same -database and prints the duplicate-tree report. `info` prints the database's -details: its path, the sizes of its files and the number of its records. A -missing/invalid subcommand — or a `scan` invocation with no `PATH` operand — -prints a usage message and exits 2. +regular file (path, size, size on disk, mtime, head hash, tail hash, content +hash). The database persists between runs. A rescan hashes only the files that +are new or changed, and removes the records of files that no longer exist. +`report` reads the database and prints the file-level duplicates report. `trees` +reads the same database and prints the duplicate-tree report. `info` prints the +database's details: its path, the sizes of its files and the number of its +records. A missing/invalid subcommand — or a `scan` invocation with no `PATH` +operand — prints a usage message and exits 2. The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan` @@ -226,12 +226,12 @@ Goals, in order: its ends and at 1 MiB per gigabyte (see "Duplicate detection" below). Therefore no file of 100 MiB or more is read in full. Scale target: tens of millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool - under resilver). Holding one small record (path, size, mtime) per file in - memory during a scan is acceptable; holding every file's hashes is not (they - stay in the database). The reporting commands do not hold every file's - hashes either: `report` reads the records in signature order and holds the - duplicate paths of one size at a time, so its memory grows with the - duplicates of the size that has the most of them, and `trees` reads the + under resilver). Holding one small record (path, size, size on disk, mtime) + per file in memory during a scan is acceptable; holding every file's hashes + is not (they stay in the database). The reporting commands do not hold every + file's hashes either: `report` reads the records in signature order and + holds the duplicate paths of one size at a time, so its memory grows with + the duplicates of the size that has the most of them, and `trees` reads the records in path order and keeps each directory's path, digest and totals, plus the hashes of only the files in the directories holding the record being read, so its memory grows with the number of directories and with the @@ -368,7 +368,8 @@ All four subcommands operate on a single SQLite database file: - While it loads the existing records (`load`) and walks: no records. A first scan's database holds none, however long its walk has run. - During the hash phase: the first records, committed 10,000 at a time. - - In the update phase: the rest of the hashed records, and the deletions. + - In the update phase: the rest of the hashed records, the new `disk_size` + of unchanged files whose size on disk changed, and the deletions. - At the end, as it closes the database: the `-wal` and `-shm` files go away, unless a report still has the database open (see above). - `report`, `trees` and `info` open the database read-only and need only read @@ -386,6 +387,7 @@ All four subcommands operate on a single SQLite database file: CREATE TABLE files ( path BLOB PRIMARY KEY, -- absolute path, raw bytes size INTEGER NOT NULL, -- bytes, from lstat + disk_size INTEGER NOT NULL, -- bytes allocated on disk (st_blocks × 512), from lstat mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999 head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB @@ -514,6 +516,11 @@ operands: - A file already in the database is **skipped without reading its contents** when its lstat size equals the recorded size and its lstat mtime is not newer than the recorded mtime. This is what makes a daily rescan cheap. +- When such a file's size on disk differs from the recorded `disk_size`, the + scan updates `disk_size` alone. It still does not read the file, and counts it + as unchanged. ZFS, for example, counts a new file's blocks only seconds after + the write. A file scanned within those seconds gets its real `disk_size` at + the next scan. - A file whose mtime is newer than recorded, or whose size differs, is hashed again. Change detection compares the mtime to the nanosecond, as finely as the filesystem records it, so a same-size rewrite counts as a change whenever the @@ -541,8 +548,8 @@ inside the walk and hash phases, and batched database writes begin during the hash phase: 1. **load** — read the existing records. Each record under the operands is kept - in memory, by path, with its size and its mtime, for the walk to compare - each file with. + in memory, by path, with its size, its size on disk and its mtime, for the + walk to compare each file with. 2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently with the walk worker pool: every operand seeds the shared queue, and each worker reads one directory at a time, handing discovered subdirectories back @@ -550,9 +557,10 @@ hash phase: (while the directory's metadata is still hot). Sequential directory enumeration is metadata-latency-bound and takes hours at tens of millions of files; per-directory parallelism is what makes the walk tractable on large - or busy pools. The walk compares each file with its record as it goes, and - an unchanged file needs nothing more. Every new or changed file is carried - to the hash phase as a (path, size, mtime) record. + or busy pools. The walk compares each file with its record as it goes. An + unchanged file is not read again. When its size on disk has changed, the + update phase writes its new `disk_size`. Every new or changed file is + carried to the hash phase as a (path, size, size on disk, mtime) record. 3. **hash** — the worker pool hashes every carried file as described in "Duplicate detection" above, which gives the file its `head`, `tail` and `content`. Zero-length files have constant hashes and are never opened. @@ -564,7 +572,8 @@ hash phase: committed in batched transactions **while hashing runs**. A scan interrupted after hours therefore keeps everything hashed so far, and the next scan resumes cheaply, skipping records already written. -4. **update** — commit the final partial batch of hashed records. Then delete +4. **update** — commit the final partial batch of hashed records, and the new + `disk_size` of each unchanged file whose size on disk changed. Then delete the records the scan did not verify: those of vanished files, and of paths that failed to stat or hash. @@ -809,14 +818,26 @@ Processing: #### Trees output format TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a -group of N). `files` is the recursive regular-file count of one copy of the -tree; `size` is the recursive total byte size of one copy: +group of N). After the `first` and `dupe` paths come three numbers: + +- `files`, the number of regular files in one copy of the tree, counted + recursively; +- `size`, the bytes of one copy, its files' sizes added up; +- `disk_size`, the bytes the `dupe` copy takes on disk, its files' `disk_size` + added up. ``` -first dupe files size -/srv/a/project /srv/backup/project 3417 104857600 +first dupe files size disk_size +/srv/a/project /srv/backup/project 3417 104857600 61865984 ``` +A file's `disk_size` is the space its blocks take on disk. It can be more than +the file's size, because space is allocated in whole blocks. It is less than the +file's size on a compressed or sparse file, so two copies of one tree can differ +in `disk_size`. When a file in one copy is a hard link to one in the other, both +copies count its blocks in full. Removing it from one copy then frees nothing, +as the known limitation under "`trees` mode" says. + The `first` and `dupe` paths are escaped as described under "Report output format". The root directory's path is `/`. @@ -1200,9 +1221,9 @@ All of the following, run in this directory, must pass: ordered by size descending. Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2 - files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal - (implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set - differs by name). + files, 3100 bytes, and a `disk_size` that depends on the filesystem. + `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal (implied by the + `t1`/`t2` group), and `t3` appears nowhere (its file set differs by name). Expected from the second `report` (after the modify/delete rescan): `one.bin` has left its group (its content changed), so diff --git a/TODO.md b/TODO.md index 8cff644..6e67f8e 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ # Completed Steps +- each record holds `disk_size`, the bytes the file takes on disk: its allocated + blocks, `st_blocks` × 512, from the lstat the walk already does. `trees` + prints a `disk_size` column after `size`, the sum over the `dupe` copy's + files; on a compressed or sparse file it differs from `size`. A rescan updates + the `disk_size` of an unchanged file whose size on disk changed, without + reading the file (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/147) + - the database keeps the number of records and the sum of their sizes in the one row of a `totals` table, which three triggers on `files` keep current. `info` reads that row, and so do `scan`'s `load` phase and the `read` phases of diff --git a/db.go b/db.go index c878f60..e92f6db 100644 --- a/db.go +++ b/db.go @@ -47,14 +47,15 @@ const dbFilePerm = 0o644 const lockFilePerm = 0o600 // createTableSQL is the schema applied to a fresh database. Paths are -// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime -// holds whole Unix seconds and mtime_nsec the nanoseconds within that -// second. head, tail and content each hold the raw 32 bytes of a -// SHA-256. +// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. +// disk_size holds the bytes the file takes on disk. mtime holds whole +// Unix seconds and mtime_nsec the nanoseconds within that second. head, +// tail and content each hold the raw 32 bytes of a SHA-256. const createTableSQL = ` CREATE TABLE files ( path BLOB PRIMARY KEY, size INTEGER NOT NULL, + disk_size INTEGER NOT NULL, mtime INTEGER NOT NULL, mtime_nsec INTEGER NOT NULL, head BLOB NOT NULL, @@ -115,11 +116,13 @@ SELECT records FROM totals // upsertSQL inserts one file record, replacing any existing record for // the same path. const upsertSQL = ` -INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content) -VALUES (?, ?, ?, ?, ?, ?, ?) +INSERT INTO files ( + path, size, disk_size, mtime, mtime_nsec, head, tail, content +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (path) DO UPDATE SET - size = excluded.size, mtime = excluded.mtime, - mtime_nsec = excluded.mtime_nsec, + size = excluded.size, disk_size = excluded.disk_size, + mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec, head = excluded.head, tail = excluded.tail, content = excluded.content ` @@ -536,8 +539,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64), total(n) rows, err := tx.QueryContext(ctx, - "SELECT path, size, mtime, mtime_nsec, head, tail, content "+ - "FROM files ORDER BY path") + "SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+ + "content FROM files ORDER BY path") if err != nil { return fmt.Errorf("read records: %w", err) } @@ -551,7 +554,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64), r scanRec ) - err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content) + err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec, + &head, &tail, &content) if err != nil { return fmt.Errorf("read record: %w", err) } @@ -586,10 +590,11 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC ` // loadSignatureRows gives total the number of records, then streams -// every record's signature and path to fn, without its mtime, in the -// order of signatureRowsSQL. The count and the records are read in one -// transaction, so they agree while a scan is committing. An error from -// fn stops the reading and is returned as it is. +// every record's signature and path to fn, without its mtime or its +// disk_size, in the order of signatureRowsSQL. The count and the +// records are read in one transaction, so they agree while a scan is +// committing. An error from fn stops the reading and is returned as it +// is. func loadSignatureRows(ctx context.Context, db *sql.DB, total func(n int64), fn func(r scanRec) error, ) error { @@ -724,15 +729,15 @@ func readDatabaseCounts(ctx context.Context, return c, nil } -// loadFileMeta streams every record's path, size and mtime to fn. Scan -// change detection needs no hash values, and skipping the hash columns -// keeps the scan's in-memory index small on multi-million-file -// databases. +// loadFileMeta streams every record's path, size, disk_size and mtime +// to fn. Scan change detection needs no hash values, and skipping the +// hash columns keeps the scan's in-memory index small on +// multi-million-file databases. func loadFileMeta(ctx context.Context, db *sql.DB, - fn func(path string, size int64, mtime time.Time), + fn func(path string, m fileMeta), ) error { rows, err := db.QueryContext(ctx, - "SELECT path, size, mtime, mtime_nsec FROM files") + "SELECT path, size, disk_size, mtime, mtime_nsec FROM files") if err != nil { return fmt.Errorf("read records: %w", err) } @@ -741,16 +746,19 @@ func loadFileMeta(ctx context.Context, db *sql.DB, for rows.Next() { var ( - path []byte - size, sec, nsec int64 + path []byte + m fileMeta + sec, nsec int64 ) - err = rows.Scan(&path, &size, &sec, &nsec) + err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec) if err != nil { return fmt.Errorf("read record: %w", err) } - fn(string(path), size, time.Unix(sec, nsec)) + m.mtime = time.Unix(sec, nsec) + + fn(string(path), m) } err = rows.Err() @@ -768,21 +776,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB, // WAL small and let concurrent reports observe progress. const updateBatchSize = 10000 -// applyChanges writes one scan's database changes — upserts for new and -// changed files, deletes for vanished ones — in batched transactions. -// Progress is rendered on prog (one increment per change). +// applyChanges writes upserts for new and changed files and deletes for +// vanished ones, in batched transactions. Progress is rendered on prog +// (one increment per change). func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec, deletes []string, prog *progress, ) error { for batch := range slices.Chunk(upserts, updateBatchSize) { - err := applyBatch(ctx, db, batch, nil, prog) + err := applyBatch(ctx, db, batch, nil, nil, prog) if err != nil { return err } } for batch := range slices.Chunk(deletes, updateBatchSize) { - err := applyBatch(ctx, db, nil, batch, prog) + err := applyBatch(ctx, db, nil, nil, batch, prog) if err != nil { return err } @@ -791,10 +799,26 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec, return nil } -// applyBatch commits one batch of upserts and deletes in a single -// transaction. +// applyDiskSizes writes the disk_size of each of recs into its record, +// in batched transactions, and leaves the rest of the record alone. +// Progress is rendered on prog (one increment per record). +func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec, + prog *progress, +) error { + for batch := range slices.Chunk(recs, updateBatchSize) { + err := applyBatch(ctx, db, nil, batch, nil, prog) + if err != nil { + return err + } + } + + return nil +} + +// applyBatch commits one batch of upserts, disk_size updates and +// deletes in a single transaction. func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec, - deletes []string, prog *progress, + diskSizes []fileRec, deletes []string, prog *progress, ) error { tx, err := db.BeginTx(ctx, nil) if err != nil { @@ -808,6 +832,11 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec, return err } + err = execDiskSizes(ctx, tx, diskSizes, prog) + if err != nil { + return err + } + err = execDeletes(ctx, tx, deletes, prog) if err != nil { return err @@ -833,7 +862,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec, defer func() { _ = st.Close() }() for _, r := range upserts { - _, err = st.ExecContext(ctx, []byte(r.path), r.size, + _, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize, r.mtime.Unix(), r.mtime.Nanosecond(), r.head[:], r.tail[:], r.content[:]) if err != nil { @@ -846,6 +875,31 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec, return nil } +// execDiskSizes sets the disk_size of the record of each of recs. It +// does not touch size, so the totals triggers do not fire. +func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec, + prog *progress, +) error { + st, err := tx.PrepareContext(ctx, + "UPDATE files SET disk_size = ? WHERE path = ?") + if err != nil { + return fmt.Errorf("prepare disk_size update: %w", err) + } + + defer func() { _ = st.Close() }() + + for _, r := range recs { + _, err = st.ExecContext(ctx, r.diskSize, []byte(r.path)) + if err != nil { + return fmt.Errorf("update disk_size of %s: %w", r.path, err) + } + + prog.increment() + } + + return nil +} + // execDeletes removes the records for paths no longer present. func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string, prog *progress, diff --git a/main_test.go b/main_test.go index cdb385c..547ad2c 100644 --- a/main_test.go +++ b/main_test.go @@ -818,6 +818,23 @@ func scanFixture(t *testing.T) []string { return dupes } +// storedDiskSize returns the disk_size of the record for path in the +// database at dbPath. A test of a trees run compares with this rather +// than with a new lstat, which on ZFS can differ from the scan's (see +// diskSizeOf). +func storedDiskSize(t *testing.T, dbPath, path string) int64 { + t.Helper() + + db, err := openReportDatabase(t.Context(), dbPath) + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + return recordByPath(t, dbRecords(t, db), path).diskSize +} + // scanOK runs scan over operands, fails the test unless it exits 0 with // nothing on stdout, and returns everything it printed to stderr. func scanOK(t *testing.T, operands ...string) string { @@ -1141,8 +1158,9 @@ func TestRunTreesSucceeds(t *testing.T) { // The two directories holding the duplicate pair are duplicate // trees of each other. - want := "first\tdupe\tfiles\tsize\n" + - filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n" + want := "first\tdupe\tfiles\tsize\tdisk_size\n" + + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" + + strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n" if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } @@ -1176,9 +1194,10 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) { cases := map[string]string{ cmdReport: "first\tdupe\tsize\n" + dupes[0] + "\t" + dupes[1] + "\t300\n", - cmdTrees: "first\tdupe\tfiles\tsize\n" + + cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + - "\t1\t300\n", + "\t1\t300\t" + + strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n", } for name, want := range cases { @@ -1213,8 +1232,11 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) { t.Errorf("report stdout = %q, want %q", got, want) } - want = "first\tdupe\tfiles\tsize\n" + - filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n" + diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1]) + + want = "first\tdupe\tfiles\tsize\tdisk_size\n" + + filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" + + strconv.FormatInt(diskSize, 10) + "\n" if got := runStdout(t, cmdTrees); got != want { t.Errorf("trees stdout = %q, want %q", got, want) } diff --git a/report.go b/report.go index f780335..0504cd8 100644 --- a/report.go +++ b/report.go @@ -21,14 +21,16 @@ const minGroupSize = 2 // scanRec is one file record from the database. The signature (size, // head, tail, content) is the duplicate key; mtime is informational -// only and used by scan for change detection. +// only and used by scan for change detection. diskSize is the bytes the +// file takes on disk, which trees adds up. type scanRec struct { - size int64 - mtime time.Time - head [sha256.Size]byte - tail [sha256.Size]byte - content [sha256.Size]byte - path string + size int64 + diskSize int64 + mtime time.Time + head [sha256.Size]byte + tail [sha256.Size]byte + content [sha256.Size]byte + path string } // reportPhase names one phase of report with its place among all of diff --git a/report_test.go b/report_test.go index 9b47034..2023cb3 100644 --- a/report_test.go +++ b/report_test.go @@ -38,8 +38,8 @@ func awkwardPairRecs() []scanRec { h, tl, c := testHash("h"), testHash("t"), testHash("c") return []scanRec{ - {size: 5, head: h, tail: tl, content: c, path: "/d/A/f"}, - {size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"}, + {size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"}, + {size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"}, } } diff --git a/scan.go b/scan.go index ac49c7e..a6dbc0a 100644 --- a/scan.go +++ b/scan.go @@ -52,23 +52,26 @@ const workQueueDepth = 1024 // has already printed its line, so run prints nothing more. var errInterrupted = errors.New("scan interrupted") -// fileRec carries one statted file between the scan phases. dev and -// ino identify the underlying inode so hard-linked paths can share -// one read; both are zero when the platform exposes no inode. +// fileRec carries one statted file between the scan phases. diskSize is +// the bytes the file takes on disk. dev and ino identify the underlying +// inode so hard-linked paths can share one read; both are zero when the +// platform exposes no inode. type fileRec struct { - path string - size int64 - mtime time.Time - dev uint64 - ino uint64 + path string + size int64 + diskSize int64 + mtime time.Time + dev uint64 + ino uint64 } // fileMeta is the in-memory index entry for one existing database -// record: just enough for change detection. Hashes stay on disk; at +// record: what the walk compares the file with. Hashes stay on disk; at // tens of millions of records they would dominate the scan's memory. type fileMeta struct { - size int64 - mtime time.Time + size int64 + diskSize int64 + mtime time.Time } // scanUnit is what every scan phase counts, as its progress display @@ -339,11 +342,12 @@ type scanStats struct { // whatever remains after the walk and hash phases is deleted by the // update phase. type scanState struct { - db *sql.DB - existing map[string]fileMeta - toHash []fileRec // new and changed files: must be read - batch []scanRec - st scanStats + db *sql.DB + existing map[string]fileMeta + toHash []fileRec // new and changed files: must be read + diskSizes []fileRec // unchanged files whose disk_size changed + batch []scanRec + st scanStats } // syncScan synchronizes the database with the filesystem under roots; @@ -472,24 +476,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error { s.existing = make(map[string]fileMeta) - return loadFileMeta(ctx, s.db, - func(path string, size int64, mtime time.Time) { - prog.increment() + return loadFileMeta(ctx, s.db, func(path string, m fileMeta) { + prog.increment() - if underAnyRoot(path, roots) { - s.existing[path] = fileMeta{size: size, mtime: mtime} - } - }) + if underAnyRoot(path, roots) { + s.existing[path] = m + } + }) } // walkPhase drains the walk and queues every new or changed file for -// the hash phase. An unchanged file needs nothing more, and its record -// counts as verified. A record at or below a path the walk left out, -// because -x stopped at it, it is a default exclude, or it is not on a -// persistent filesystem, counts as outside the roots: it is not -// deleted. That does not hold below a root at or inside that path, -// since that root's own walk verifies the records below it; nor is a -// line printed for such a root. +// the hash phase. An unchanged file is not read, and its record counts +// as verified. When its disk_size differs from the record's, it is +// queued for the update phase, which writes the new disk_size. A record +// at or below a path the walk left out, because -x stopped at it, it is +// a default exclude, or it is not on a persistent filesystem, counts as +// outside the roots: it is not deleted. That does not hold below a root +// at or inside that path, since that root's own walk verifies the +// records below it; nor is a line printed for such a root. func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) { // The records the last scan left under the roots estimate how many // files the walk will find. A first scan has none, and no estimate. @@ -547,6 +551,13 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) { delete(s.existing, ev.rec.path) s.st.unchanged++ + + // The space a file takes on disk can change while its size and + // mtime stay the same. ZFS, for one, counts a new file's blocks + // only seconds after the write. + if old.diskSize != ev.rec.diskSize { + s.diskSizes = append(s.diskSizes, ev.rec) + } } prog.finish() @@ -695,12 +706,13 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error { s.resolve(rec.path) s.batch = append(s.batch, scanRec{ - size: rec.size, - mtime: rec.mtime, - head: r.head, - tail: r.tail, - content: r.content, - path: rec.path, + size: rec.size, + diskSize: rec.diskSize, + mtime: rec.mtime, + head: r.head, + tail: r.tail, + content: r.content, + path: rec.path, }) } @@ -716,7 +728,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error { return nil } - err := applyBatch(ctx, s.db, s.batch, nil, nil) + err := applyBatch(ctx, s.db, s.batch, nil, nil, nil) if err != nil { return err } @@ -727,9 +739,10 @@ func (s *scanState) commitFullBatch(ctx context.Context) error { } // updatePhase writes the scan's tail under one progress display: the -// final partial batch of hashed records, and deletions for every record -// the scan did not verify (vanished files, plus paths that failed to -// stat or hash). +// final partial batch of hashed records, the new disk_size of every +// unchanged file whose disk_size changed, and deletions for every +// record the scan did not verify (vanished files, plus paths that +// failed to stat or hash). func (s *scanState) updatePhase(ctx context.Context) error { deletes := make([]string, 0, len(s.existing)) for path := range s.existing { @@ -741,7 +754,7 @@ func (s *scanState) updatePhase(ctx context.Context) error { s.st.removed = len(deletes) - total := len(s.batch) + len(deletes) + total := len(s.batch) + len(s.diskSizes) + len(deletes) prog := newProgress(progressParams{ label: scanPhase("update"), unit: scanUnit, total: int64(total), stalledOn: "db", @@ -756,6 +769,11 @@ func (s *scanState) updatePhase(ctx context.Context) error { s.batch = nil + err = applyDiskSizes(ctx, s.db, s.diskSizes, prog) + if err != nil { + return err + } + return applyChanges(ctx, s.db, nil, deletes, prog) } @@ -961,11 +979,12 @@ func seedRoot(ctx context.Context, root string, rules walkRules, dev, ino := inodeOfInfo(fi) rec := fileRec{ - path: root, - size: fi.Size(), - mtime: fi.ModTime(), - dev: dev, - ino: ino, + path: root, + size: fi.Size(), + diskSize: diskSizeOfInfo(fi), + mtime: fi.ModTime(), + dev: dev, + ino: ino, } if !isDatabaseFile(rec, rules) { @@ -1145,11 +1164,12 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry, dev, ino := inodeOfInfo(info) rec := fileRec{ - path: p, - size: info.Size(), - mtime: info.ModTime(), - dev: dev, - ino: ino, + path: p, + size: info.Size(), + diskSize: diskSizeOfInfo(info), + mtime: info.ModTime(), + dev: dev, + ino: ino, } if !isDatabaseFile(rec, rules) { @@ -1241,6 +1261,23 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) { return statDev(st), st.Ino } +// statBlockSize is the unit of st_blocks in an lstat result on Linux and +// macOS, whatever the filesystem's own block size. +const statBlockSize = 512 + +// diskSizeOfInfo returns the bytes a file takes on disk: its allocated +// blocks, st_blocks × 512. On a compressed or sparse file it is less +// than the file's size. It is 0 when the platform exposes no block +// count. +func diskSizeOfInfo(fi fs.FileInfo) int64 { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0 + } + + return st.Blocks * statBlockSize +} + // hashResult carries the hashes computed for one inode run (or the // error that prevented computing them) from the pool's workers to the // hash phase. Its run leaves out the paths found removed. diff --git a/scan_test.go b/scan_test.go index 4ed8ae1..91e9c09 100644 --- a/scan_test.go +++ b/scan_test.go @@ -42,6 +42,24 @@ func writeFile(t *testing.T, dir, name string, data []byte) string { return p } +// diskSizeOf returns the bytes the file at path takes on disk, from its +// own lstat: st_blocks counts 512-byte units. ZFS counts a new file's +// blocks only seconds after the write, and fsync does not hasten that. +// A test therefore calls unix.Sync, which waits for it, before the scan +// whose disk_size it compares with this. +func diskSizeOf(t *testing.T, path string) int64 { + t.Helper() + + var st syscall.Stat_t + + err := syscall.Lstat(path, &st) + if err != nil { + t.Fatal(err) + } + + return st.Blocks * 512 +} + // pattern returns n bytes of deterministic content seeded by tag. func pattern(tag byte, n int) []byte { data := make([]byte, n) @@ -1416,6 +1434,116 @@ func TestScanStoresRawHashes(t *testing.T) { } } +// TestScanStoresDiskSize scans a sparse file, which takes less space on +// disk than its length, once through its directory and once as the +// operand itself. Each scan stores the disk_size of the file's lstat. +func TestScanStoresDiskSize(t *testing.T) { + t.Parallel() + + const length = 1024 * 1024 + + p := sparseFile(t, t.TempDir(), "sparse", length) + pokeAt(t, p, 0, []byte("x")) + + unix.Sync() // see diskSizeOf + + want := diskSizeOf(t, p) + if want == 0 || want >= length { + t.Fatalf("the file takes %d bytes on disk, want more than none "+ + "and fewer than its length %d", want, length) + } + + for _, operand := range []string{filepath.Dir(p), p} { + db := openTestDB(t) + syncTree(t, db, operand) + + got := recordByPath(t, dbRecords(t, db), p).diskSize + if got != want { + t.Errorf("scan of %s: disk_size = %d, want %d", + operand, got, want) + } + } +} + +// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned +// sparse file and moves its mtime on. The rescan hashes the file again +// and rewrites its record with the new disk_size. +func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + p := sparseFile(t, dir, "sparse", 1024*1024) + pokeAt(t, p, 0, []byte("x")) + + syncTree(t, db, dir) + + before := recordByPath(t, dbRecords(t, db), p).diskSize + + pokeAt(t, p, 512*1024, pattern(7, 64*1024)) + + // The write may leave the mtime the scan recorded; a later one makes + // the rescan read the file again. + later := time.Now().Add(time.Hour) + + err := os.Chtimes(p, later, later) + if err != nil { + t.Fatal(err) + } + + unix.Sync() // see diskSizeOf + + want := diskSizeOf(t, p) + if want == before { + t.Fatalf("the file still takes %d bytes on disk after the write "+ + "into its hole", want) + } + + st := syncTree(t, db, dir) + if st != (scanStats{walked: 1, updated: 1}) { + t.Fatalf("rescan stats = %+v, want 1 updated", st) + } + + got := recordByPath(t, dbRecords(t, db), p).diskSize + if got != want { + t.Errorf("disk_size = %d, want %d", got, want) + } +} + +// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a +// disk_size the file does not have, as when ZFS had not yet counted the +// file's blocks at the last scan. The rescan writes the file's own +// disk_size into the record without reading the file again. +func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + p := writeFile(t, dir, "a.bin", pattern(1, 500)) + + syncTree(t, db, dir) + + // lstat counts 512-byte units, so no file takes 1 byte on disk. + _, err := db.ExecContext(t.Context(), + "UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p)) + if err != nil { + t.Fatal(err) + } + + unix.Sync() // see diskSizeOf + + // A file read again would count as updated. + st := syncTree(t, db, dir) + if st != (scanStats{walked: 1, unchanged: 1}) { + t.Fatalf("rescan stats = %+v, want 1 unchanged", st) + } + + got := recordByPath(t, dbRecords(t, db), p).diskSize + if want := diskSizeOf(t, p); got != want { + t.Errorf("disk_size = %d, want %d", got, want) + } +} + // TestSyncScanSameSecondRewrite rewrites a file in place at the same // size with an mtime later in the same second as the recorded one: the // next scan must notice the change and re-hash the file. The operand is diff --git a/trees.go b/trees.go index 3006931..554666f 100644 --- a/trees.go +++ b/trees.go @@ -22,6 +22,10 @@ type treeNode struct { digest [sha256.Size]byte fileCount int64 totalSize int64 + // diskSize is the bytes the directory's files take on disk. Two + // copies of one tree have the same totalSize, but their diskSize + // can differ. + diskSize int64 } // treesPhase names one phase of trees with its place among all of them, @@ -139,15 +143,16 @@ func readTree(ctx context.Context, db *sql.DB, // writeTreeGroups writes dupes to stdout as TSV, a header line and then // a row for every member of each group but its first, counting each -// group on prog once its rows are written. stdout may be the terminal -// prog is drawn on, so the lines go through prog's writer. It returns -// the number of rows (dupe trees) and the bytes they could reclaim. +// group on prog once its rows are written. The disk_size column is the +// dupe's own. stdout may be the terminal prog is drawn on, so the lines +// go through prog's writer. It returns the number of rows (dupe trees) +// and the bytes they could reclaim. func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode, prog *progress, ) (int, int64, error) { out := prog.writer(stdout) - _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize") + _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size") if err != nil { return 0, 0, fmt.Errorf("write stdout: %w", err) } @@ -159,9 +164,9 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode, for _, g := range dupes { first := g[0] for _, n := range g[1:] { - _, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n", + _, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n", escapePath(first.path), escapePath(n.path), - first.fileCount, first.totalSize) + first.fileCount, first.totalSize, n.diskSize) if err != nil { return 0, 0, fmt.Errorf("write stdout: %w", err) } @@ -232,6 +237,7 @@ func (b *treeBuilder) add(r scanRec) { dir.entries = append(dir.entries, fileEntry(name, r)) dir.fileCount++ dir.totalSize += r.size + dir.diskSize += r.diskSize } // openDir starts the directory called name inside the innermost open @@ -270,6 +276,7 @@ func (b *treeBuilder) closeTo(n int) { "d\x00"+name+"\x00"+string(dir.digest[:])) dir.parent.fileCount += dir.fileCount dir.parent.totalSize += dir.totalSize + dir.parent.diskSize += dir.diskSize b.dirs = append(b.dirs, dir) } } diff --git a/trees_test.go b/trees_test.go index 540c117..2a472e8 100644 --- a/trees_test.go +++ b/trees_test.go @@ -17,18 +17,24 @@ const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " + // smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and // /d/t2 are identical, /d/t3 differs from them only by one filename. +// Each file takes one 4 KiB block on disk. func smokeTreeRecs() []scanRec { f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c") f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c") return []scanRec{ - {size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"}, - {size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"}, - {size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"}, - {size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"}, - {size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"}, - {size: 100, head: f2Head, tail: f2Tail, content: f2Content, - path: "/d/t3/sub/f2renamed"}, + {size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail, + content: f1Content, path: "/d/t1/f1"}, + {size: 100, diskSize: 4096, head: f2Head, tail: f2Tail, + content: f2Content, path: "/d/t1/sub/f2"}, + {size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail, + content: f1Content, path: "/d/t2/f1"}, + {size: 100, diskSize: 4096, head: f2Head, tail: f2Tail, + content: f2Content, path: "/d/t2/sub/f2"}, + {size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail, + content: f1Content, path: "/d/t3/f1"}, + {size: 100, diskSize: 4096, head: f2Head, tail: f2Tail, + content: f2Content, path: "/d/t3/sub/f2renamed"}, } } @@ -198,13 +204,33 @@ func TestRunTreesEscapesPaths(t *testing.T) { code, exitOK, stderr.String()) } - want := "first\tdupe\tfiles\tsize\n" + - `/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n" + want := "first\tdupe\tfiles\tsize\tdisk_size\n" + + `/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\t4096\n" if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } } +// TestRunTreesDiskSize runs trees on two copies of a tree whose files +// take different space on disk. The row's disk_size is the sum over the +// dupe copy's files, those in its subdirectory included. +func TestRunTreesDiskSize(t *testing.T) { + f, g := testHash("f"), testHash("g") + recs := []scanRec{ + {size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"}, + {size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"}, + {size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"}, + {size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"}, + } + + t.Setenv(databaseEnv, seedDatabase(t, recs)) + + want := "first\tdupe\tfiles\tsize\tdisk_size\n/a\t/b\t2\t30\t1536\n" + if got := runStdout(t, cmdTrees); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} + // TestRunTreesProgressNamesEachPhase runs trees on the smoke-test // records with stderr not a terminal, and checks every line it prints // there. After the line naming the database, each phase prints a line @@ -226,7 +252,8 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) { code, exitOK, stderr()) } - report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n" + report := "first\tdupe\tfiles\tsize\tdisk_size\n" + + "/d/t1\t/d/t2\t2\t3100\t8192\n" if got := stdout.String(); got != report { t.Errorf("stdout = %q, want %q", got, report) } @@ -291,7 +318,9 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) { shown = append(shown, line[strings.LastIndex(line, "\r")+1:]) } - want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"} + want := []string{ + "first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192", + } if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) { t.Errorf("terminal shows lines %q, want %q first", shown, want) } -- 2.54.0