Record each file's size on disk and print it in trees (closes #147)
check / check (push) Canceled after 0s

The files table gets a disk_size column: the file's allocated blocks,
st_blocks × 512, from the lstat the walk already does. scan writes it
with every record it writes. trees adds it up per directory and prints
it as a column after size. On a compressed or sparse file it is less
than size, so two copies of one tree can differ; each row shows the
dupe copy's.

Judgement call: a row's disk_size is the dupe copy's, not the first's.
Unchanged files keep the disk_size from their last hash: a change in
allocation that leaves size and mtime alone is not picked up.

Model: opus-5-5
This commit is contained in:
2026-10-27 18:08:27 +00:00
parent 98193b2896
commit dfbfdd0601
10 changed files with 230 additions and 88 deletions
+21 -16
View File
@@ -47,14 +47,15 @@ const dbFilePerm = 0o644
const lockFilePerm = 0o600
// createTableSQL is the schema applied to a fresh database. Paths are
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
// second. head, tail and content each hold the raw 32 bytes of a
// SHA-256.
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
// disk_size holds the bytes the file takes on disk. mtime holds whole
// Unix seconds and mtime_nsec the nanoseconds within that second. head,
// tail and content each hold the raw 32 bytes of a SHA-256.
const createTableSQL = `
CREATE TABLE files (
path BLOB PRIMARY KEY,
size INTEGER NOT NULL,
disk_size INTEGER NOT NULL,
mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL,
head BLOB NOT NULL,
@@ -115,11 +116,13 @@ SELECT records FROM totals
// upsertSQL inserts one file record, replacing any existing record for
// the same path.
const upsertSQL = `
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
VALUES (?, ?, ?, ?, ?, ?, ?)
INSERT INTO files (
path, size, disk_size, mtime, mtime_nsec, head, tail, content
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (path) DO UPDATE SET
size = excluded.size, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
size = excluded.size, disk_size = excluded.disk_size,
mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
head = excluded.head, tail = excluded.tail,
content = excluded.content
`
@@ -536,8 +539,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
total(n)
rows, err := tx.QueryContext(ctx,
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
"FROM files ORDER BY path")
"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
"content FROM files ORDER BY path")
if err != nil {
return fmt.Errorf("read records: %w", err)
}
@@ -551,7 +554,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
r scanRec
)
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
&head, &tail, &content)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
@@ -586,10 +590,11 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
`
// loadSignatureRows gives total the number of records, then streams
// every record's signature and path to fn, without its mtime, in the
// order of signatureRowsSQL. The count and the records are read in one
// transaction, so they agree while a scan is committing. An error from
// fn stops the reading and is returned as it is.
// every record's signature and path to fn, without its mtime or its
// disk_size, in the order of signatureRowsSQL. The count and the
// records are read in one transaction, so they agree while a scan is
// committing. An error from fn stops the reading and is returned as it
// is.
func loadSignatureRows(ctx context.Context, db *sql.DB,
total func(n int64), fn func(r scanRec) error,
) error {
@@ -833,7 +838,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
defer func() { _ = st.Close() }()
for _, r := range upserts {
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
r.mtime.Unix(), r.mtime.Nanosecond(),
r.head[:], r.tail[:], r.content[:])
if err != nil {