Record each file's size on disk and print it in trees (closes #147)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
The files table gets a disk_size column: the file's allocated blocks, st_blocks × 512, from the lstat the walk already does. scan writes it with every record it writes. trees adds it up per directory and prints it as a column after size. On a compressed or sparse file it is less than size, so two copies of one tree can differ; each row shows the dupe copy's. Judgement call: a row's disk_size is the dupe copy's, not the first's. Unchanged files keep the disk_size from their last hash: a change in allocation that leaves size and mtime alone is not picked up. Model: opus-5-5
This commit is contained in:
@@ -47,14 +47,15 @@ const dbFilePerm = 0o644
|
||||
const lockFilePerm = 0o600
|
||||
|
||||
// createTableSQL is the schema applied to a fresh database. Paths are
|
||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
||||
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
||||
// second. head, tail and content each hold the raw 32 bytes of a
|
||||
// SHA-256.
|
||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
|
||||
// disk_size holds the bytes the file takes on disk. mtime holds whole
|
||||
// Unix seconds and mtime_nsec the nanoseconds within that second. head,
|
||||
// tail and content each hold the raw 32 bytes of a SHA-256.
|
||||
const createTableSQL = `
|
||||
CREATE TABLE files (
|
||||
path BLOB PRIMARY KEY,
|
||||
size INTEGER NOT NULL,
|
||||
disk_size INTEGER NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
mtime_nsec INTEGER NOT NULL,
|
||||
head BLOB NOT NULL,
|
||||
@@ -115,11 +116,13 @@ SELECT records FROM totals
|
||||
// upsertSQL inserts one file record, replacing any existing record for
|
||||
// the same path.
|
||||
const upsertSQL = `
|
||||
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO files (
|
||||
path, size, disk_size, mtime, mtime_nsec, head, tail, content
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (path) DO UPDATE SET
|
||||
size = excluded.size, mtime = excluded.mtime,
|
||||
mtime_nsec = excluded.mtime_nsec,
|
||||
size = excluded.size, disk_size = excluded.disk_size,
|
||||
mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
|
||||
head = excluded.head, tail = excluded.tail,
|
||||
content = excluded.content
|
||||
`
|
||||
@@ -536,8 +539,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
||||
total(n)
|
||||
|
||||
rows, err := tx.QueryContext(ctx,
|
||||
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
|
||||
"FROM files ORDER BY path")
|
||||
"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
|
||||
"content FROM files ORDER BY path")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read records: %w", err)
|
||||
}
|
||||
@@ -551,7 +554,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
||||
r scanRec
|
||||
)
|
||||
|
||||
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
|
||||
err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
|
||||
&head, &tail, &content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
@@ -586,10 +590,11 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
|
||||
`
|
||||
|
||||
// loadSignatureRows gives total the number of records, then streams
|
||||
// every record's signature and path to fn, without its mtime, in the
|
||||
// order of signatureRowsSQL. The count and the records are read in one
|
||||
// transaction, so they agree while a scan is committing. An error from
|
||||
// fn stops the reading and is returned as it is.
|
||||
// every record's signature and path to fn, without its mtime or its
|
||||
// disk_size, in the order of signatureRowsSQL. The count and the
|
||||
// records are read in one transaction, so they agree while a scan is
|
||||
// committing. An error from fn stops the reading and is returned as it
|
||||
// is.
|
||||
func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||
total func(n int64), fn func(r scanRec) error,
|
||||
) error {
|
||||
@@ -833,7 +838,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
for _, r := range upserts {
|
||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
|
||||
r.mtime.Unix(), r.mtime.Nanosecond(),
|
||||
r.head[:], r.tail[:], r.content[:])
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user