Record each snapshot's new chunk size when its backup runs (closes #311)
check / check (push) Waiting to run
check / check (push) Waiting to run
snapshot list worked out NEW CHUNK SIZE from the snapshot's file rows. A file's row is shared by every snapshot that lists it and rewritten when the file changes, so a snapshot whose change was a modified file showed 0 B, and older snapshots' figures moved after later backups. The scanner now totals the size of the chunks it stores that were not stored before (the chunks chunk_count counts), the backup writes that total to a new snapshots column, new_chunk_size, and snapshot list reads it. Judgement call: the figure is fixed when the backup runs, so removing an earlier snapshot no longer raises a later snapshot's figure. Model: opus-5-5
This commit is contained in:
@@ -129,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
|
||||
compressionLevel int,
|
||||
uploadBytes int64,
|
||||
uploadDurationMs int64,
|
||||
newChunkSize int64,
|
||||
) error {
|
||||
compressionRatio, err := r.extendedCompressionRatio(
|
||||
ctx, tx, snapshotID, blobUncompressedSize,
|
||||
@@ -143,18 +144,19 @@ func (r *SnapshotRepository) UpdateExtendedStats(
|
||||
compression_ratio = ?,
|
||||
compression_level = ?,
|
||||
upload_bytes = ?,
|
||||
upload_duration_ms = ?
|
||||
upload_duration_ms = ?,
|
||||
new_chunk_size = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
blobUncompressedSize, compressionRatio, compressionLevel,
|
||||
uploadBytes, uploadDurationMs, snapshotID)
|
||||
uploadBytes, uploadDurationMs, newChunkSize, snapshotID)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
blobUncompressedSize, compressionRatio, compressionLevel,
|
||||
uploadBytes, uploadDurationMs, snapshotID)
|
||||
uploadBytes, uploadDurationMs, newChunkSize, snapshotID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -594,45 +596,24 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// GetSnapshotNewChunkSize returns the sum of plaintext sizes of chunks that are
|
||||
// referenced by this snapshot but not by any earlier completed snapshot known to
|
||||
// the local database. The result is the marginal uncompressed data this snapshot
|
||||
// added to the dedup pool — i.e., the delta from prior snapshots.
|
||||
// GetSnapshotNewChunkSize returns the total plaintext size of the chunks the
|
||||
// snapshot's backup stored that were not stored before, as the backup
|
||||
// recorded it. It is not derived from the snapshot's files: a file's row is
|
||||
// shared by every snapshot that lists it and is rewritten when the file
|
||||
// changes, so it shows only the file's latest chunks.
|
||||
func (r *SnapshotRepository) GetSnapshotNewChunkSize(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `
|
||||
WITH this_snap_chunks AS (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshot_files sf
|
||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
||||
WHERE sf.snapshot_id = ?
|
||||
),
|
||||
prior_chunks AS (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshots s
|
||||
JOIN snapshot_files sf ON sf.snapshot_id = s.id
|
||||
JOIN file_chunks fc ON fc.file_id = sf.file_id
|
||||
WHERE s.completed_at IS NOT NULL
|
||||
AND s.id != ?
|
||||
AND s.started_at < (SELECT started_at FROM snapshots WHERE id = ?)
|
||||
)
|
||||
SELECT COALESCE(SUM(c.size), 0)
|
||||
FROM chunks c
|
||||
JOIN this_snap_chunks t ON c.chunk_hash = t.chunk_hash
|
||||
WHERE c.chunk_hash NOT IN (SELECT chunk_hash FROM prior_chunks)
|
||||
`
|
||||
query := `SELECT new_chunk_size FROM snapshots WHERE id = ?`
|
||||
|
||||
var totalSize int64
|
||||
var newChunkSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(
|
||||
ctx, query, snapshotID, snapshotID, snapshotID,
|
||||
).Scan(&totalSize)
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&newChunkSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying new chunk size: %w", err)
|
||||
}
|
||||
|
||||
return totalSize, nil
|
||||
return newChunkSize, nil
|
||||
}
|
||||
|
||||
// GetIncompleteSnapshots returns all snapshots that haven't been completed
|
||||
|
||||
Reference in New Issue
Block a user