Record the stored size of a blob a backup finds already stored (closes #300)
check / check (push) Failing after 3m37s

When a finished blob already existed at the destination, the upload was
skipped but the blob row, and so the manifest, kept the size of the
local copy. That copy differs from the stored object when
compression_level or the recipients changed since it was written, so
shallow and deep verify failed on a good snapshot. Marking a blob
uploaded now records the size of the object in storage: for an existing
blob, the size Stat reports.

Judgement call: the test adds a recipient rather than changing the
compression level, because an extra age recipient always makes the
local copy larger, while two zstd levels need not compress the test
data differently.

Model: opus-5-5
This commit is contained in:
2026-10-13 01:26:19 +00:00
parent c9a519c233
commit cf146db51e
6 changed files with 126 additions and 20 deletions
+16 -10
View File
@@ -1509,7 +1509,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
// snapshot references has uploaded_ts set, so deduplication and
// interrupted-run repair treat these blobs as trustworthy.
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
err := s.repos.Blobs.UpdateUploaded(ctx, tx, b.ID)
err := s.repos.Blobs.UpdateUploaded(
ctx, tx, b.ID, b.Compressed)
if err != nil {
return fmt.Errorf("marking blob uploaded: %w", err)
}
@@ -1545,7 +1546,7 @@ func (s *Scanner) handleBlobReady(
blobPath := fmt.Sprintf("blobs/%s/%s/%s",
finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
blobExists, err := s.uploadBlobIfNeeded(
storedSize, blobExists, err := s.uploadBlobIfNeeded(
ctx, blobPath, blobWithReader, startTime, result)
if err != nil {
s.cleanupBlobTempFile(blobWithReader)
@@ -1553,7 +1554,8 @@ func (s *Scanner) handleBlobReady(
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
}
err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime)
err = s.recordBlobMetadata(
ctx, finishedBlob, storedSize, blobExists, startTime)
if err != nil {
s.cleanupBlobTempFile(blobWithReader)
@@ -1575,20 +1577,22 @@ func (s *Scanner) handleBlobReady(
}
// uploadBlobIfNeeded uploads the blob to storage if it doesn't already
// exist, returns whether it existed
// exist. It returns the size of the object in storage, which for an
// existing object can differ from the local copy's, and whether it
// already existed.
func (s *Scanner) uploadBlobIfNeeded(
ctx context.Context,
blobPath string,
blobWithReader *blob.WithReader,
startTime time.Time,
result *ScanResult,
) (bool, error) {
) (int64, bool, error) {
finishedBlob := blobWithReader.FinishedBlob
// Check if blob already exists (deduplication after restart)
destination := s.storage.Info().Location
_, err := s.storage.Stat(ctx, blobPath)
stored, err := s.storage.Stat(ctx, blobPath)
if err == nil {
log.Info("Blob already exists in storage, skipping upload",
"hash", finishedBlob.Hash,
@@ -1597,7 +1601,7 @@ func (s *Scanner) uploadBlobIfNeeded(
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
s.ui.Path(destination))
return true, nil
return stored.Size, true, nil
}
s.ui.Beginf("Uploading blob %s (%s) to %s.",
@@ -1611,7 +1615,7 @@ func (s *Scanner) uploadBlobIfNeeded(
if err != nil {
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
return false, fmt.Errorf("uploading blob to storage: %w", err)
return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
}
uploadDuration := time.Since(startTime)
@@ -1642,7 +1646,7 @@ func (s *Scanner) uploadBlobIfNeeded(
stats.BytesUploaded.Add(finishedBlob.Compressed)
}
return false, nil
return finishedBlob.Compressed, false, nil
}
// makeUploadProgressCallback creates a progress callback for blob uploads.
@@ -1712,6 +1716,7 @@ func (s *Scanner) makeUploadProgressCallback(
func (s *Scanner) recordBlobMetadata(
ctx context.Context,
finishedBlob *blob.FinishedBlob,
storedSize int64,
blobExists bool,
startTime time.Time,
) error {
@@ -1723,7 +1728,8 @@ func (s *Scanner) recordBlobMetadata(
uploadDuration := time.Since(startTime)
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
err := s.repos.Blobs.UpdateUploaded(
txCtx, tx, finishedBlob.ID, storedSize)
if err != nil {
return fmt.Errorf("updating blob upload timestamp: %w", err)
}