Write rclone uploads under a temporary name, re-upload short blobs (closes #266)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
The rclone backend wrote each object straight to its key, so killing an upload to a local or sftp remote left a truncated object there. The next backup found the key, skipped the upload and recorded a snapshot that could not be restored. On a remote where rclone says a file can be seen while it is still being written, an object is now written under a name ending in `.partial` and moved onto its key with the remote's server-side move, as rclone's own copy does; listings skip such names. A backup also uploads a blob again when the stored object's size differs from the blob's. Judgement call: the temporary name is used only where rclone sets its PartialUploads feature; other remotes already show an object only once complete. Model: opus-5-5
This commit is contained in:
@@ -1535,8 +1535,8 @@ func (s *Scanner) handleBlobReady(
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadBlobIfNeeded uploads the blob to storage if it doesn't already
|
||||
// exist, returns whether it existed
|
||||
// uploadBlobIfNeeded uploads the blob to storage unless an object of the
|
||||
// blob's size is already there, and returns whether it was there
|
||||
func (s *Scanner) uploadBlobIfNeeded(
|
||||
ctx context.Context,
|
||||
blobPath string,
|
||||
@@ -1546,11 +1546,13 @@ func (s *Scanner) uploadBlobIfNeeded(
|
||||
) (bool, error) {
|
||||
finishedBlob := blobWithReader.FinishedBlob
|
||||
|
||||
// Check if blob already exists (deduplication after restart)
|
||||
// Check if blob already exists (deduplication after restart). An
|
||||
// object of another size was left by an upload cut off part-way and
|
||||
// is uploaded again.
|
||||
destination := s.storage.Info().Location
|
||||
|
||||
_, err := s.storage.Stat(ctx, blobPath)
|
||||
if err == nil {
|
||||
stored, err := s.storage.Stat(ctx, blobPath)
|
||||
if err == nil && stored.Size == finishedBlob.Compressed {
|
||||
log.Info("Blob already exists in storage, skipping upload",
|
||||
"hash", finishedBlob.Hash,
|
||||
"size", humanize.Bytes(safeUint64(finishedBlob.Compressed)))
|
||||
@@ -1561,6 +1563,11 @@ func (s *Scanner) uploadBlobIfNeeded(
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
log.Warn("Blob in storage has the wrong size, uploading it again",
|
||||
"hash", finishedBlob.Hash, "stored_size", stored.Size)
|
||||
}
|
||||
|
||||
s.ui.Beginf("Uploading blob %s (%s) to %s.",
|
||||
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
|
||||
s.ui.Path(destination))
|
||||
|
||||
Reference in New Issue
Block a user