Write rclone uploads under a temporary name, re-upload short blobs (closes #266)

The rclone backend wrote each object straight to its key, so killing an
upload to a local or sftp remote left a truncated object there. The next
backup found the key, skipped the upload and recorded a snapshot that
could not be restored.

On a remote where rclone says a file can be seen while it is still being
written, an object is now written under a name ending in `.partial` and
moved onto its key with the remote's server-side move, as rclone's own
copy does; listings skip such names. A backup also uploads a blob again
when the stored object's size differs from the blob's.

Judgement call: the temporary name is used only where rclone sets its
PartialUploads feature; other remotes already show an object only once
complete.

Model: opus-5-5

internal/storage/rclone.go

@@ -3,6 +3,7 @@ package storage
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"errors"
 	"fmt"
 	"io"
@@ -68,14 +69,7 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro
 		return fmt.Errorf("reading data: %w", err)
 	}
 
-	// Upload the object
-	_, err = operations.Rcat(ctx, r.fsys, key,
-		io.NopCloser(bytes.NewReader(buf)), time.Now(), nil)
-	if err != nil {
-		return fmt.Errorf("uploading object: %w", err)
-	}
-
-	return nil
+	return r.upload(ctx, key, bytes.NewReader(buf))
 }
 
 // PutWithProgress stores data with progress reporting.
@@ -89,13 +83,7 @@ func (r *RcloneStorer) PutWithProgress(
 		callback: progress,
 	}
 
-	// Upload the object
-	_, err := operations.Rcat(ctx, r.fsys, key, io.NopCloser(pr), time.Now(), nil)
-	if err != nil {
-		return fmt.Errorf("uploading object: %w", err)
-	}
-
-	return nil
+	return r.upload(ctx, key, pr)
 }
 
 // Get retrieves data from the specified key.
@@ -173,6 +161,10 @@ func (r *RcloneStorer) List(ctx context.Context, prefix string) ([]string, error
 
 	err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) {
 		key := obj.Remote()
+		if strings.HasSuffix(key, tempSuffix) {
+			return
+		}
+
 		if prefix == "" || strings.HasPrefix(key, prefix) {
 			keys = append(keys, key)
 		}
@@ -202,6 +194,10 @@ func (r *RcloneStorer) ListStream(
 			}
 
 			key := obj.Remote()
+			if strings.HasSuffix(key, tempSuffix) {
+				return
+			}
+
 			if prefix == "" || strings.HasPrefix(key, prefix) {
 				ch <- ObjectInfo{
 					Key:  key,
@@ -230,6 +226,45 @@ func (r *RcloneStorer) Info() Info {
 	}
 }
 
+// upload writes data to key. On a remote where rclone says a file can be
+// seen while it is still being written (its PartialUploads feature: local,
+// sftp, ftp, smb and others), it writes under a temporary name ending in
+// tempSuffix and moves the object onto key once it is complete, as
+// rclone's own copy does, so a killed upload cannot leave a truncated
+// object at key. List and ListStream skip a temporary object left behind.
+func (r *RcloneStorer) upload(ctx context.Context, key string, data io.Reader) error {
+	features := r.fsys.Features()
+	if !features.PartialUploads || features.Move == nil {
+		_, err := operations.Rcat(ctx, r.fsys, key, io.NopCloser(data), time.Now(), nil)
+		if err != nil {
+			return fmt.Errorf("uploading object: %w", err)
+		}
+
+		return nil
+	}
+
+	tempKey := key + "-" + rand.Text() + tempSuffix
+
+	obj, err := operations.Rcat(ctx, r.fsys, tempKey, io.NopCloser(data), time.Now(), nil)
+	if err != nil {
+		// Rcat returns the object it wrote when the written data fails its check.
+		if obj != nil {
+			_ = obj.Remove(ctx)
+		}
+
+		return fmt.Errorf("uploading object: %w", err)
+	}
+
+	_, err = features.Move(ctx, obj, key)
+	if err != nil {
+		_ = obj.Remove(ctx)
+
+		return fmt.Errorf("moving object into place: %w", err)
+	}
+
+	return nil
+}
+
 // progressReader wraps an io.Reader to track read progress.
 type progressReader struct {
 	reader   io.Reader