Pass s3.part_size to the multipart uploader (closes #232)

s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, an explicit 0
included. A blob too large for S3's limit of 10,000 parts at that size
is uploaded in larger parts, since the uploader cannot learn the size of
the reader it is given. The docs gave the default as 5MB, which the
config file reads as 5,000,000 bytes, below the minimum; they now say
5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.

Model: opus-5-5

TODO.md

@@ -27,9 +27,10 @@ the tag exists and is exercised; what is left is merging `next` to
   loaded and defaulted but never passed to the S3 client, whose uploader
   used a fixed 10MiB part. It now reaches the uploader for `storage_url`
   and for the `s3.*` fields, and a part size S3 refuses, below 5MiB or
-  above 5GiB, fails at config load. The docs gave the default as `5MB`,
+  above 5GiB, `0` included, fails at config load. A blob too large for
-  which the config file reads as 5,000,000 bytes, below the minimum; they
+  S3's limit of 10,000 parts at the configured size is uploaded in larger
-  now say `5MiB`.
+  parts. The docs gave the default as `5MB`, which the config file reads
+  as 5,000,000 bytes, below the minimum; they now say `5MiB`.
 
 - 2026-10-07: Made per-name retention work when the hostname contains `_`
   ([issue #230](https://git.eeqj.de/sneak/vaultik/issues/230)). A

config.example.yml

@@ -288,6 +288,7 @@ storage_url: "rclone://myremote/path/to/backups"
 #
 #  # Part size for multipart uploads
 #  # Minimum 5MiB, maximum 5GiB; affects memory usage during upload
+#  # A blob too large for 10,000 parts of this size gets larger parts
 #  # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum)
 #  # Default: 5MiB
 #  #part_size: 5MiB

internal/config/config.go

@@ -248,6 +248,7 @@ func Load(path string) (*Config, error) {
 		ChunkSize:        defaultChunkSize,
 		IndexPath:        filepath.Join(xdg.DataHome, appName, "index.sqlite"),
 		CompressionLevel: defaultCompressionLevel,
+		S3:               S3Config{PartSize: defaultS3PartSize},
 	}
 
 	// Convert smartconfig data to YAML then unmarshal
@@ -298,10 +299,6 @@ func Load(path string) (*Config, error) {
 		cfg.S3.Region = "us-east-1"
 	}
 
-	if cfg.S3.PartSize == 0 {
-		cfg.S3.PartSize = defaultS3PartSize
-	}
-
 	// Check config file permissions (warn if world or group readable)
 	//nolint:gosec // G703: config path is operator-supplied by design
 	info, statErr := os.Stat(path)

internal/config/config_test.go

@@ -288,6 +288,54 @@ func TestValidateS3PartSize(t *testing.T) {
 	}
 }
 
+// TestLoadS3PartSize checks that a config file without s3.part_size loads
+// with the 5MiB default, and that an explicit 0 fails at load like any other
+// part size S3 refuses.
+func TestLoadS3PartSize(t *testing.T) {
+	t.Parallel()
+
+	const withoutPartSize = "snapshots:\n" +
+		"  test:\n" +
+		"    paths: [/tmp/vaultik-test-source]\n" +
+		"storage_url: file:///tmp/vaultik-test-storage\n"
+
+	writeConfig := func(t *testing.T, text string) string {
+		t.Helper()
+
+		path := filepath.Join(t.TempDir(), "config.yml")
+
+		err := os.WriteFile(path, []byte(text), 0o600)
+		if err != nil {
+			t.Fatalf("write config: %v", err)
+		}
+
+		return path
+	}
+
+	t.Run("absent loads as 5MiB", func(t *testing.T) {
+		t.Parallel()
+
+		cfg, err := Load(writeConfig(t, withoutPartSize))
+		if err != nil {
+			t.Fatalf("Load() unexpected error: %v", err)
+		}
+
+		if cfg.S3.PartSize != defaultS3PartSize {
+			t.Errorf("s3.part_size = %d, want %d",
+				cfg.S3.PartSize, defaultS3PartSize)
+		}
+	})
+
+	t.Run("0 is rejected", func(t *testing.T) {
+		t.Parallel()
+
+		_, err := Load(writeConfig(t, withoutPartSize+"s3:\n  part_size: 0\n"))
+		if !errors.Is(err, errBadS3PartSize) {
+			t.Fatalf("Load() error = %v, want errBadS3PartSize", err)
+		}
+	})
+}
+
 // TestValidateAgeRecipients checks that recipients are parsed at config load
 // (a bad entry fails immediately, not mid-backup) and that no invalid entry —
 // least of all a pasted secret key — is echoed in the error. An empty list

internal/s3/client.go

@@ -44,6 +44,7 @@ type Config struct {
 	SecretAccessKey string
 	Region          string
 	// PartSize is the size in bytes of each part of a multipart upload.
+	// An upload too large for S3's limit of 10,000 parts gets larger parts.
 	PartSize int64
 }
 
@@ -130,7 +131,7 @@ func (c *Client) PutObjectWithProgress(
 
 	// Create an uploader with the S3 client
 	uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
-		u.PartSize = c.partSize
+		u.PartSize = uploadPartSize(c.partSize, size)
 	})
 
 	// Create a progress reader that tracks upload progress
@@ -151,6 +152,21 @@ func (c *Client) PutObjectWithProgress(
 	return err
 }
 
+// uploadPartSize returns the part size for an upload of size bytes: the
+// configured part size (the SDK default when zero), raised where needed so
+// the upload fits in S3's limit of 10,000 parts. The uploader cannot raise
+// it itself, because it cannot seek the progress reader to learn its size.
+func uploadPartSize(configured, size int64) int64 {
+	if configured == 0 {
+		configured = manager.DefaultUploadPartSize
+	}
+
+	maxParts := int64(manager.MaxUploadParts)
+	smallestThatFits := (size + maxParts - 1) / maxParts // rounded up
+
+	return max(configured, smallestThatFits)
+}
+
 // GetObject downloads an object from S3 with the specified key.
 // The key is automatically prefixed with the configured prefix.
 // Returns a ReadCloser containing the object data. The caller must

internal/s3/client_internal_test.go

@@ -0,0 +1,55 @@
+package s3
+
+import "testing"
+
+// TestUploadPartSize checks that an upload too large for 10,000 parts of the
+// configured size gets parts just large enough to fit in 10,000.
+func TestUploadPartSize(t *testing.T) {
+	t.Parallel()
+
+	const mib = 1024 * 1024
+
+	tests := []struct {
+		name       string
+		configured int64
+		size       int64
+		want       int64
+	}{
+		{
+			name:       "an upload that fits keeps the configured size",
+			configured: 5 * mib,
+			size:       10 * 1024 * mib,
+			want:       5 * mib,
+		},
+		{
+			name:       "exactly 10,000 parts keeps the configured size",
+			configured: 6 * mib,
+			size:       10_000 * 6 * mib,
+			want:       6 * mib,
+		},
+		{
+			name:       "one byte more than 10,000 parts adds a byte to each",
+			configured: 6 * mib,
+			size:       10_000*6*mib + 1,
+			want:       6*mib + 1,
+		},
+		{
+			name:       "zero means the SDK default of 5MiB",
+			configured: 0,
+			size:       1,
+			want:       5 * mib,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := uploadPartSize(tt.configured, tt.size)
+			if got != tt.want {
+				t.Errorf("uploadPartSize(%d, %d) = %d, want %d",
+					tt.configured, tt.size, got, tt.want)
+			}
+		})
+	}
+}

test/config.yaml

@@ -19,7 +19,7 @@ s3:
   secret_access_key: test-secret-key
   region: us-east-1
   use_ssl: true
-  part_size: 5242880  # 5MB
+  part_size: 5242880  # 5MiB
 index_path: /tmp/vaultik-test.sqlite
 chunk_size: 10MB
 blob_size_limit: 10GB