Pass s3.part_size to the multipart uploader (closes #232)

s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, an explicit 0
included. A blob too large for S3's limit of 10,000 parts at that size
is uploaded in larger parts, since the uploader cannot learn the size of
the reader it is given. The docs gave the default as 5MB, which the
config file reads as 5,000,000 bytes, below the minimum; they now say
5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.

Model: opus-5-5

internal/s3/client.go

@@ -27,10 +27,12 @@ type Client struct {
 	bucket   string
 	prefix   string
 	endpoint string
+	partSize int64
 }
 
 // Config contains S3 client configuration.
-// All fields are required except Prefix, which defaults to an empty string.
+// All fields are required except Prefix, which defaults to an empty string,
+// and PartSize, where zero means the SDK default of 5 MiB.
 // A non-empty Prefix is joined to every key with one "/", whether or not
 // it ends with one.
 // The Endpoint field should include the protocol (http:// or https://).
@@ -41,6 +43,9 @@ type Config struct {
 	AccessKeyID     string
 	SecretAccessKey string
 	Region          string
+	// PartSize is the size in bytes of each part of a multipart upload.
+	// An upload too large for S3's limit of 10,000 parts gets larger parts.
+	PartSize int64
 }
 
 // nopLogger is a logger that discards all output.
@@ -90,6 +95,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		bucket:   cfg.Bucket,
 		prefix:   prefix,
 		endpoint: cfg.Endpoint,
+		partSize: cfg.PartSize,
 	}, nil
 }
 
@@ -123,12 +129,9 @@ func (c *Client) PutObjectWithProgress(
 ) error {
 	fullKey := c.prefix + key
 
-	// uploadPartSize is 10MB for better progress granularity.
-	const uploadPartSize = 10 * 1024 * 1024
-
 	// Create an uploader with the S3 client
 	uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
-		u.PartSize = uploadPartSize
+		u.PartSize = uploadPartSize(c.partSize, size)
 	})
 
 	// Create a progress reader that tracks upload progress
@@ -149,6 +152,21 @@ func (c *Client) PutObjectWithProgress(
 	return err
 }
 
+// uploadPartSize returns the part size for an upload of size bytes: the
+// configured part size (the SDK default when zero), raised where needed so
+// the upload fits in S3's limit of 10,000 parts. The uploader cannot raise
+// it itself, because it cannot seek the progress reader to learn its size.
+func uploadPartSize(configured, size int64) int64 {
+	if configured == 0 {
+		configured = manager.DefaultUploadPartSize
+	}
+
+	maxParts := int64(manager.MaxUploadParts)
+	smallestThatFits := (size + maxParts - 1) / maxParts // rounded up
+
+	return max(configured, smallestThatFits)
+}
+
 // GetObject downloads an object from S3 with the specified key.
 // The key is automatically prefixed with the configured prefix.
 // Returns a ReadCloser containing the object data. The caller must

internal/s3/client_internal_test.go

@@ -0,0 +1,55 @@
+package s3
+
+import "testing"
+
+// TestUploadPartSize checks that an upload too large for 10,000 parts of the
+// configured size gets parts just large enough to fit in 10,000.
+func TestUploadPartSize(t *testing.T) {
+	t.Parallel()
+
+	const mib = 1024 * 1024
+
+	tests := []struct {
+		name       string
+		configured int64
+		size       int64
+		want       int64
+	}{
+		{
+			name:       "an upload that fits keeps the configured size",
+			configured: 5 * mib,
+			size:       10 * 1024 * mib,
+			want:       5 * mib,
+		},
+		{
+			name:       "exactly 10,000 parts keeps the configured size",
+			configured: 6 * mib,
+			size:       10_000 * 6 * mib,
+			want:       6 * mib,
+		},
+		{
+			name:       "one byte more than 10,000 parts adds a byte to each",
+			configured: 6 * mib,
+			size:       10_000*6*mib + 1,
+			want:       6*mib + 1,
+		},
+		{
+			name:       "zero means the SDK default of 5MiB",
+			configured: 0,
+			size:       1,
+			want:       5 * mib,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := uploadPartSize(tt.configured, tt.size)
+			if got != tt.want {
+				t.Errorf("uploadPartSize(%d, %d) = %d, want %d",
+					tt.configured, tt.size, got, tt.want)
+			}
+		})
+	}
+}

internal/s3/module.go

@@ -28,6 +28,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) {
 		AccessKeyID:     cfg.S3.AccessKeyID,
 		SecretAccessKey: cfg.S3.SecretAccessKey,
 		Region:          cfg.S3.Region,
+		PartSize:        cfg.S3.PartSize.Int64(),
 	})
 	if err != nil {
 		return nil, err