Pass s3.part_size to the multipart uploader (closes #232)
check / check (push) Waiting to run

s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, the part sizes S3
accepts. The docs gave the default as 5MB, which the config file reads
as 5,000,000 bytes, below the minimum; they now say 5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.
Trap: at the 5MiB default the uploader's 10,000-part limit caps one
upload at about 48.8GiB, down from about 97.7GiB; blob_size_limit is
not checked against it.

Model: opus-5-5
This commit is contained in:
2026-10-07 09:18:31 +00:00
parent 7696f83258
commit 6e3d2e0941
9 changed files with 193 additions and 11 deletions
+71
View File
@@ -166,6 +166,7 @@ func TestValidateBlobSizeLimit(t *testing.T) {
ChunkSize: chunkSize,
BlobSizeLimit: blobLimit,
CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
}
}
@@ -221,6 +222,75 @@ func TestValidateBlobSizeLimit(t *testing.T) {
}
}
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
// S3 accepts, 5MiB to 5GiB. "5MB" in the config file is 5,000,000 bytes,
// below the minimum.
func TestValidateS3PartSize(t *testing.T) {
t.Parallel()
newConfig := func(partSize Size) *Config {
return &Config{
Snapshots: map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}},
StorageURL: "file:///tmp/vaultik-test-store",
ChunkSize: defaultChunkSize,
BlobSizeLimit: defaultBlobSizeLimit,
CompressionLevel: defaultCompressionLevel,
S3: S3Config{PartSize: partSize},
}
}
tests := []struct {
name string
partSize Size
wantErr bool
}{
{
name: "5MB is rejected",
partSize: 5_000_000,
wantErr: true,
},
{
name: "one byte below 5MiB is rejected",
partSize: minS3PartSize - 1,
wantErr: true,
},
{
name: "5MiB is accepted",
partSize: minS3PartSize,
wantErr: false,
},
{
name: "5GiB is accepted",
partSize: maxS3PartSize,
wantErr: false,
},
{
name: "one byte above 5GiB is rejected",
partSize: maxS3PartSize + 1,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
err := newConfig(tt.partSize).Validate()
if tt.wantErr {
if !errors.Is(err, errBadS3PartSize) {
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
}
return
}
if err != nil {
t.Fatalf("Validate() unexpected error: %v", err)
}
})
}
}
// TestValidateAgeRecipients checks that recipients are parsed at config load
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
// least of all a pasted secret key — is echoed in the error. An empty list
@@ -236,6 +306,7 @@ func TestValidateAgeRecipients(t *testing.T) {
ChunkSize: Size(10 * 1024 * 1024),
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
}
}