Pass s3.part_size to the multipart uploader (closes #232)
check / check (push) Waiting to run

s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, an explicit 0
included. A blob too large for S3's limit of 10,000 parts at that size
is uploaded in larger parts, since the uploader cannot learn the size of
the reader it is given. The docs gave the default as 5MB, which the
config file reads as 5,000,000 bytes, below the minimum; they now say
5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.

Model: opus-5-5
This commit is contained in:
2026-10-07 11:52:50 +00:00
parent 3fc8a8f2f4
commit 28299a3f4f
11 changed files with 322 additions and 18 deletions
+116
View File
@@ -166,6 +166,7 @@ func TestValidateBlobSizeLimit(t *testing.T) {
ChunkSize: chunkSize,
BlobSizeLimit: blobLimit,
CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
}
}
@@ -221,6 +222,120 @@ func TestValidateBlobSizeLimit(t *testing.T) {
}
}
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
// S3 accepts, 5MiB to 5GiB, by changing only the part size of the test
// config. "5MB" in the config file is 5,000,000 bytes, below the minimum.
func TestValidateS3PartSize(t *testing.T) {
t.Parallel()
base, err := Load(os.Getenv("VAULTIK_CONFIG"))
if err != nil {
t.Fatalf("Failed to load config: %v", err)
}
tests := []struct {
name string
partSize Size
wantErr bool
}{
{
name: "5MB is rejected",
partSize: 5_000_000,
wantErr: true,
},
{
name: "one byte below 5MiB is rejected",
partSize: minS3PartSize - 1,
wantErr: true,
},
{
name: "5MiB is accepted",
partSize: minS3PartSize,
wantErr: false,
},
{
name: "5GiB is accepted",
partSize: maxS3PartSize,
wantErr: false,
},
{
name: "one byte above 5GiB is rejected",
partSize: maxS3PartSize + 1,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
cfg := *base
cfg.S3.PartSize = tt.partSize
err := cfg.Validate()
if tt.wantErr {
if !errors.Is(err, errBadS3PartSize) {
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
}
return
}
if err != nil {
t.Fatalf("Validate() unexpected error: %v", err)
}
})
}
}
// TestLoadS3PartSize checks that a config file without s3.part_size loads
// with the 5MiB default, and that an explicit 0 fails at load like any other
// part size S3 refuses.
func TestLoadS3PartSize(t *testing.T) {
t.Parallel()
const withoutPartSize = "snapshots:\n" +
" test:\n" +
" paths: [/tmp/vaultik-test-source]\n" +
"storage_url: file:///tmp/vaultik-test-storage\n"
writeConfig := func(t *testing.T, text string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(text), 0o600)
if err != nil {
t.Fatalf("write config: %v", err)
}
return path
}
t.Run("absent loads as 5MiB", func(t *testing.T) {
t.Parallel()
cfg, err := Load(writeConfig(t, withoutPartSize))
if err != nil {
t.Fatalf("Load() unexpected error: %v", err)
}
if cfg.S3.PartSize != defaultS3PartSize {
t.Errorf("s3.part_size = %d, want %d",
cfg.S3.PartSize, defaultS3PartSize)
}
})
t.Run("0 is rejected", func(t *testing.T) {
t.Parallel()
_, err := Load(writeConfig(t, withoutPartSize+"s3:\n part_size: 0\n"))
if !errors.Is(err, errBadS3PartSize) {
t.Fatalf("Load() error = %v, want errBadS3PartSize", err)
}
})
}
// TestValidateAgeRecipients checks that recipients are parsed at config load
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
// least of all a pasted secret key — is echoed in the error. An empty list
@@ -236,6 +351,7 @@ func TestValidateAgeRecipients(t *testing.T) {
ChunkSize: Size(10 * 1024 * 1024),
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
}
}