From 05a2c86d65ba0f7b58fe5ec3ba7d44e1761c1952 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 23:56:22 +0200 Subject: [PATCH] Make the size defaults and the chunk_size minimum 1000-based (closes #313) The chunk_size and blob_size_limit defaults and the chunk_size minimum were 1024-based, while the size parser reads 10MB as 10,000,000 bytes and the docs and info use 1000-based units. So chunk_size: 1MB was rejected, writing out a documented default changed it, and info showed an 11 GB blob limit. They are now 1000-based like the parser. The examples below the 1MB minimum in config.example.yml and the config init template are replaced, and both now state the minimum. Judgement call: s3.part_size keeps its 5MiB to 5GiB bounds, which are S3's own limits, and MiB and GiB stay listed as accepted spellings. Model: opus-5-5 --- config.example.yml | 3 +- internal/cli/config.go | 2 +- internal/config/config.go | 10 +++-- internal/config/config_test.go | 75 ++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 6 deletions(-) diff --git a/config.example.yml b/config.example.yml index fd76d48..55dae8e 100644 --- a/config.example.yml +++ b/config.example.yml @@ -302,7 +302,8 @@ storage_url: "rclone://myremote/path/to/backups" # Average chunk size for content-defined chunking # Smaller chunks = better deduplication but more metadata -# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc. +# Minimum 1MB +# Supports: 10MB, 5M, 1GB, 64MiB, etc. # Default: 10MB #chunk_size: 10MB diff --git a/internal/cli/config.go b/internal/cli/config.go index bedfe89..5a00a2f 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -224,7 +224,7 @@ storage_url: "" # Average chunk size for content-defined chunking (FastCDC). # Smaller = better deduplication but more metadata overhead. -# Accepts: 1MB, 10M, 64KB, etc. +# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc. # Default: 10MB # chunk_size: 10MB diff --git a/internal/config/config.go b/internal/config/config.go index c9dd2ec..64778c5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,13 +30,15 @@ const appName = "vaultik" //nolint:gosec // G101: marker for detecting a pasted secret key, not a credential const secretKeyPrefix = "AGE-SECRET-KEY-" -// Defaults and validation bounds for tunable settings. +// Defaults and validation bounds for tunable settings. MB and GB are +// 1000-based, as the size parser reads them; the S3 part sizes are S3's own +// limits, which are 1024-based (MiB, GiB). const ( - defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB - defaultChunkSize = Size(10 * 1024 * 1024) // 10MB + defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB + defaultChunkSize = Size(10 * 1000 * 1000) // 10MB defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB defaultCompressionLevel = 3 - minChunkSize = 1024 * 1024 // 1MB + minChunkSize = 1000 * 1000 // 1MB minCompressionLevel = 1 maxCompressionLevel = 19 // S3 accepts a multipart upload part from 5MiB to 5GiB. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bd7abe3..897b7e7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -337,6 +337,81 @@ func TestLoadS3PartSize(t *testing.T) { }) } +// TestDocumentedSizeDefaults checks that the defaults are the documented +// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values +// (1000-based), so writing a default out in the config file changes nothing. +func TestDocumentedSizeDefaults(t *testing.T) { + t.Parallel() + + cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n")) + if err != nil { + t.Fatalf("Parse() unexpected error: %v", err) + } + + if cfg.ChunkSize != defaultChunkSize { + t.Errorf("chunk_size: 10MB is %d bytes, the default is %d", + cfg.ChunkSize, defaultChunkSize) + } + + if cfg.BlobSizeLimit != defaultBlobSizeLimit { + t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d", + cfg.BlobSizeLimit, defaultBlobSizeLimit) + } +} + +// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that +// every size the README, config.example.yml and the config init template +// list as an example is accepted. +func TestDocumentedSizeExamples(t *testing.T) { + t.Parallel() + + const withoutSizes = "snapshots:\n" + + " test:\n" + + " paths: [/tmp/vaultik-test-source]\n" + + "storage_url: file:///tmp/vaultik-test-storage\n" + + tests := []struct { + setting string + wantErr error + }{ + {"chunk_size: 1MB", nil}, + {"chunk_size: 999999", errChunkSizeTooSmall}, + {"chunk_size: 5M", nil}, + {"chunk_size: 10M", nil}, + {"chunk_size: 10MB", nil}, + {"chunk_size: 64MiB", nil}, + {"chunk_size: 1GB", nil}, + // Four times the default chunk_size, the documented minimum. + {"blob_size_limit: 40MB", nil}, + {"blob_size_limit: 500MB", nil}, + {"blob_size_limit: 1GB", nil}, + {"blob_size_limit: 1GiB", nil}, + {"blob_size_limit: 10G", nil}, + {"blob_size_limit: 10GB", nil}, + {"s3: {part_size: 5MiB}", nil}, + {"s3: {part_size: 10MB}", nil}, + {"s3: {part_size: 16MiB}", nil}, + {"s3: {part_size: 100MiB}", nil}, + {"s3: {part_size: 5GiB}", nil}, + } + + for _, tt := range tests { + t.Run(tt.setting, func(t *testing.T) { + t.Parallel() + + cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n")) + if err != nil { + t.Fatalf("Parse() unexpected error: %v", err) + } + + err = cfg.Validate() + if !errors.Is(err, tt.wantErr) { + t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr) + } + }) + } +} + // TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a // relative path fails at load. Accepted, it would resolve against the // working directory, so a backup run from another directory would write