From de6afbaf39084011ec57b2c0d7f4cd012b6f5d70 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 22 Sep 2026 09:19:41 +0000 Subject: [PATCH] Reject a blob_size_limit below the largest possible chunk (closes #167) Validate only rejected blob_size_limit below chunk_size, but the chunker can emit chunks up to chunk_size times the FastCDC size spread (four times), and the packer puts a single chunk of any size into an otherwise empty blob. A limit between one and four times chunk_size therefore let a blob reach four times the configured maximum, with most blobs holding a single chunk and so exposing individual chunk lengths to anyone who can list the destination. Validate now rejects blob_size_limit below chunk_size times the spread, reusing the chunker's one constant (now exported as ChunkSizeSpread) instead of a second literal. The rule is stated in the error text, the Validate comment, the README config table, config.example.yml, and the generated config template. Model: opus-4-8 --- README.md | 2 +- config.example.yml | 2 + internal/chunker/chunker.go | 9 ++-- internal/cli/config.go | 2 + internal/config/config.go | 33 ++++++++++----- internal/config/config_test.go | 77 ++++++++++++++++++++++++++++++++++ 6 files changed, 109 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ab1c9f0..06fcb65 100644 --- a/README.md +++ b/README.md @@ -518,7 +518,7 @@ complete annotated example also lives in | `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) | | `exclude` | | Global exclude patterns (applied to all snapshots) | | `chunk_size` | `10MB` | Average chunk size for content-defined chunking | -| `blob_size_limit` | `10GB` | Maximum blob size before splitting | +| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit | | `compression_level` | `3` | zstd compression level (1-19) | | `hostname` | system hostname | Hostname used in snapshot IDs | | `index_path` | platform data dir | Local SQLite index path | diff --git a/config.example.yml b/config.example.yml index 433d844..b023b5a 100644 --- a/config.example.yml +++ b/config.example.yml @@ -304,6 +304,8 @@ storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion" # Maximum blob size # Multiple chunks are packed into blobs up to this size +# Must be at least four times chunk_size (the largest chunk the chunker can +# emit); a smaller limit would let a single-chunk blob exceed it. # Supports: 1GB, 10G, 500MB, 1GiB, etc. # Default: 10GB #blob_size_limit: 10GB diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index c5cee7b..559983a 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -33,9 +33,10 @@ type Chunker struct { maxChunkSize int } -// chunkSizeSpread is the FastCDC-recommended factor between the average +// ChunkSizeSpread is the FastCDC-recommended factor between the average // chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes. -const chunkSizeSpread = 4 +// The largest chunk the chunker can emit is therefore avg*ChunkSizeSpread. +const ChunkSizeSpread = 4 // NewChunker creates a new chunker with the specified average chunk size. // The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4 @@ -45,8 +46,8 @@ func NewChunker(avgChunkSize int64) *Chunker { // FastCDC recommends min = avg/4 and max = avg*4 return &Chunker{ avgChunkSize: int(avgChunkSize), - minChunkSize: int(avgChunkSize / chunkSizeSpread), - maxChunkSize: int(avgChunkSize * chunkSizeSpread), + minChunkSize: int(avgChunkSize / ChunkSizeSpread), + maxChunkSize: int(avgChunkSize * ChunkSizeSpread), } } diff --git a/internal/cli/config.go b/internal/cli/config.go index 212fb5a..066b32d 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -212,6 +212,8 @@ storage_url: "" # chunk_size: 10MB # Maximum blob size before splitting into a new blob. +# Must be at least four times chunk_size (the largest chunk the chunker can +# emit); a smaller limit would let a single-chunk blob exceed it. # Accepts: 1GB, 10G, 500MB, etc. # Default: 10GB # blob_size_limit: 10GB diff --git a/internal/config/config.go b/internal/config/config.go index 4e9fcc8..a103192 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,6 +16,7 @@ import ( "github.com/adrg/xdg" "go.uber.org/fx" "gopkg.in/yaml.v3" + "sneak.berlin/go/vaultik/internal/chunker" "sneak.berlin/go/vaultik/internal/log" ) @@ -41,9 +42,11 @@ var ( "at least one snapshot must be configured (see config.example.yml)") errSnapshotNoPaths = errors.New("snapshot must have at least one path") errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB") - errBlobSizeTooSmall = errors.New("blob_size_limit must be at least chunk_size") - errBadCompression = errors.New("compression_level must be between 1 and 19") - errBadStorageScheme = errors.New( + errBlobSizeTooSmall = errors.New( + "blob_size_limit must be at least the largest chunk the chunker can " + + "emit (chunk_size times the FastCDC size spread)") + errBadCompression = errors.New("compression_level must be between 1 and 19") + errBadStorageScheme = errors.New( "storage_url must start with s3://, file://, or rclone://") errStorageNotConfigured = errors.New( "storage not configured; set storage_url or provide s3.endpoint + " + @@ -287,12 +290,15 @@ func Load(path string) (*Config, error) { // Validate checks if the configuration is valid and complete. // It ensures all required fields are present and have valid values: -// - At least one age recipient must be specified -// - At least one snapshot must be configured with at least one path -// - Storage must be configured (either storage_url or s3.* fields) -// - Chunk size must be at least 1MB -// - Blob size limit must be at least the chunk size -// - Compression level must be between 1 and 19 +// - At least one age recipient must be specified +// - At least one snapshot must be configured with at least one path +// - Storage must be configured (either storage_url or s3.* fields) +// - Chunk size must be at least 1MB +// - Blob size limit must be at least the largest chunk the chunker can emit +// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never +// exceeds the configured limit +// - Compression level must be between 1 and 19 +// // Returns an error describing the first validation failure encountered. func (c *Config) Validate() error { if len(c.AgeRecipients) == 0 { @@ -319,8 +325,13 @@ func (c *Config) Validate() error { return errChunkSizeTooSmall } - if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() { - return errBlobSizeTooSmall + // The chunker can emit chunks up to chunk_size * ChunkSizeSpread, and the + // packer places a single such chunk into an otherwise empty blob. A limit + // below that bound would let a blob exceed it, so reject it. + largestChunk := c.ChunkSize.Int64() * chunker.ChunkSizeSpread + if c.BlobSizeLimit.Int64() < largestChunk { + return fmt.Errorf("%w: need at least %d bytes", + errBlobSizeTooSmall, largestChunk) } if c.CompressionLevel < minCompressionLevel || diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0dfdd34..3231175 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,9 +1,12 @@ package config //nolint:testpackage // exercises unexported extractAgeSecretKey import ( + "errors" "os" "path/filepath" "testing" + + "sneak.berlin/go/vaultik/internal/chunker" ) const ( @@ -101,6 +104,80 @@ func TestConfigFromEnv(t *testing.T) { } } +// TestValidateBlobSizeLimit checks the blob_size_limit boundary: it must be at +// least the largest chunk the chunker can emit (chunk_size times +// chunker.ChunkSizeSpread), because the packer places a single such chunk into +// an otherwise empty blob. A limit between chunk_size and that bound is rejected. +func TestValidateBlobSizeLimit(t *testing.T) { + t.Parallel() + + const chunkSize = Size(10 * 1024 * 1024) // 10MB + + largestChunk := chunkSize.Int64() * chunker.ChunkSizeSpread + + newConfig := func(blobLimit Size) *Config { + return &Config{ + AgeRecipients: []string{testSneakAgePublicKey}, + Snapshots: map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}}, + StorageURL: "file:///tmp/vaultik-test-store", + ChunkSize: chunkSize, + BlobSizeLimit: blobLimit, + CompressionLevel: 3, + } + } + + tests := []struct { + name string + blobLimit Size + wantErr bool + }{ + { + name: "at chunk_size but below largest chunk is rejected", + blobLimit: chunkSize, + wantErr: true, + }, + { + name: "between chunk_size and largest chunk is rejected", + blobLimit: Size(chunkSize.Int64() * 2), + wantErr: true, + }, + { + name: "one byte below largest chunk is rejected", + blobLimit: Size(largestChunk - 1), + wantErr: true, + }, + { + name: "exactly at largest chunk is accepted", + blobLimit: Size(largestChunk), + wantErr: false, + }, + { + name: "above largest chunk is accepted", + blobLimit: Size(largestChunk * 100), + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := newConfig(tt.blobLimit).Validate() + if tt.wantErr { + if !errors.Is(err, errBlobSizeTooSmall) { + t.Fatalf("Validate() error = %v, want errBlobSizeTooSmall", err) + } + + return + } + + if err != nil { + t.Fatalf("Validate() unexpected error: %v", err) + } + }) + } +} + // TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs func TestExtractAgeSecretKey(t *testing.T) { t.Parallel() -- 2.54.0