Reject a blob_size_limit below the largest possible chunk (closes #167)

Validate only rejected blob_size_limit below chunk_size, but the chunker
can emit chunks up to chunk_size times the FastCDC size spread (four
times), and the packer puts a single chunk of any size into an otherwise
empty blob. A limit between one and four times chunk_size therefore let a
blob reach four times the configured maximum, with most blobs holding a
single chunk and so exposing individual chunk lengths to anyone who can
list the destination.

Validate now rejects blob_size_limit below chunk_size times the spread,
reusing the chunker's one constant (now exported as ChunkSizeSpread)
instead of a second literal. The rule is stated in the error text, the
Validate comment, the README config table, config.example.yml, and the
generated config template.

Model: opus-4-8

README.md

@@ -518,7 +518,7 @@ complete annotated example also lives in
 | `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
 | `exclude` | | Global exclude patterns (applied to all snapshots) |
 | `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
-| `blob_size_limit` | `10GB` | Maximum blob size before splitting |
+| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit |
 | `compression_level` | `3` | zstd compression level (1-19) |
 | `hostname` | system hostname | Hostname used in snapshot IDs |
 | `index_path` | platform data dir | Local SQLite index path |

config.example.yml

@@ -304,6 +304,8 @@ storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion"
 
 # Maximum blob size
 # Multiple chunks are packed into blobs up to this size
+# Must be at least four times chunk_size (the largest chunk the chunker can
+# emit); a smaller limit would let a single-chunk blob exceed it.
 # Supports: 1GB, 10G, 500MB, 1GiB, etc.
 # Default: 10GB
 #blob_size_limit: 10GB

internal/chunker/chunker.go

@@ -33,9 +33,10 @@ type Chunker struct {
 	maxChunkSize int
 }
 
-// chunkSizeSpread is the FastCDC-recommended factor between the average
+// ChunkSizeSpread is the FastCDC-recommended factor between the average
 // chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes.
-const chunkSizeSpread = 4
+// The largest chunk the chunker can emit is therefore avg*ChunkSizeSpread.
+const ChunkSizeSpread = 4
 
 // NewChunker creates a new chunker with the specified average chunk size.
 // The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4
@@ -45,8 +46,8 @@ func NewChunker(avgChunkSize int64) *Chunker {
 	// FastCDC recommends min = avg/4 and max = avg*4
 	return &Chunker{
 		avgChunkSize: int(avgChunkSize),
-		minChunkSize: int(avgChunkSize / chunkSizeSpread),
-		maxChunkSize: int(avgChunkSize * chunkSizeSpread),
+		minChunkSize: int(avgChunkSize / ChunkSizeSpread),
+		maxChunkSize: int(avgChunkSize * ChunkSizeSpread),
 	}
 }
 

internal/cli/config.go

@@ -212,6 +212,8 @@ storage_url: ""
 # chunk_size: 10MB
 
 # Maximum blob size before splitting into a new blob.
+# Must be at least four times chunk_size (the largest chunk the chunker can
+# emit); a smaller limit would let a single-chunk blob exceed it.
 # Accepts: 1GB, 10G, 500MB, etc.
 # Default: 10GB
 # blob_size_limit: 10GB

internal/config/config.go

@@ -16,6 +16,7 @@ import (
 	"github.com/adrg/xdg"
 	"go.uber.org/fx"
 	"gopkg.in/yaml.v3"
+	"sneak.berlin/go/vaultik/internal/chunker"
 	"sneak.berlin/go/vaultik/internal/log"
 )
 
@@ -41,9 +42,11 @@ var (
 		"at least one snapshot must be configured (see config.example.yml)")
 	errSnapshotNoPaths   = errors.New("snapshot must have at least one path")
 	errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB")
-	errBlobSizeTooSmall  = errors.New("blob_size_limit must be at least chunk_size")
-	errBadCompression    = errors.New("compression_level must be between 1 and 19")
-	errBadStorageScheme  = errors.New(
+	errBlobSizeTooSmall  = errors.New(
+		"blob_size_limit must be at least the largest chunk the chunker can " +
+			"emit (chunk_size times the FastCDC size spread)")
+	errBadCompression   = errors.New("compression_level must be between 1 and 19")
+	errBadStorageScheme = errors.New(
 		"storage_url must start with s3://, file://, or rclone://")
 	errStorageNotConfigured = errors.New(
 		"storage not configured; set storage_url or provide s3.endpoint + " +
@@ -287,12 +290,15 @@ func Load(path string) (*Config, error) {
 
 // Validate checks if the configuration is valid and complete.
 // It ensures all required fields are present and have valid values:
-// - At least one age recipient must be specified
-// - At least one snapshot must be configured with at least one path
-// - Storage must be configured (either storage_url or s3.* fields)
-// - Chunk size must be at least 1MB
-// - Blob size limit must be at least the chunk size
-// - Compression level must be between 1 and 19
+//   - At least one age recipient must be specified
+//   - At least one snapshot must be configured with at least one path
+//   - Storage must be configured (either storage_url or s3.* fields)
+//   - Chunk size must be at least 1MB
+//   - Blob size limit must be at least the largest chunk the chunker can emit
+//     (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
+//     exceeds the configured limit
+//   - Compression level must be between 1 and 19
+//
 // Returns an error describing the first validation failure encountered.
 func (c *Config) Validate() error {
 	if len(c.AgeRecipients) == 0 {
@@ -319,8 +325,13 @@ func (c *Config) Validate() error {
 		return errChunkSizeTooSmall
 	}
 
-	if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
-		return errBlobSizeTooSmall
+	// The chunker can emit chunks up to chunk_size * ChunkSizeSpread, and the
+	// packer places a single such chunk into an otherwise empty blob. A limit
+	// below that bound would let a blob exceed it, so reject it.
+	largestChunk := c.ChunkSize.Int64() * chunker.ChunkSizeSpread
+	if c.BlobSizeLimit.Int64() < largestChunk {
+		return fmt.Errorf("%w: need at least %d bytes",
+			errBlobSizeTooSmall, largestChunk)
 	}
 
 	if c.CompressionLevel < minCompressionLevel ||

internal/config/config_test.go

@@ -1,9 +1,12 @@
 package config //nolint:testpackage // exercises unexported extractAgeSecretKey
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"sneak.berlin/go/vaultik/internal/chunker"
 )
 
 const (
@@ -101,6 +104,80 @@ func TestConfigFromEnv(t *testing.T) {
 	}
 }
 
+// TestValidateBlobSizeLimit checks the blob_size_limit boundary: it must be at
+// least the largest chunk the chunker can emit (chunk_size times
+// chunker.ChunkSizeSpread), because the packer places a single such chunk into
+// an otherwise empty blob. A limit between chunk_size and that bound is rejected.
+func TestValidateBlobSizeLimit(t *testing.T) {
+	t.Parallel()
+
+	const chunkSize = Size(10 * 1024 * 1024) // 10MB
+
+	largestChunk := chunkSize.Int64() * chunker.ChunkSizeSpread
+
+	newConfig := func(blobLimit Size) *Config {
+		return &Config{
+			AgeRecipients:    []string{testSneakAgePublicKey},
+			Snapshots:        map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}},
+			StorageURL:       "file:///tmp/vaultik-test-store",
+			ChunkSize:        chunkSize,
+			BlobSizeLimit:    blobLimit,
+			CompressionLevel: 3,
+		}
+	}
+
+	tests := []struct {
+		name      string
+		blobLimit Size
+		wantErr   bool
+	}{
+		{
+			name:      "at chunk_size but below largest chunk is rejected",
+			blobLimit: chunkSize,
+			wantErr:   true,
+		},
+		{
+			name:      "between chunk_size and largest chunk is rejected",
+			blobLimit: Size(chunkSize.Int64() * 2),
+			wantErr:   true,
+		},
+		{
+			name:      "one byte below largest chunk is rejected",
+			blobLimit: Size(largestChunk - 1),
+			wantErr:   true,
+		},
+		{
+			name:      "exactly at largest chunk is accepted",
+			blobLimit: Size(largestChunk),
+			wantErr:   false,
+		},
+		{
+			name:      "above largest chunk is accepted",
+			blobLimit: Size(largestChunk * 100),
+			wantErr:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			err := newConfig(tt.blobLimit).Validate()
+			if tt.wantErr {
+				if !errors.Is(err, errBlobSizeTooSmall) {
+					t.Fatalf("Validate() error = %v, want errBlobSizeTooSmall", err)
+				}
+
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Validate() unexpected error: %v", err)
+			}
+		})
+	}
+}
+
 // TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs
 func TestExtractAgeSecretKey(t *testing.T) {
 	t.Parallel()