Bound download expansion and escape control chars on the terminal (closes #164)
Objects fetched from the store are untrusted; several decode paths let one expand or print without limit. - blobgen.LimitReader errors past a byte cap (not io.LimitReader silent EOF). DecodeManifest reads through caps on both compressed input and decompressed output, far above any real manifest, so json.Decode cannot buffer a compressible bomb. FetchAndDecryptBlob bounds decompression to the blob recorded uncompressed_size (not the restoring host blob_size_limit). - downloadSnapshotDB streams straight from storage to its temp file with io.Copy, replacing two ReadAll calls that held the whole database twice. - FetchBlob drops the per-blob Stat round-trip, its expectedSize parameter and returned size, all of which only fed a debug log. - TTYHandler and ui.Writer escape control characters in messages, attribute keys/values, and rendered identifiers/paths before colour codes are applied, so a crafted value cannot drive the terminal. Model: opus-4-8
This commit was merged in pull request #197.
This commit is contained in:
@@ -7,6 +7,19 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// Manifest size bounds. A manifest lists one small entry per blob, and
|
||||
// blobs are large (the default target is 10 GB), so even a manifest for a
|
||||
// petabyte-scale backup is a few megabytes. These caps are far above any
|
||||
// manifest the writer can emit, yet stop a crafted, highly compressible
|
||||
// manifest from expanding without limit when decoded: the manifest is
|
||||
// fetched from the store, which is not trusted, and json.Decode buffers
|
||||
// the whole value in memory.
|
||||
const (
|
||||
manifestMaxCompressed = 256 * 1024 * 1024 // 256 MiB
|
||||
manifestMaxDecompressed = 1024 * 1024 * 1024 // 1 GiB
|
||||
)
|
||||
|
||||
// Manifest represents the structure of a snapshot's blob manifest
|
||||
@@ -28,19 +41,31 @@ type BlobInfo struct {
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
}
|
||||
|
||||
// DecodeManifest decodes a manifest from a reader containing compressed JSON
|
||||
// DecodeManifest decodes a manifest from a reader containing compressed
|
||||
// JSON, reading through byte limits on both the compressed input and the
|
||||
// decompressed output so an untrusted manifest cannot exhaust memory.
|
||||
func DecodeManifest(r io.Reader) (*Manifest, error) {
|
||||
// Decompress using zstd
|
||||
zr, err := zstd.NewReader(r)
|
||||
return decodeManifest(r, manifestMaxCompressed, manifestMaxDecompressed)
|
||||
}
|
||||
|
||||
// decodeManifest is DecodeManifest with explicit limits, so tests can drive
|
||||
// the bounds with small inputs instead of gigabyte-scale ones.
|
||||
func decodeManifest(
|
||||
r io.Reader, maxCompressed, maxDecompressed int64,
|
||||
) (*Manifest, error) {
|
||||
// Decompress using zstd, bounding how many compressed bytes are read.
|
||||
zr, err := zstd.NewReader(blobgen.LimitReader(r, maxCompressed))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating zstd reader: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
// Decode JSON manifest
|
||||
// Decode JSON manifest, bounding how far the compressed input may
|
||||
// expand: json.Decode buffers the whole value, so without this a
|
||||
// small, highly compressible manifest could expand to gigabytes.
|
||||
var manifest Manifest
|
||||
|
||||
err = json.NewDecoder(zr).Decode(&manifest)
|
||||
err = json.NewDecoder(blobgen.LimitReader(zr, maxDecompressed)).Decode(&manifest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding manifest: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
//nolint:testpackage // exercises the unexported decodeManifest bounds
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// testSnapshotID is a stand-in snapshot ID reused across the bound cases.
|
||||
const testSnapshotID = "host_home_2026-01-01T00:00:00Z"
|
||||
|
||||
// TestDecodeManifestRoundTrip is the baseline: with generous bounds a
|
||||
// manifest the writer produced decodes back unchanged.
|
||||
func TestDecodeManifestRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
want := &Manifest{
|
||||
SnapshotID: testSnapshotID,
|
||||
Timestamp: "2026-01-01T00:00:00Z",
|
||||
BlobCount: 2,
|
||||
TotalCompressedSize: 42,
|
||||
Blobs: []BlobInfo{
|
||||
{Hash: "aa", CompressedSize: 21},
|
||||
{Hash: "bb", CompressedSize: 21},
|
||||
},
|
||||
}
|
||||
|
||||
compressed, err := EncodeManifest(want, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := decodeManifest(
|
||||
bytes.NewReader(compressed), manifestMaxCompressed, manifestMaxDecompressed)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, want, got)
|
||||
}
|
||||
|
||||
// TestDecodeManifestBoundsDecompressedOutput feeds a valid but highly
|
||||
// compressible manifest — one whose timestamp is a megabyte of the same
|
||||
// character — through a small decompressed bound. The compressed form is
|
||||
// tiny, so only the decompressed bound stops it; decoding must fail within
|
||||
// that bound rather than expanding the value in memory.
|
||||
func TestDecodeManifestBoundsDecompressedOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bomb := &Manifest{
|
||||
SnapshotID: testSnapshotID,
|
||||
Timestamp: strings.Repeat("a", 1<<20),
|
||||
}
|
||||
|
||||
compressed, err := EncodeManifest(bomb, 3)
|
||||
require.NoError(t, err)
|
||||
require.Less(t, len(compressed), 4096,
|
||||
"the compressible manifest must be small compressed")
|
||||
|
||||
_, err = decodeManifest(bytes.NewReader(compressed), 1<<20, 4096)
|
||||
require.ErrorIs(t, err, blobgen.ErrOutputTooLarge)
|
||||
}
|
||||
|
||||
// TestDecodeManifestBoundsCompressedInput checks the compressed-input
|
||||
// bound fires independently: a valid manifest with a generous decompressed
|
||||
// bound but a tiny compressed bound still fails.
|
||||
func TestDecodeManifestBoundsCompressedInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
manifest := &Manifest{
|
||||
SnapshotID: testSnapshotID,
|
||||
Timestamp: strings.Repeat("a", 4096),
|
||||
}
|
||||
|
||||
compressed, err := EncodeManifest(manifest, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = decodeManifest(bytes.NewReader(compressed), 8, manifestMaxDecompressed)
|
||||
require.Error(t, err)
|
||||
}
|
||||
Reference in New Issue
Block a user