Bound download expansion and escape control chars on the terminal (closes #164)

Objects fetched from the store are untrusted; several decode paths let one expand or print without limit.

- blobgen.LimitReader errors past a byte cap (not io.LimitReader silent EOF). DecodeManifest reads through caps on both compressed input and decompressed output, far above any real manifest, so json.Decode cannot buffer a compressible bomb. FetchAndDecryptBlob bounds decompression to the blob recorded uncompressed_size (not the restoring host blob_size_limit).
- downloadSnapshotDB streams straight from storage to its temp file with io.Copy, replacing two ReadAll calls that held the whole database twice.
- FetchBlob drops the per-blob Stat round-trip, its expectedSize parameter and returned size, all of which only fed a debug log.
- TTYHandler and ui.Writer escape control characters in messages, attribute keys/values, and rendered identifiers/paths before colour codes are applied, so a crafted value cannot drive the terminal.

Model: opus-4-8

internal/vaultik/blob_fetch.go

@@ -6,11 +6,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"time"
 
 	"filippo.io/age"
 	"sneak.berlin/go/vaultik/internal/blobgen"
-	"sneak.berlin/go/vaultik/internal/log"
 )
 
 // errBlobHashMismatch is returned when a fetched blob's content hash does
@@ -30,13 +28,14 @@ var errBlobNotFullyRead = errors.New(
 // redundant SHA-256 computation.
 type hashVerifyReader struct {
 	reader   *blobgen.Reader // underlying decrypted blob reader (has internal hasher)
+	limited  io.Reader       // reader bounded to the blob's recorded plaintext size
 	fetcher  io.ReadCloser   // raw fetched stream (closed on Close)
 	blobHash string          // expected double-SHA-256 hex
 	done     bool            // EOF reached
 }
 
 func (h *hashVerifyReader) Read(p []byte) (int, error) {
-	n, err := h.reader.Read(p)
+	n, err := h.limited.Read(p)
 	if errors.Is(err, io.EOF) {
 		h.done = true
 	}
@@ -73,11 +72,17 @@ func (h *hashVerifyReader) Close() error {
 // returns a streaming reader that computes the double-SHA-256 hash on the fly.
 // The hash is verified when the returned reader is closed (after fully reading).
 // This avoids buffering the entire blob in memory.
+//
+// maxPlaintextSize is the blob's uncompressed_size as recorded in the
+// snapshot database. Decompression stops with blobgen.ErrOutputTooLarge
+// once the plaintext exceeds it, so a tampered blob cannot expand without
+// limit — using the recorded size, not the restoring host's
+// blob_size_limit, since that config may differ from the backup host's.
 func (v *Vaultik) FetchAndDecryptBlob(
-	ctx context.Context, blobHash string, expectedSize int64,
+	ctx context.Context, blobHash string, maxPlaintextSize int64,
 	identities ...age.Identity,
 ) (io.ReadCloser, error) {
-	rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize)
+	rc, err := v.FetchBlob(ctx, blobHash)
 	if err != nil {
 		return nil, err
 	}
@@ -91,52 +96,29 @@ func (v *Vaultik) FetchAndDecryptBlob(
 
 	return &hashVerifyReader{
 		reader:   reader,
+		limited:  blobgen.LimitReader(reader, maxPlaintextSize),
 		fetcher:  rc,
 		blobHash: blobHash,
 	}, nil
 }
 
 // FetchBlob downloads a blob and returns a reader for the encrypted data.
-// Times the Storage.Get and Storage.Stat round-trips separately at
-// debug level so we can see whether the size-only Stat (which is an
-// extra request on every fetch) is hurting throughput.
 func (v *Vaultik) FetchBlob(
-	ctx context.Context, blobHash string, expectedSize int64,
-) (io.ReadCloser, int64, error) {
+	ctx context.Context, blobHash string,
+) (io.ReadCloser, error) {
 	// blobHash reaches here from the snapshot database, which is not
 	// trusted. Reject a malformed hash before it is spliced into a storage
 	// path (blobHash[:2]/blobHash[2:4]) or a fetch is attempted.
 	if !isBlobHash(blobHash) {
-		return nil, 0, fmt.Errorf("%w: %s", errInvalidBlobHash, shortHash(blobHash))
+		return nil, fmt.Errorf("%w: %s", errInvalidBlobHash, shortHash(blobHash))
 	}
 
 	blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
 
-	t0 := time.Now()
 	rc, err := v.Storage.Get(ctx, blobPath)
-	getDur := time.Since(t0)
-
 	if err != nil {
-		return nil, 0, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
+		return nil, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
 	}
 
-	t0 = time.Now()
-	info, err := v.Storage.Stat(ctx, blobPath)
-	statDur := time.Since(t0)
-
-	if err != nil {
-		_ = rc.Close()
-
-		return nil, 0, fmt.Errorf("stat blob %s: %w", shortHash(blobHash), err)
-	}
-
-	log.Debug("FetchBlob round-trips",
-		"hash", shortHash(blobHash),
-		"ms_storage_get", getDur.Milliseconds(),
-		"ms_storage_stat", statDur.Milliseconds(),
-		"expected_size", expectedSize,
-		"stat_size", info.Size,
-	)
-
-	return rc, info.Size, nil
+	return rc, nil
 }

internal/vaultik/blob_fetch_bound_test.go

@@ -0,0 +1,50 @@
+package vaultik_test
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"filippo.io/age"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/blobgen"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestFetchAndDecryptBlobBoundsPlaintext feeds a small, highly
+// compressible blob (256 KiB of zeros) whose decompressed size far exceeds
+// the plaintext bound passed to FetchAndDecryptBlob. Decompression must
+// stop with blobgen.ErrOutputTooLarge within the bound rather than
+// expanding the whole blob into the restore cache.
+func TestFetchAndDecryptBlobBoundsPlaintext(t *testing.T) {
+	t.Parallel()
+
+	identity, err := age.GenerateX25519Identity()
+	require.NoError(t, err)
+
+	plaintext := make([]byte, 256*1024)
+	encryptedData, correctHash := buildHashTestBlob(t, identity, plaintext)
+
+	mockStorage := NewMockStorer()
+	blobPath := "blobs/" + correctHash[:2] + "/" +
+		correctHash[2:4] + "/" + correctHash
+
+	mockStorage.mu.Lock()
+	mockStorage.data[blobPath] = encryptedData
+	mockStorage.mu.Unlock()
+
+	tv := vaultik.NewForTesting(mockStorage)
+
+	const maxPlaintext = 1024
+
+	rc, err := tv.FetchAndDecryptBlob(
+		context.Background(), correctHash, maxPlaintext, identity)
+	require.NoError(t, err)
+
+	n, copyErr := io.Copy(io.Discard, rc)
+	_ = rc.Close()
+
+	require.ErrorIs(t, copyErr, blobgen.ErrOutputTooLarge)
+	require.LessOrEqual(t, n, int64(maxPlaintext)+1,
+		"decompression must stop within the recorded plaintext bound")
+}

internal/vaultik/blob_fetch_hash_test.go

@@ -73,7 +73,7 @@ func TestFetchBlobRejectsMalformedHash(t *testing.T) {
 		strings.Repeat("A", 64), // uppercase hex is not accepted
 		strings.Repeat("g", 64), // not hex
 	} {
-		_, _, err := tv.FetchBlob(ctx, bad, 0)
+		_, err := tv.FetchBlob(ctx, bad)
 		if err == nil {
 			t.Fatalf("expected error for malformed hash %q, got nil", bad)
 		}
@@ -113,7 +113,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
 		t.Parallel()
 
 		rc, err := tv.FetchAndDecryptBlob(
-			ctx, correctHash, int64(len(encryptedData)), identity)
+			ctx, correctHash, int64(len(plaintext)), identity)
 		if err != nil {
 			t.Fatalf("expected success, got error: %v", err)
 		}
@@ -145,7 +145,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
 		mockStorage.mu.Unlock()
 
 		rc, err := tv.FetchAndDecryptBlob(
-			ctx, fakeHash, int64(len(encryptedData)), identity)
+			ctx, fakeHash, int64(len(plaintext)), identity)
 		if err != nil {
 			t.Fatalf("unexpected error opening stream: %v", err)
 		}
@@ -188,7 +188,7 @@ func TestFetchAndDecryptBlobCloseBeforeEOFFails(t *testing.T) {
 	tv := vaultik.NewForTesting(mockStorage)
 
 	rc, err := tv.FetchAndDecryptBlob(
-		context.Background(), correctHash, int64(len(encryptedData)), identity)
+		context.Background(), correctHash, int64(len(plaintext)), identity)
 	if err != nil {
 		t.Fatalf("unexpected error opening stream: %v", err)
 	}

internal/vaultik/restore.go

@@ -1,7 +1,6 @@
 package vaultik
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -428,7 +427,7 @@ func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
 			return false, fmt.Errorf("%w: %s", errBlobMissingFromIndex, shortHash(hash))
 		}
 
-		err := s.downloadBlobToCache(hash, blob.CompressedSize)
+		err := s.downloadBlobToCache(hash, blob.CompressedSize, blob.UncompressedSize)
 		if err != nil {
 			return false, fmt.Errorf("downloading blob %s: %w", shortHash(hash), err)
 		}
@@ -655,32 +654,18 @@ func (v *Vaultik) downloadSnapshotDB(
 
 	defer func() { _ = reader.Close() }()
 
-	// Read all data
-	encryptedData, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, "", fmt.Errorf("reading encrypted data: %w", err)
-	}
-
-	log.Debug("Downloaded encrypted database",
-		"size", ubytes(int64(len(encryptedData))))
-
-	// Decrypt and decompress using blobgen.Reader
-	blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identities...)
+	// Decrypt and decompress straight from the storage stream, then stream
+	// the plaintext to a temp file. Neither the encrypted bytes nor the
+	// decrypted database is ever held whole in memory; a snapshot database
+	// can be large.
+	blobReader, err := blobgen.NewReader(reader, identities...)
 	if err != nil {
 		return nil, "", fmt.Errorf("creating decryption reader: %w", err)
 	}
 
 	defer func() { _ = blobReader.Close() }()
 
-	// Read the binary SQLite database
-	dbData, err := io.ReadAll(blobReader)
-	if err != nil {
-		return nil, "", fmt.Errorf("decrypting and decompressing: %w", err)
-	}
-
-	log.Debug("Decrypted database", "size", ubytes(int64(len(dbData))))
-
-	db, tempDir, err := v.materializeSnapshotDB(dbData)
+	db, tempDir, err := v.materializeSnapshotDB(blobReader)
 	if err != nil {
 		return nil, "", err
 	}
@@ -729,13 +714,15 @@ func (v *Vaultik) verifySnapshotDBIdentity(
 	return nil
 }
 
-// materializeSnapshotDB writes the decrypted snapshot database bytes into
-// a fresh private (0700) temp directory and opens the file read-only. On
-// any failure it removes the directory before returning, so no decrypted
-// metadata is left on disk when the open is interrupted or the payload is
-// damaged. On success the returned directory is the caller's to remove.
+// materializeSnapshotDB streams the decrypted snapshot database into a
+// fresh private (0700) temp directory and opens the file read-only. The
+// database is copied through an io.Copy buffer rather than read whole into
+// memory. On any failure it removes the directory before returning, so no
+// decrypted metadata is left on disk when the copy is interrupted or the
+// payload is damaged. On success the returned directory is the caller's to
+// remove.
 func (v *Vaultik) materializeSnapshotDB(
-	dbData []byte,
+	dbReader io.Reader,
 ) (*database.DB, string, error) {
 	tempDir, err := afero.TempDir(v.Fs, "", "vaultik-restore-")
 	if err != nil {
@@ -752,12 +739,24 @@ func (v *Vaultik) materializeSnapshotDB(
 
 	dbPath := filepath.Join(tempDir, snapshotDBFilename)
 
-	err = afero.WriteFile(v.Fs, dbPath, dbData, restoreFileMode)
+	dbFile, err := v.Fs.OpenFile(
+		dbPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, restoreFileMode)
 	if err != nil {
-		return nil, "", fmt.Errorf("writing database file: %w", err)
+		return nil, "", fmt.Errorf("creating database file: %w", err)
 	}
 
-	log.Debug("Created restore database", "path", dbPath)
+	written, copyErr := io.Copy(dbFile, dbReader)
+	closeErr := dbFile.Close()
+
+	if copyErr != nil {
+		return nil, "", fmt.Errorf("writing database file: %w", copyErr)
+	}
+
+	if closeErr != nil {
+		return nil, "", fmt.Errorf("closing database file: %w", closeErr)
+	}
+
+	log.Debug("Created restore database", "path", dbPath, "size", ubytes(written))
 
 	db, err := database.OpenReadOnly(v.ctx, dbPath)
 	if err != nil {
@@ -1212,12 +1211,12 @@ func (s *restoreSession) writeFileChunks(
 // size, which is what makes multi-GB blobs tractable on machines with
 // less RAM than the blob.
 func (s *restoreSession) downloadBlobToCache(
-	blobHash string, expectedSize int64,
+	blobHash string, compressedSize, uncompressedSize int64,
 ) error {
 	start := time.Now()
 
 	t0 := time.Now()
-	rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, expectedSize, s.identities...)
+	rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, uncompressedSize, s.identities...)
 	fetchSetupDur := time.Since(t0)
 
 	if err != nil {
@@ -1247,7 +1246,7 @@ func (s *restoreSession) downloadBlobToCache(
 
 	log.Debug("Streamed blob into disk cache",
 		"hash", blobHash[:16],
-		"compressed_bytes", expectedSize,
+		"compressed_bytes", compressedSize,
 		"plaintext_bytes", written,
 		"ms_total", time.Since(start).Milliseconds(),
 		"ms_fetch_setup", fetchSetupDur.Milliseconds(),

internal/vaultik/restore_snapshotdb_test.go

@@ -1,6 +1,7 @@
 package vaultik //nolint:testpackage // inspects unexported snapshot-db materialization
 
 import (
+	"bytes"
 	"context"
 	"os"
 	"path/filepath"
@@ -37,7 +38,7 @@ func TestMaterializeSnapshotDBPrivateDir(t *testing.T) {
 
 	v := &Vaultik{ctx: context.Background(), Fs: afero.NewOsFs()}
 
-	db, dir, err := v.materializeSnapshotDB(dbData)
+	db, dir, err := v.materializeSnapshotDB(bytes.NewReader(dbData))
 	require.NoError(t, err)
 
 	t.Cleanup(func() {
@@ -64,7 +65,8 @@ func TestMaterializeSnapshotDBRemovesDirOnOpenFailure(t *testing.T) {
 
 	v := &Vaultik{ctx: context.Background(), Fs: afero.NewOsFs()}
 
-	_, _, err := v.materializeSnapshotDB([]byte("this is not a sqlite database"))
+	_, _, err := v.materializeSnapshotDB(
+		bytes.NewReader([]byte("this is not a sqlite database")))
 	require.Error(t, err)
 
 	entries, rerr := os.ReadDir(base)

internal/vaultik/verify.go

@@ -393,7 +393,7 @@ func (v *Vaultik) verifyBlob(
 	blobInfo snapshot.BlobInfo, db *sql.DB, identities []age.Identity,
 ) error {
 	// Download blob using shared fetch method
-	reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
+	reader, err := v.FetchBlob(v.ctx, blobInfo.Hash)
 	if err != nil {
 		return fmt.Errorf("failed to download: %w", err)
 	}