Limit how much fetch and check read for a manifest or a file (closes #168)

NewManifestFromReader reads at most one byte past MaxManifestSize, a new
constant of 258 MiB: the 256 MiB decompressed limit grown by zstd's worst
case of 1/256, plus 1 MiB for the signature, the signing key and the
other outer fields. It refuses a larger manifest. fetch, and check given
a URL, stop downloading a manifest one byte past the same size and report
it as too large; tests lower that size to keep their memory small. fetch
stops reading a file one byte past its listed size, so a longer body ends
in the size mismatch at once instead of filling the disk. docs/FORMAT.md
states the limit and gives the decompressed limit as 256 MiB, the size
the code uses.

Model: opus-5-5

mfer/constants.go

@@ -8,10 +8,17 @@ const (
 	ReleaseDate = "2025-12-17"
 
 	// MaxDecompressedSize is the maximum allowed size of decompressed manifest
-	// data (256 MB). This prevents decompression bombs from consuming excessive
+	// data (256 MiB). This prevents decompression bombs from consuming excessive
 	// memory.
 	MaxDecompressedSize int64 = 256 * 1024 * 1024
 
+	// MaxManifestSize is the largest manifest file mfer reads (258 MiB).
+	// zstd's worst case grows data it cannot compress by 1/256, so an inner
+	// message of MaxDecompressedSize compresses to at most 257 MiB; the
+	// last MiB is room for the signature, the signing key and the other
+	// outer fields.
+	MaxManifestSize = MaxDecompressedSize + MaxDecompressedSize/256 + 1<<20
+
 	// zstdWindowSize is the zstd window zstd.SpeedBestCompression gives mfer's writer.
 	zstdWindowSize = 8 << 20
 

mfer/deserialize.go

@@ -24,6 +24,7 @@ var (
 	errCompressedHashWrong  = errors.New("compressed data hash mismatch")
 	errSignatureNoPubKey    = errors.New("signature present but no public key")
 	errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
+	errManifestTooLarge     = errors.New("manifest exceeds maximum allowed size")
 	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
 	errInvalidFileFormat    = errors.New("invalid file format")
 	errInvalidManifestPath  = errors.New("manifest contains invalid path")
@@ -318,13 +319,14 @@ func validateMagic(dat []byte) bool {
 	return bytes.Equal(got, expected)
 }
 
-// NewManifestFromReader reads a manifest from an io.Reader.
+// NewManifestFromReader reads a manifest from an io.Reader. It refuses a
+// manifest larger than MaxManifestSize, reading at most one byte past it.
 //
 //nolint:revive // unexported-return: exporting manifest is owner question 13
 func NewManifestFromReader(input io.Reader) (*manifest, error) {
 	m := &manifest{}
 
-	dat, err := io.ReadAll(input)
+	dat, err := readAtMost(input, MaxManifestSize)
 	if err != nil {
 		return nil, err
 	}
@@ -356,6 +358,21 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
 	return m, nil
 }
 
+// readAtMost reads all of input, or refuses it with errManifestTooLarge
+// once it passes maxSize bytes, after reading one byte past maxSize.
+func readAtMost(input io.Reader, maxSize int64) ([]byte, error) {
+	dat, err := io.ReadAll(io.LimitReader(input, maxSize+1))
+	if err != nil {
+		return nil, err
+	}
+
+	if int64(len(dat)) > maxSize {
+		return nil, fmt.Errorf("%w of %d bytes", errManifestTooLarge, maxSize)
+	}
+
+	return dat, nil
+}
+
 // ManifestFromFileOptions configures NewManifestFromFile.
 type ManifestFromFileOptions struct {
 	// Path is the manifest file to read (required).

mfer/deserialize_test.go

@@ -0,0 +1,33 @@
+//nolint:testpackage // white-box tests exercise unexported internals
+package mfer
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestReadAtMost gives readAtMost exactly its maximum, which it must
+// return whole, and twice its maximum, which it must refuse after reading
+// one byte past the maximum, and no more. NewManifestFromReader reads
+// through it with MaxManifestSize; the test uses 64 KiB, since reading
+// MaxManifestSize under the race detector takes gigabytes of memory.
+func TestReadAtMost(t *testing.T) {
+	t.Parallel()
+
+	const maxSize = 64 << 10
+
+	dat, err := readAtMost(bytes.NewReader(make([]byte, maxSize)), maxSize)
+	require.NoError(t, err)
+	assert.Len(t, dat, maxSize)
+
+	input := bytes.NewReader(make([]byte, 2*maxSize))
+
+	_, err = readAtMost(input, maxSize)
+	require.ErrorIs(t, err, errManifestTooLarge)
+	require.EqualError(t, err,
+		"manifest exceeds maximum allowed size of 65536 bytes")
+	assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
+}