Reject manifests whose file entries decode far larger than their bytes (closes #123)

Parser fix: before decoding the manifest, the parser walks its file
entries and adds up what decoding sets aside for each entry, hash,
timestamp and MIME type, however short its encoding. It refuses the
manifest once that sum passes 8 times the decompressed size; the
densest manifests mfer writes come to about 7 times. Empty entries
decoded to about 50 times their size, so a 1.6 KB manifest allocated
nearly 1 GB. The fuzz target's ceiling falls to 20 times the input and
decompressed data, and a new seed of entries holding only an empty MIME
type and empty times fails it without the fix.

Model: opus-5-5

mfer/constants.go

@@ -17,4 +17,24 @@ const (
 
 	// uuidLength is the length in bytes of a binary UUID.
 	uuidLength = 16
+
+	// Numbers in mf.proto of MFFile.files and of the MFFilePath fields
+	// that decoding sets aside a fixed amount of memory for.
+	filesFieldNumber    = 101
+	hashesFieldNumber   = 3
+	mimeTypeFieldNumber = 301
+	mtimeFieldNumber    = 302
+	ctimeFieldNumber    = 303
+
+	// Bytes decoding sets aside for each file entry, hash, timestamp and
+	// MIME type, however short its encoding. checkDecodedSize refuses an
+	// inner message for which these add up to more than maxDecodedGrowth
+	// times its size.
+	decodedFileEntrySize = 160
+	decodedHashSize      = 112
+	decodedTimestampSize = 64
+	decodedMIMETypeSize  = 16
+
+	// The densest manifests mfer writes add up to about 7 times their size.
+	maxDecodedGrowth = 8
 )

mfer/deserialize.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/afero"
+	"google.golang.org/protobuf/encoding/protowire"
 	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/internal/bork"
 	"sneak.berlin/go/mfer/internal/log"
@@ -27,6 +28,8 @@ var (
 	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
 	errInvalidFileFormat    = errors.New("invalid file format")
 	errInvalidManifestPath  = errors.New("manifest contains invalid path")
+	errDecodedTooLarge      = errors.New(
+		"manifest would take too much memory to decode")
 )
 
 // validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
@@ -154,6 +157,85 @@ func (m *manifest) decompressInner() ([]byte, error) {
 	return dat, nil
 }
 
+// checkDecodedSize refuses an encoded inner message whose file entries,
+// hashes, timestamps and MIME types would take more than maxDecodedGrowth
+// times its size to decode. Decoding sets aside a fixed amount for each,
+// however short its encoding, so a message of empty ones would take about
+// 50 times its size.
+func checkDecodedSize(inner []byte) error {
+	limit := maxDecodedGrowth * int64(len(inner))
+
+	var decoded int64
+
+	add := func(size int64) error {
+		decoded += size
+		if decoded > limit {
+			return errDecodedTooLarge
+		}
+
+		return nil
+	}
+
+	return forEachBytesField(inner, func(num protowire.Number, entry []byte) error {
+		if num != filesFieldNumber {
+			return nil
+		}
+
+		err := add(decodedFileEntrySize)
+		if err != nil {
+			return err
+		}
+
+		return forEachBytesField(entry, func(num protowire.Number, _ []byte) error {
+			if num == hashesFieldNumber {
+				return add(decodedHashSize)
+			}
+
+			if num == mtimeFieldNumber || num == ctimeFieldNumber {
+				return add(decodedTimestampSize)
+			}
+
+			if num == mimeTypeFieldNumber {
+				return add(decodedMIMETypeSize)
+			}
+
+			return nil
+		})
+	})
+}
+
+// forEachBytesField calls fn with the number and value of each
+// length-delimited field in the encoded message msg, and fails if msg is
+// malformed.
+func forEachBytesField(
+	msg []byte, fn func(num protowire.Number, value []byte) error,
+) error {
+	for len(msg) > 0 {
+		num, wireType, tagLen := protowire.ConsumeTag(msg)
+		if tagLen < 0 {
+			return protowire.ParseError(tagLen)
+		}
+
+		valueLen := protowire.ConsumeFieldValue(num, wireType, msg[tagLen:])
+		if valueLen < 0 {
+			return protowire.ParseError(valueLen)
+		}
+
+		if wireType == protowire.BytesType {
+			value, _ := protowire.ConsumeBytes(msg[tagLen:])
+
+			err := fn(num, value)
+			if err != nil {
+				return err
+			}
+		}
+
+		msg = msg[tagLen+valueLen:]
+	}
+
+	return nil
+}
+
 func (m *manifest) deserializeInner() error {
 	err := m.validateOuterHeader()
 	if err != nil {
@@ -177,6 +259,11 @@ func (m *manifest) deserializeInner() error {
 		return bork.ErrFileTruncated
 	}
 
+	err = checkDecodedSize(dat)
+	if err != nil {
+		return fmt.Errorf("deserialize: unmarshal inner: %w", err)
+	}
+
 	// Deserialize inner message
 	m.pbInner = new(MFFile)
 

mfer/deserialize_fuzz_test.go

@@ -54,9 +54,14 @@ func FuzzNewManifestFromReader(f *testing.F) {
 		}
 
 		// It also keeps a few copies of its input. Buffers grow by
-		// copying, so reaching those sizes allocates a few times them in
+		// copying, so reaching those sizes allocates up to about six times
-		// total: sixteen times the input and the decompressed data leaves
+		// them in total. Decoding the decompressed data takes up to
-		// room for that.
+		// maxDecodedGrowth times its size for file entries, hashes,
+		// timestamps and MIME types, and up to about five times more for
+		// the bytes it copies out of it, such as fields it does not know,
+		// which it keeps in buffers that also grow by copying. Twenty
+		// times the input and the decompressed data leaves room for all of
+		// that.
 		//
 		// The decoder also sets aside a new buffer of one to two times the
 		// window for each frame that asks for a larger window than the
@@ -71,8 +76,10 @@ func FuzzNewManifestFromReader(f *testing.F) {
 		// fails if the decoder accepts windows of twice zstdWindowSize; the
 		// seed whose two frames together exceed MaxDecompressedSize fails
 		// if the decoder decodes them in full instead of stopping at the
-		// declared size.
+		// declared size; the seeds of empty file entries, of a file entry
-		limit := 16*(uint64(len(data))+decompressed) + 24*zstdWindowSize
+		// of empty hashes, and of file entries of only an empty MIME type
+		// and empty times fail if the parser decodes them.
+		limit := 20*(uint64(len(data))+decompressed) + 24*zstdWindowSize
 
 		allocated := after.TotalAlloc - before.TotalAlloc
 		if allocated > limit {

mfer/deserialize_path_test.go

@@ -6,7 +6,9 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
@@ -114,6 +116,57 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
 	}
 }
 
+// Entries of a one-character path and empty modification and change times
+// pass every other check, but would take about 23 times their size to decode.
+func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
+	t.Parallel()
+
+	entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
+	entry = protowire.AppendString(entry, "a")
+	entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
+	entry = protowire.AppendBytes(entry, nil)
+	entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
+	entry = protowire.AppendBytes(entry, nil)
+
+	id := uuid.New()
+	inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
+	inner = protowire.AppendBytes(inner, id[:])
+
+	for range 1000 {
+		inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
+		inner = protowire.AppendBytes(inner, entry)
+	}
+
+	_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
+	require.ErrorIs(t, err, errDecodedTooLarge)
+}
+
+// Many empty files with names of at most three characters and modification
+// times at the epoch make about the densest manifest mfer writes: it takes
+// about 7 times its size to decode, and still loads. A signature would not
+// change the inner message, so none is added.
+func TestDeserializeLoadsDensestManifest(t *testing.T) {
+	t.Parallel()
+
+	hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
+
+	b := NewBuilder()
+	b.SetIncludeTimestamps(true)
+
+	const files = 10000
+	for i := range files {
+		name := RelFilePath(strconv.FormatInt(int64(i), 36))
+		require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), hash))
+	}
+
+	var buf bytes.Buffer
+	require.NoError(t, b.Build(context.Background(), &buf))
+
+	m, err := NewManifestFromReader(&buf)
+	require.NoError(t, err)
+	assert.Len(t, m.Files(), files)
+}
+
 func TestDeserializeValidManifestRoundTrips(t *testing.T) {
 	t.Parallel()