Fuzz NewManifestFromReader and cap the zstd decoder (closes #65)

FuzzNewManifestFromReader fails when the parser returns both or neither
of a manifest and an error, or allocates more than sixteen times its
input plus MaxDecompressedSize. make test runs the committed seed
corpus; make fuzz fuzzes for one minute, by hand only.

Parser bug: MaxDecompressedSize did not bound decompression. The zstd
decoder kept its own 64 GiB limit, set aside whatever size a frame
header claimed, and decoded payloads under 128 KiB in full before the
LimitReader read any of it, so a 92-byte manifest claiming 8 GiB made
the parser allocate 8 GiB. The decoder now has MaxDecompressedSize as
its limit; that manifest is a regression seed.

Model: opus-5-5

Makefile

@@ -13,7 +13,7 @@ GOLDFLAGS += -X main.Version=$(VERSION)
 GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
 GOFLAGS := -ldflags "$(GOLDFLAGS)"
 
-.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme
+.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme
 
 default: fmt test
 
@@ -32,6 +32,9 @@ ci: test
 test:
 	@script/test
 
+fuzz:
+	@script/fuzz
+
 $(PROTOC_GEN_GO):
 	test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
 

README.md

@@ -44,6 +44,9 @@ provide:
   such as `script/docker`
 - `script/test` — run the test suite (`go test`), regenerating the protobuf code
   first if it is stale
+- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
+  (`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
+  as ordinary tests
 - `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
 - `script/fmt` — format all code and docs (writes): `gofumpt`,
   `golangci-lint run --fix`, and `script/prettier --write`

TODO.md

@@ -24,6 +24,10 @@ only thing left of the `chore/align-repo-policies` branch is the list below.
 
 # Completed Steps
 
+- 2026-10-03: added `FuzzNewManifestFromReader` and its seed corpus, which
+  `make test` runs, plus `make fuzz` for a one-minute run by hand; the zstd
+  decoder now has `MaxDecompressedSize` as its limit, so a frame claiming a
+  large size can no longer make the parser allocate that size (#65)
 - 2026-10-03: `script/cibuild` builds the image with the same command as
   `script/docker`, `--no-cache` included, so the checks in the Dockerfile run on
   every build, also on an unchanged tree (#89)
@@ -123,7 +127,7 @@ only thing left of the `chore/align-repo-policies` branch is the list below.
       rate-limit Checker progress output; add --deterministic flag or default;
       wire top-level --version properly
 - Testing:
-    - Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch
+    - End-to-end tests for freshen and fetch
 - Documentation:
     - Promote docs/FORMAT.md as primary spec reference; audit error messages;
       document the signature scheme fully

mfer/deserialize.go

@@ -111,7 +111,12 @@ func (m *manifest) verifyOuterIntegrity() error {
 func (m *manifest) decompressInner() ([]byte, error) {
 	bb := bytes.NewBuffer(m.pbOuter.GetInnerMessage())
 
-	zr, err := zstd.NewReader(bb)
+	// The decoder's own limit is 64 GiB. Up to that it sets aside as many
+	// bytes as a frame header claims, and it decodes a payload under
+	// 128 KiB in full before the LimitReader below reads any of it, so it
+	// needs MaxDecompressedSize as its limit too.
+	zr, err := zstd.NewReader(bb,
+		zstd.WithDecoderMaxMemory(uint64(MaxDecompressedSize)))
 	if err != nil {
 		return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
 	}

mfer/deserialize_fuzz_test.go

@@ -0,0 +1,54 @@
+//nolint:testpackage // white-box tests exercise unexported internals
+package mfer
+
+import (
+	"bytes"
+	"runtime"
+	"testing"
+)
+
+// FuzzNewManifestFromReader feeds arbitrary bytes to the manifest parser.
+// `make test` runs it on the seed corpus in
+// testdata/fuzz/FuzzNewManifestFromReader; `make fuzz` searches for new
+// inputs.
+//
+// For every input the parser must return a manifest or an error, not both
+// and not neither, and must not allocate more than a fixed multiple of its
+// input plus MaxDecompressedSize. A panic or a hang fails the test on its
+// own.
+func FuzzNewManifestFromReader(f *testing.F) {
+	// A signed manifest makes the parser write the key and signature to a
+	// temporary directory and run gpg on them. With gpg off the PATH and
+	// temporary files kept in the test's own directory, no process is
+	// started and nothing is written elsewhere; such input ends in an
+	// error instead.
+	f.Setenv("PATH", "")
+	f.Setenv("TMPDIR", f.TempDir())
+
+	f.Fuzz(func(t *testing.T, data []byte) {
+		var before, after runtime.MemStats
+
+		runtime.ReadMemStats(&before)
+
+		m, err := NewManifestFromReader(bytes.NewReader(data))
+
+		runtime.ReadMemStats(&after)
+
+		if (m == nil) == (err == nil) {
+			t.Fatalf("got manifest %p and error %v, want exactly one", m, err)
+		}
+
+		// The parser keeps a copy of its input and at most
+		// MaxDecompressedSize bytes of decompressed data. Buffers grow by
+		// copying, so reaching those sizes allocates a few times them in
+		// total. Sixteen times leaves room for that, and none for a
+		// decoder that sets aside whatever size its input claims.
+		limit := 16 * (uint64(len(data)) + uint64(MaxDecompressedSize))
+
+		allocated := after.TotalAlloc - before.TotalAlloc
+		if allocated > limit {
+			t.Fatalf("allocated %d bytes for %d bytes of input, limit %d",
+				allocated, len(data), limit)
+		}
+	})
+}

mfer/testdata/fuzz/FuzzNewManifestFromReader/empty

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("")

mfer/testdata/fuzz/FuzzNewManifestFromReader/magic-then-garbage

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFGy[i\x04\xe5O\x82A\x1d\xf4\xb0\xe2z7:U\xee\xa3\xf9\xd6m\xacZ\x9b\xce\x1d\xd9/{@\x1d\xa5y[i\x04\xe5O\x82A\x1d\xf4\xb0\xe2z7:U\xee\xa3\xf9\xd6m\xacZ\x9b\xce\x1d\xd9/{@\x1d\xa5")

mfer/testdata/fuzz/FuzzNewManifestFromReader/minimal

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG\x80")

mfer/testdata/fuzz/FuzzNewManifestFromReader/signed

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG\x80\xca\f\xe8\x03-----BEGIN PGP SIGNATURE-----\n\niQEzBAABCgAdFiEET1Yr+4Y/3GtRtO6IhypRF2zvI64FAmrBIXAACgkQhypRF2zv\nI67BQAf/QrpX2MjY15YGMGkjR5oIhnx/YV96aGYZyZThzb+l/R/N75iVFVkhX21d\nZhQqdCsORrodTPAXic2g2UGVXP9PhNMh7n6Wm3LsvQYjrRQGrQnqtCkut+3tUt8K\n7pt4OAnnwRSieaVImA1COmzxIrQQKNOs6UkgmAstGuPV0XZoeDiSG8TUYJ/vieCn\np5hC0FFXtzfw4NtkxSmkewE0xBxIwFCA/RfSHCGH3m5K+tRz41vMEgGbL1iEp6+V\nuBaoEc4hqCgEt+Af2pA8VHfqeu2vKiwggOpYpaILXZKVqH9+tWHL1EBv9t0vTsYE\n9D57euuR9+kOdngYNPieP1yn5dOSHg==\n=kvgL\n-----END PGP SIGNATURE-----\n\xd2\f(4F562BFB863FDC6B51B4EE88872A51176CEF23AE\xda\f\xb5\a-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQENBGrBIW8BCADESetN5EdxIe7Fafgxl99Yoo5cOexf7wJyYT0wfUYlRaxt3neR\nhir7LOfH4PZWWoDx7qghxCS4+vs7yGypl6JOm7jnJlhn4HneDa2zeIlgGW2TamyE\nua9KPWBQqkFOYmKPmzp+KnL6ncnBLR5mDkNKFyON812KVvteu6Dp/DNk4Meufe44\nWWr49LSFZa9gEbmRCoQGKby9F0H0yIi4FAc74VdQudy0+fMKcfkKjEvByMzlbBEK\n92Hq3sRFzWd3kvPliNjZTmlh5n5m9aBhMpoy3GkKy8gpDdFc6NLA9iAJe7oNMriR\nkVoa5EjQL1xCXAiAWTYA9NScFfU/574sCTxZABEBAAG0Hk1GRVIgVGVzdCBLZXkg\nPHRlc3RAbWZlci50ZXN0PokBTwQTAQoAORYhBE9WK/uGP9xrUbTuiIcqURds7yOu\nBQJqwSFvAxsvBAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCHKlEXbO8jrjml\nCAC8wUK9wmvxq0+NZUpFyP+P29klLZYzBDaBrLPJFs0GjnG4kvfUAktWx0Ro80F7\ncjTJ4f44XjDj4glvSjbe2VaDnZl9FTfzUfG+xjD4462NgntQ4fHk/uG4F6d1ikWx\nkEoMpIn1PlSMas1jTQSGlxUr+zFwWuUbGq4n6hRxEnwLlwJlwQt/Aw1vPDYuPDE3\nOYDhJIAJyP+6e9W8ToaAG9byg/22KA1u1qxnNQqsx5Tped2VltAzdYub+yeCuNc8\nIUo5ILo/fQq3GM5sUEaHjPolv88WlDm3vcdbSbDoh5m2inDtg5zuUKJwu32UGu8l\nKoTjp4nxgQGy5WmeBzs4Hdm/\n=TCB8\n-----END PGP PUBLIC KEY BLOCK-----\n")

mfer/testdata/fuzz/FuzzNewManifestFromReader/size-above-payload

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06!\xc2\x06 \x91\x90*\xa5>\fݐ \x87\xbeaL\xc1\x05?\x0eR\xc18\xa4eՕ\xa95\xb9KʺoZ\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\f-(\xb5/\xfd\x04\x00\x01\x01\x00\xa0\x06\x01\xaa\x06\a\n\x05a.txt\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3a[k'")

mfer/testdata/fuzz/FuzzNewManifestFromReader/size-below-payload

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\x1f\xc2\x06 \x91\x90*\xa5>\fݐ \x87\xbeaL\xc1\x05?\x0eR\xc18\xa4eՕ\xa95\xb9KʺoZ\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\f-(\xb5/\xfd\x04\x00\x01\x01\x00\xa0\x06\x01\xaa\x06\a\n\x05a.txt\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3a[k'")

mfer/testdata/fuzz/FuzzNewManifestFromReader/truncated-at-172

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG")

mfer/testdata/fuzz/FuzzNewManifestFromReader/truncated-at-20

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 ")

mfer/testdata/fuzz/FuzzNewManifestFromReader/truncated-at-4

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAV")

mfer/testdata/fuzz/FuzzNewManifestFromReader/truncated-at-8

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG")

mfer/testdata/fuzz/FuzzNewManifestFromReader/truncated-at-86

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01")

mfer/testdata/fuzz/FuzzNewManifestFromReader/wrong-magic

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFX\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG\x80")

mfer/testdata/fuzz/FuzzNewManifestFromReader/zstd-frame-claims-8gib

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\x01\xc2\x06 \xd6aQ\xa4\x85\xc0+\xbb\xca\x11\x11\a<\x019\x97\xb3\xbb3\xd0 \xd5U\xfa!\xaeAf<N@\x9d\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\f\x12(\xb5/\xfd\xc0\x00\x00\x00\x00\x00\x02\x00\x00\x00\v\x00\x00\x00")

script/fuzz

@@ -0,0 +1,17 @@
+#!/bin/sh
+# script/fuzz: fuzz the manifest parser for one minute. Run by hand only:
+# script/test already runs the committed seed corpus as ordinary tests,
+# and CI never fuzzes. An input that fails is written to
+# mfer/testdata/fuzz/FuzzNewManifestFromReader/; once the parser is fixed,
+# commit it there as a regression seed.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go test -run '^$' -fuzz '^FuzzNewManifestFromReader$' \
+        -fuzztime 1m -parallel 2 ./mfer
+}
+
+main "$@"