Build a static binary so the scratch image runs (closes #126)

The final stage is scratch, which has no C library, but the builder
compiled mfer with cgo on (the golang image's default). The binary
imports net (mfer's own HTTP code does, and so does google/uuid), so
with cgo on it came out dynamically linked and the image could not
start. The image's go build now sets CGO_ENABLED=0; nothing in mfer
needs cgo. A new builder step runs ldd on the binary and fails the
build unless it reports a static executable.

Model: opus-5-5

Dockerfile

@@ -67,7 +67,10 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
         exit 1; \
     fi; \
     cd cmd/mfer && \
-    go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
+    CGO_ENABLED=0 go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
+
+# Fail unless /mfer is statically linked: scratch has no C library to run it.
+RUN ldd /mfer 2>&1 | grep -q 'not a dynamic executable'
 
 FROM scratch
 COPY --from=builder /mfer /mfer

mfer/constants.go

@@ -17,24 +17,4 @@ const (
 
 	// uuidLength is the length in bytes of a binary UUID.
 	uuidLength = 16
-
-	// Numbers in mf.proto of MFFile.files and of the MFFilePath fields
-	// that decoding sets aside a fixed amount of memory for.
-	filesFieldNumber    = 101
-	hashesFieldNumber   = 3
-	mimeTypeFieldNumber = 301
-	mtimeFieldNumber    = 302
-	ctimeFieldNumber    = 303
-
-	// Bytes decoding sets aside for each file entry, hash, timestamp and
-	// MIME type, however short its encoding. checkDecodedSize refuses an
-	// inner message for which these add up to more than maxDecodedGrowth
-	// times its size.
-	decodedFileEntrySize = 160
-	decodedHashSize      = 112
-	decodedTimestampSize = 64
-	decodedMIMETypeSize  = 16
-
-	// The densest manifests mfer writes add up to about 7 times their size.
-	maxDecodedGrowth = 8
 )

mfer/deserialize.go

@@ -11,7 +11,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/afero"
-	"google.golang.org/protobuf/encoding/protowire"
 	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/internal/bork"
 	"sneak.berlin/go/mfer/internal/log"
@@ -28,8 +27,6 @@ var (
 	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
 	errInvalidFileFormat    = errors.New("invalid file format")
 	errInvalidManifestPath  = errors.New("manifest contains invalid path")
-	errDecodedTooLarge      = errors.New(
-		"manifest would take too much memory to decode")
 )
 
 // validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
@@ -157,85 +154,6 @@ func (m *manifest) decompressInner() ([]byte, error) {
 	return dat, nil
 }
 
-// checkDecodedSize refuses an encoded inner message whose file entries,
-// hashes, timestamps and MIME types would take more than maxDecodedGrowth
-// times its size to decode. Decoding sets aside a fixed amount for each,
-// however short its encoding, so a message of empty ones would take about
-// 50 times its size.
-func checkDecodedSize(inner []byte) error {
-	limit := maxDecodedGrowth * int64(len(inner))
-
-	var decoded int64
-
-	add := func(size int64) error {
-		decoded += size
-		if decoded > limit {
-			return errDecodedTooLarge
-		}
-
-		return nil
-	}
-
-	return forEachBytesField(inner, func(num protowire.Number, entry []byte) error {
-		if num != filesFieldNumber {
-			return nil
-		}
-
-		err := add(decodedFileEntrySize)
-		if err != nil {
-			return err
-		}
-
-		return forEachBytesField(entry, func(num protowire.Number, _ []byte) error {
-			if num == hashesFieldNumber {
-				return add(decodedHashSize)
-			}
-
-			if num == mtimeFieldNumber || num == ctimeFieldNumber {
-				return add(decodedTimestampSize)
-			}
-
-			if num == mimeTypeFieldNumber {
-				return add(decodedMIMETypeSize)
-			}
-
-			return nil
-		})
-	})
-}
-
-// forEachBytesField calls fn with the number and value of each
-// length-delimited field in the encoded message msg, and fails if msg is
-// malformed.
-func forEachBytesField(
-	msg []byte, fn func(num protowire.Number, value []byte) error,
-) error {
-	for len(msg) > 0 {
-		num, wireType, tagLen := protowire.ConsumeTag(msg)
-		if tagLen < 0 {
-			return protowire.ParseError(tagLen)
-		}
-
-		valueLen := protowire.ConsumeFieldValue(num, wireType, msg[tagLen:])
-		if valueLen < 0 {
-			return protowire.ParseError(valueLen)
-		}
-
-		if wireType == protowire.BytesType {
-			value, _ := protowire.ConsumeBytes(msg[tagLen:])
-
-			err := fn(num, value)
-			if err != nil {
-				return err
-			}
-		}
-
-		msg = msg[tagLen+valueLen:]
-	}
-
-	return nil
-}
-
 func (m *manifest) deserializeInner() error {
 	err := m.validateOuterHeader()
 	if err != nil {
@@ -259,11 +177,6 @@ func (m *manifest) deserializeInner() error {
 		return bork.ErrFileTruncated
 	}
 
-	err = checkDecodedSize(dat)
-	if err != nil {
-		return fmt.Errorf("deserialize: unmarshal inner: %w", err)
-	}
-
 	// Deserialize inner message
 	m.pbInner = new(MFFile)
 

mfer/deserialize_fuzz_test.go

@@ -54,14 +54,9 @@ func FuzzNewManifestFromReader(f *testing.F) {
 		}
 
 		// It also keeps a few copies of its input. Buffers grow by
-		// copying, so reaching those sizes allocates up to about six times
-		// them in total. Decoding the decompressed data takes up to
-		// maxDecodedGrowth times its size for file entries, hashes,
-		// timestamps and MIME types, and up to about five times more for
-		// the bytes it copies out of it, such as fields it does not know,
-		// which it keeps in buffers that also grow by copying. Twenty
-		// times the input and the decompressed data leaves room for all of
-		// that.
+		// copying, so reaching those sizes allocates a few times them in
+		// total: sixteen times the input and the decompressed data leaves
+		// room for that.
 		//
 		// The decoder also sets aside a new buffer of one to two times the
 		// window for each frame that asks for a larger window than the
@@ -76,10 +71,8 @@ func FuzzNewManifestFromReader(f *testing.F) {
 		// fails if the decoder accepts windows of twice zstdWindowSize; the
 		// seed whose two frames together exceed MaxDecompressedSize fails
 		// if the decoder decodes them in full instead of stopping at the
-		// declared size; the seeds of empty file entries, of a file entry
-		// of empty hashes, and of file entries of only an empty MIME type
-		// and empty times fail if the parser decodes them.
-		limit := 20*(uint64(len(data))+decompressed) + 24*zstdWindowSize
+		// declared size.
+		limit := 16*(uint64(len(data))+decompressed) + 24*zstdWindowSize
 
 		allocated := after.TotalAlloc - before.TotalAlloc
 		if allocated > limit {

mfer/deserialize_path_test.go

@@ -6,9 +6,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
-	"strconv"
 	"testing"
-	"time"
 
 	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
@@ -116,57 +114,6 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
 	}
 }
 
-// Entries of a one-character path and empty modification and change times
-// pass every other check, but would take about 23 times their size to decode.
-func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
-	t.Parallel()
-
-	entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
-	entry = protowire.AppendString(entry, "a")
-	entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
-	entry = protowire.AppendBytes(entry, nil)
-	entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
-	entry = protowire.AppendBytes(entry, nil)
-
-	id := uuid.New()
-	inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
-	inner = protowire.AppendBytes(inner, id[:])
-
-	for range 1000 {
-		inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
-		inner = protowire.AppendBytes(inner, entry)
-	}
-
-	_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
-	require.ErrorIs(t, err, errDecodedTooLarge)
-}
-
-// Many empty files with names of at most three characters and modification
-// times at the epoch make about the densest manifest mfer writes: it takes
-// about 7 times its size to decode, and still loads. A signature would not
-// change the inner message, so none is added.
-func TestDeserializeLoadsDensestManifest(t *testing.T) {
-	t.Parallel()
-
-	hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
-
-	b := NewBuilder()
-	b.SetIncludeTimestamps(true)
-
-	const files = 10000
-	for i := range files {
-		name := RelFilePath(strconv.FormatInt(int64(i), 36))
-		require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), hash))
-	}
-
-	var buf bytes.Buffer
-	require.NoError(t, b.Build(context.Background(), &buf))
-
-	m, err := NewManifestFromReader(&buf)
-	require.NoError(t, err)
-	assert.Len(t, m.Files(), files)
-}
-
 func TestDeserializeValidManifestRoundTrips(t *testing.T) {
 	t.Parallel()