clawbot
76116005c8
check / check (push) Failing after 2s
Before decoding the manifest, the parser adds up what decoding sets aside for each file entry, hash, timestamp and MIME type, however short its encoding, and refuses the manifest once that passes 8 times the decompressed size; manifests mfer writes come to at most about 7.15 times. Empty entries decoded to about 50 times their size: under 1 KB of manifest allocated about 500 MB. Fields the decoder does not know are dropped; kept, they took up to 5 times more. A test refuses entries counted just over 8 times and loads them just under. The fuzz ceiling rises from 16 to 20 times the input and decompressed data; seeds of empty entries and of empty hashes fail it without the fix. Model: opus-5-5
257 lines
8.1 KiB
Go
257 lines
8.1 KiB
Go
//nolint:testpackage // white-box tests exercise unexported internals
|
|
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/protobuf/encoding/protowire"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// craftInnerBytes builds the wire bytes of an inner MFFile holding a single
|
|
// file entry whose path is exactly pathBytes. It writes the wire form by hand
|
|
// so a hostile path — including one that is not valid UTF-8 — can be embedded
|
|
// without proto.Marshal's own UTF-8 enforcement rejecting it first.
|
|
func craftInnerBytes(id uuid.UUID, pathBytes string) []byte {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, pathBytes)
|
|
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
return inner
|
|
}
|
|
|
|
// wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf
|
|
// envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so
|
|
// that deserialization reaches path validation rather than failing earlier on
|
|
// an integrity check.
|
|
func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte {
|
|
t.Helper()
|
|
|
|
var cbuf bytes.Buffer
|
|
|
|
zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
|
require.NoError(t, err)
|
|
|
|
_, err = zw.Write(innerData)
|
|
require.NoError(t, err)
|
|
require.NoError(t, zw.Close())
|
|
|
|
compressed := cbuf.Bytes()
|
|
sum := sha256.Sum256(compressed)
|
|
|
|
outer := &MFFileOuter{
|
|
InnerMessage: compressed,
|
|
Size: int64(len(innerData)),
|
|
Sha256: sum[:],
|
|
Uuid: id[:],
|
|
Version: MFFileOuter_VERSION_ONE,
|
|
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
|
|
}
|
|
|
|
ob, err := proto.Marshal(outer)
|
|
require.NoError(t, err)
|
|
|
|
return append([]byte(MAGIC), ob...)
|
|
}
|
|
|
|
func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
path string
|
|
}{
|
|
{"parent traversal", "../escape"},
|
|
{"interior traversal", "a/../../escape"},
|
|
{"absolute path", "/etc/passwd"},
|
|
{"backslash path", `a\b`},
|
|
{"double slash", "a//b"},
|
|
{"empty path", ""},
|
|
{"invalid utf-8", "abc\xff"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.New()
|
|
data := wrapInner(t, id, craftInnerBytes(id, tt.path))
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.Error(t, err)
|
|
|
|
if tt.path == "abc\xff" {
|
|
// A path that is not valid UTF-8 cannot survive the proto3
|
|
// string decoder, which rejects it before path validation
|
|
// runs; the manifest is still refused at load time.
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errInvalidManifestPath)
|
|
|
|
if tt.path != "" {
|
|
// ValidatePath quotes the path with %q; assert against the
|
|
// same rendering so escaped characters (e.g. a backslash)
|
|
// still match.
|
|
assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path),
|
|
"error must name the offending path")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
|
// and change times are counted at 416 bytes each (160 + 112 + 16 + 64 + 64)
|
|
// and take 16 bytes plus the path to encode. A 35-character path makes that
|
|
// 51 bytes, about 8.2 times: refused, and leaving any one of the five
|
|
// uncounted, even the MIME type, brings it under 8. A 37-character path makes
|
|
// it 53 bytes, about 7.8 times: loaded.
|
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
pathLen int
|
|
refused bool
|
|
}{
|
|
{35, true},
|
|
{37, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen))
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
|
|
id := uuid.New()
|
|
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
for range 1000 {
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
}
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDecodedTooLarge)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Fields the decoder does not know are dropped, in the outer message, the inner
|
|
// message and a file entry, so that they take no memory once loaded.
|
|
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
|
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, "a")
|
|
entry = append(entry, unknown...)
|
|
|
|
id := uuid.New()
|
|
inner := protowire.AppendTag(nil, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
inner = append(inner, unknown...)
|
|
|
|
data := wrapInner(t, id, inner)
|
|
data = append(data, unknown...) // the outer message ends the file
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
}
|
|
|
|
// Many empty files with names of at most three characters and modification
|
|
// times at the epoch make about the densest manifest mfer writes: it takes
|
|
// about 7 times its size to decode, and still loads. A signature would not
|
|
// change the inner message, so none is added.
|
|
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
|
|
|
b := NewBuilder()
|
|
b.SetIncludeTimestamps(true)
|
|
|
|
const files = 10000
|
|
for i := range files {
|
|
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
|
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), hash))
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), files)
|
|
}
|
|
|
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
|
|
|
b := NewBuilder()
|
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
|
require.NoError(t, err)
|
|
|
|
files := m.Files()
|
|
require.Len(t, files, 1)
|
|
assert.Equal(t, "dir/file.txt", files[0].GetPath())
|
|
assert.Equal(t, int64(123), files[0].GetSize())
|
|
}
|
|
|
|
// TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest
|
|
// path must be valid UTF-8, independent of the proto decoder that also enforces
|
|
// it on the wire.
|
|
func TestValidatePathRejectsInvalidUTF8(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := ValidatePath("abc\xff")
|
|
require.ErrorIs(t, err, errPathNotUTF8)
|
|
assert.Contains(t, err.Error(), "UTF-8")
|
|
}
|