clawbot
4fe1ff2fe1
check / check (push) Waiting to run
gen given arguments whose files share a path, such as gen a b with a.txt in both, or gen . ., now fails while listing the files, before hashing any, naming the path and both files. Builder.AddFile and Builder.AddFileWithHash refuse a path already added. Loading refuses a manifest that lists a path twice, compared byte for byte; fetch keeps its own letter-case check. The Path Rules in docs/FORMAT.md say each path appears at most once. The decode-size test listed one path 1000 times; each entry now has its own path of the same length. Model: opus-5-5
312 lines
10 KiB
Go
312 lines
10 KiB
Go
//nolint:testpackage // white-box tests exercise unexported internals
|
|
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
"uuid"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/multiformats/go-multihash"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/protobuf/encoding/protowire"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// craftInnerBytes builds the wire bytes of an inner MFFile holding a single
|
|
// file entry whose path is exactly pathBytes. It writes the wire form by hand
|
|
// so a hostile path — including one that is not valid UTF-8 — can be embedded
|
|
// without proto.Marshal's own UTF-8 enforcement rejecting it first.
|
|
func craftInnerBytes(id uuid.UUID, pathBytes string) []byte {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, pathBytes)
|
|
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
return inner
|
|
}
|
|
|
|
// wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf
|
|
// envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so
|
|
// that deserialization reaches path validation rather than failing earlier on
|
|
// an integrity check.
|
|
func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte {
|
|
t.Helper()
|
|
|
|
var cbuf bytes.Buffer
|
|
|
|
zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
|
require.NoError(t, err)
|
|
|
|
_, err = zw.Write(innerData)
|
|
require.NoError(t, err)
|
|
require.NoError(t, zw.Close())
|
|
|
|
compressed := cbuf.Bytes()
|
|
sum := sha256.Sum256(compressed)
|
|
|
|
outer := &MFFileOuter{
|
|
InnerMessage: compressed,
|
|
Size: int64(len(innerData)),
|
|
Sha256: sum[:],
|
|
Uuid: id[:],
|
|
Version: MFFileOuter_VERSION_ONE,
|
|
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
|
|
}
|
|
|
|
ob, err := proto.Marshal(outer)
|
|
require.NoError(t, err)
|
|
|
|
return append([]byte(MAGIC), ob...)
|
|
}
|
|
|
|
func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
path string
|
|
}{
|
|
{"parent traversal", "../escape"},
|
|
{"interior traversal", "a/../../escape"},
|
|
{"absolute path", "/etc/passwd"},
|
|
{"backslash path", `a\b`},
|
|
{"double slash", "a//b"},
|
|
{"empty path", ""},
|
|
{"invalid utf-8", "abc\xff"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
data := wrapInner(t, id, craftInnerBytes(id, tt.path))
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.Error(t, err)
|
|
|
|
if tt.path == "abc\xff" {
|
|
// A path that is not valid UTF-8 cannot survive the proto3
|
|
// string decoder, which rejects it before path validation
|
|
// runs; the manifest is still refused at load time.
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errInvalidManifestPath)
|
|
|
|
if tt.path != "" {
|
|
// ValidatePath quotes the path with %q; assert against the
|
|
// same rendering so escaped characters (e.g. a backslash)
|
|
// still match.
|
|
assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path),
|
|
"error must name the offending path")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A manifest that lists a path twice is refused as it is loaded, naming the
|
|
// path. Paths are compared byte for byte: two that differ only in letter case
|
|
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
|
|
// writes do; entries of a path alone would take too much memory to decode.
|
|
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
paths []string
|
|
refused bool
|
|
}{
|
|
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
|
|
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
|
|
|
|
for _, p := range tt.paths {
|
|
inner.Files = append(inner.Files, &MFFilePath{
|
|
Path: p,
|
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
})
|
|
}
|
|
|
|
innerData, err := proto.Marshal(inner)
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDuplicatePath)
|
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), len(tt.paths))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
|
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
|
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
|
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
|
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
|
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
|
|
// padded with zeros to that length, since a manifest lists a path only once.
|
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
pathLen int
|
|
refused bool
|
|
}{
|
|
{37, true},
|
|
{39, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
for i := range 1000 {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
}
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDecodedTooLarge)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Fields the decoder does not know are dropped, in the outer message, the inner
|
|
// message and a file entry, so that they take no memory once loaded.
|
|
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
|
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, "a")
|
|
entry = append(entry, unknown...)
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
inner = append(inner, unknown...)
|
|
|
|
data := wrapInner(t, id, inner)
|
|
data = append(data, unknown...) // the outer message ends the file
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
}
|
|
|
|
// Many empty files with names of at most three characters and modification
|
|
// times at the epoch make about the densest manifest mfer writes: it takes
|
|
// about 7 times its size to decode, and still loads. A signature would not
|
|
// change the inner message, so none is added.
|
|
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
b.SetIncludeTimestamps(true)
|
|
|
|
const files = 10000
|
|
for i := range files {
|
|
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
|
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), 0, hash))
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), files)
|
|
}
|
|
|
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, 0, hash))
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
|
require.NoError(t, err)
|
|
|
|
files := m.Files()
|
|
require.Len(t, files, 1)
|
|
assert.Equal(t, "dir/file.txt", files[0].GetPath())
|
|
assert.Equal(t, int64(123), files[0].GetSize())
|
|
}
|
|
|
|
// TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest
|
|
// path must be valid UTF-8, independent of the proto decoder that also enforces
|
|
// it on the wire.
|
|
func TestValidatePathRejectsInvalidUTF8(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := ValidatePath("abc\xff")
|
|
require.ErrorIs(t, err, errPathNotUTF8)
|
|
assert.Contains(t, err.Error(), "UTF-8")
|
|
}
|