clawbot
e2deea2e34
check / check (push) Canceled after 0s
Nothing goes in the 1.0 manifest that 1.0 does not read or write, and mfer never wrote either field. mf.proto now reserves the numbers 301 and 303 and both names in MFFilePath, and mf.pb.go is regenerated. The decoded-size check counts only file entries, hashes and timestamps, and counts a file entry, 16 bytes smaller without the two fields, at 160 bytes. export prints no ctime, and FileEntry loses the Ctime nothing set. FORMAT.md and the README no longer describe either field. The test of entries that decode too large gives each entry a second, empty hash: an entry with one hash and a modification time can no longer reach 8 times its size. Model: opus-5-5
499 lines
16 KiB
Go
499 lines
16 KiB
Go
//nolint:testpackage // white-box tests exercise unexported internals
|
|
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/sha512"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
"uuid"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/multiformats/go-multihash"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/protobuf/encoding/protowire"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// craftInnerBytes builds the wire bytes of an inner MFFile holding a single
|
|
// file entry whose path is exactly pathBytes. It writes the wire form by hand
|
|
// so a hostile path — including one that is not valid UTF-8 — can be embedded
|
|
// without proto.Marshal's own UTF-8 enforcement rejecting it first.
|
|
func craftInnerBytes(id uuid.UUID, pathBytes string) []byte {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, pathBytes)
|
|
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
return inner
|
|
}
|
|
|
|
// wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf
|
|
// envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so
|
|
// that deserialization reaches path validation rather than failing earlier on
|
|
// an integrity check.
|
|
func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte {
|
|
t.Helper()
|
|
|
|
var cbuf bytes.Buffer
|
|
|
|
zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
|
require.NoError(t, err)
|
|
|
|
_, err = zw.Write(innerData)
|
|
require.NoError(t, err)
|
|
require.NoError(t, zw.Close())
|
|
|
|
compressed := cbuf.Bytes()
|
|
sum := sha256.Sum256(compressed)
|
|
|
|
outer := &MFFileOuter{
|
|
InnerMessage: compressed,
|
|
Size: int64(len(innerData)),
|
|
Sha256: sum[:],
|
|
Uuid: id[:],
|
|
Version: MFFileOuter_VERSION_ONE,
|
|
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
|
|
}
|
|
|
|
ob, err := proto.Marshal(outer)
|
|
require.NoError(t, err)
|
|
|
|
return append([]byte(MAGIC), ob...)
|
|
}
|
|
|
|
func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
path string
|
|
}{
|
|
{"parent traversal", "../escape"},
|
|
{"interior traversal", "a/../../escape"},
|
|
{"leading dot segment", "./a.txt"},
|
|
{"interior dot segment", "dir/./a.txt"},
|
|
{"absolute path", "/etc/passwd"},
|
|
{"backslash path", `a\b`},
|
|
{"double slash", "a//b"},
|
|
{"empty path", ""},
|
|
{"invalid utf-8", "abc\xff"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
data := wrapInner(t, id, craftInnerBytes(id, tt.path))
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.Error(t, err)
|
|
|
|
if tt.path == "abc\xff" {
|
|
// A path that is not valid UTF-8 cannot survive the proto3
|
|
// string decoder, which rejects it before path validation
|
|
// runs; the manifest is still refused at load time.
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errInvalidManifestPath)
|
|
|
|
if tt.path != "" {
|
|
// ValidatePath quotes the path with %q; assert against the
|
|
// same rendering so escaped characters (e.g. a backslash)
|
|
// still match.
|
|
assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path),
|
|
"error must name the offending path")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A manifest that lists a path twice is refused as it is loaded, naming the
|
|
// path. Paths are compared byte for byte: two that differ only in letter case
|
|
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
|
|
// writes do; entries of a path alone would take too much memory to decode.
|
|
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
paths []string
|
|
refused bool
|
|
}{
|
|
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
|
|
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
|
|
|
|
for _, p := range tt.paths {
|
|
inner.Files = append(inner.Files, &MFFilePath{
|
|
Path: p,
|
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
})
|
|
}
|
|
|
|
innerData, err := proto.Marshal(inner)
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDuplicatePath)
|
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), len(tt.paths))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A file entry must list exactly one SHA-256 hash, and may list hashes of
|
|
// other algorithms beside it. An entry with none, even one that lists a
|
|
// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
|
|
// each refusal names the path. An entry listing a SHA-512 hash and then a
|
|
// SHA-256 one loads with both.
|
|
func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
otherSHA256Hash, err := multihash.Encode(
|
|
bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
hashes [][]byte
|
|
want error // nil for an entry that loads
|
|
}{
|
|
{"no hash", nil, errEntryMissingHash},
|
|
{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
|
|
{
|
|
"two SHA-256 hashes",
|
|
[][]byte{sha256Hash, otherSHA256Hash},
|
|
errEntryExtraSHA256Hash,
|
|
},
|
|
{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
entry := &MFFilePath{Path: "x", Size: 1}
|
|
for _, h := range tt.hashes {
|
|
entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
|
|
}
|
|
|
|
id := uuid.NewV4()
|
|
inner, err := proto.Marshal(&MFFile{
|
|
Version: MFFile_VERSION_ONE,
|
|
Files: []*MFFilePath{entry},
|
|
Uuid: id[:],
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.want == nil {
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
|
|
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, tt.want)
|
|
assert.Contains(t, err.Error(), `"x"`)
|
|
})
|
|
}
|
|
}
|
|
|
|
// A file entry that records a mode with more than the permission bits is
|
|
// refused as it is loaded, naming the path and the mode: once with setuid as
|
|
// Unix writes it (04755) and once as Go keeps it in os.FileMode. An entry
|
|
// recording 0777 loads.
|
|
func TestDeserializeRefusesModeOutsidePermissionBits(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
mode uint32
|
|
want string // "" for an entry that loads
|
|
}{
|
|
{
|
|
"setuid as Unix writes it",
|
|
0o4755,
|
|
`file entry has a mode above 0777: "x" (04755)`,
|
|
},
|
|
{
|
|
"setuid as Go keeps it",
|
|
uint32(os.ModeSetuid | 0o755),
|
|
`file entry has a mode above 0777: "x" (040000755)`,
|
|
},
|
|
{"permission bits only", 0o777, ""},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner, err := proto.Marshal(&MFFile{
|
|
Version: MFFile_VERSION_ONE,
|
|
Files: []*MFFilePath{{
|
|
Path: "x",
|
|
Size: 1,
|
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
Mode: tt.mode,
|
|
}},
|
|
Uuid: id[:],
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.want == "" {
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Equal(t, tt.mode, m.Files()[0].GetMode())
|
|
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errModeOutOfRange)
|
|
require.EqualError(t, err, tt.want)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Entries of a path, a SHA-256 hash, an empty second hash and an empty
|
|
// modification time are counted at 448 bytes each (160 + 112 + 112 + 64)
|
|
// and take 48 bytes plus the path to encode. A 7-character path makes that
|
|
// 55 bytes, about 8.1 times: refused, and leaving any one of the four
|
|
// uncounted, even the modification time, brings it under 8. An 8-character
|
|
// path makes it 56 bytes, 8 times exactly: loaded. Without the second hash
|
|
// an entry is never refused: it takes at least 47 bytes and is counted at
|
|
// 336. There is one entry for each base-36 digit, enough that the 22 bytes
|
|
// of the version and UUID change neither outcome, and each entry's path is
|
|
// its number in base 36, padded with zeros to that length, since a manifest
|
|
// lists a path only once.
|
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
pathLen int
|
|
refused bool
|
|
}{
|
|
{7, true},
|
|
{8, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
for i := range 36 {
|
|
path := strconv.FormatInt(int64(i), 36)
|
|
path = strings.Repeat("0", tt.pathLen-len(path)) + path
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, path)
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, checksum)
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
}
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDecodedTooLarge)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Fields the decoder does not know are dropped, in the outer message, the inner
|
|
// message and a file entry, so that they take no memory once loaded.
|
|
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
|
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
require.NoError(t, err)
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, "a")
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, checksum)
|
|
entry = append(entry, unknown...)
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
inner = append(inner, unknown...)
|
|
|
|
data := wrapInner(t, id, inner)
|
|
data = append(data, unknown...) // the outer message ends the file
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
}
|
|
|
|
// A file entry holding fields 301 and 303, which mf.proto reserves, loads,
|
|
// and the loaded entry keeps neither.
|
|
func TestDeserializeDropsReservedFields(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
require.NoError(t, err)
|
|
|
|
timestamp, err := proto.Marshal(&Timestamp{Seconds: 1})
|
|
require.NoError(t, err)
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, "a")
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, checksum)
|
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // reserved
|
|
entry = protowire.AppendString(entry, "text/plain")
|
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // reserved
|
|
entry = protowire.AppendBytes(entry, timestamp)
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Equal(t, "a", m.Files()[0].GetPath())
|
|
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
}
|
|
|
|
// Many empty files with names of at most three characters and modification
|
|
// times at the epoch make about the densest manifest mfer writes: it takes
|
|
// about 7 times its size to decode, and still loads. A signature would not
|
|
// change the inner message, so none is added.
|
|
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
b.SetIncludeTimestamps(true)
|
|
|
|
const files = 10000
|
|
for i := range files {
|
|
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
|
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), 0, hash))
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), files)
|
|
}
|
|
|
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, 0, hash))
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
|
require.NoError(t, err)
|
|
|
|
files := m.Files()
|
|
require.Len(t, files, 1)
|
|
assert.Equal(t, "dir/file.txt", files[0].GetPath())
|
|
assert.Equal(t, int64(123), files[0].GetSize())
|
|
}
|
|
|
|
// TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest
|
|
// path must be valid UTF-8, independent of the proto decoder that also enforces
|
|
// it on the wire.
|
|
func TestValidatePathRejectsInvalidUTF8(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := ValidatePath("abc\xff")
|
|
require.ErrorIs(t, err, errPathNotUTF8)
|
|
assert.Contains(t, err.Error(), "UTF-8")
|
|
}
|