clawbot
61238e4071
check / check (push) Waiting to run
Loading accepted a file entry whose hashes list was empty. check and fetch then failed the file as a hash mismatch, and freshen left it out of the rewritten manifest without a word. NewManifestFromReader now refuses such an entry, naming its path, after checking the path itself. freshen's addExistingToBuilder returns an error for an entry with no hash instead of skipping it. The file entry table in docs/FORMAT.md says readers reject an entry with none. Model: opus-5-5
332 lines
11 KiB
Go
332 lines
11 KiB
Go
//nolint:testpackage // white-box tests exercise unexported internals
|
|
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
"uuid"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/multiformats/go-multihash"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/protobuf/encoding/protowire"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// craftInnerBytes builds the wire bytes of an inner MFFile holding a single
|
|
// file entry whose path is exactly pathBytes. It writes the wire form by hand
|
|
// so a hostile path — including one that is not valid UTF-8 — can be embedded
|
|
// without proto.Marshal's own UTF-8 enforcement rejecting it first.
|
|
func craftInnerBytes(id uuid.UUID, pathBytes string) []byte {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, pathBytes)
|
|
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
return inner
|
|
}
|
|
|
|
// wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf
|
|
// envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so
|
|
// that deserialization reaches path validation rather than failing earlier on
|
|
// an integrity check.
|
|
func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte {
|
|
t.Helper()
|
|
|
|
var cbuf bytes.Buffer
|
|
|
|
zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
|
require.NoError(t, err)
|
|
|
|
_, err = zw.Write(innerData)
|
|
require.NoError(t, err)
|
|
require.NoError(t, zw.Close())
|
|
|
|
compressed := cbuf.Bytes()
|
|
sum := sha256.Sum256(compressed)
|
|
|
|
outer := &MFFileOuter{
|
|
InnerMessage: compressed,
|
|
Size: int64(len(innerData)),
|
|
Sha256: sum[:],
|
|
Uuid: id[:],
|
|
Version: MFFileOuter_VERSION_ONE,
|
|
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
|
|
}
|
|
|
|
ob, err := proto.Marshal(outer)
|
|
require.NoError(t, err)
|
|
|
|
return append([]byte(MAGIC), ob...)
|
|
}
|
|
|
|
func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
path string
|
|
}{
|
|
{"parent traversal", "../escape"},
|
|
{"interior traversal", "a/../../escape"},
|
|
{"absolute path", "/etc/passwd"},
|
|
{"backslash path", `a\b`},
|
|
{"double slash", "a//b"},
|
|
{"empty path", ""},
|
|
{"invalid utf-8", "abc\xff"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
data := wrapInner(t, id, craftInnerBytes(id, tt.path))
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.Error(t, err)
|
|
|
|
if tt.path == "abc\xff" {
|
|
// A path that is not valid UTF-8 cannot survive the proto3
|
|
// string decoder, which rejects it before path validation
|
|
// runs; the manifest is still refused at load time.
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errInvalidManifestPath)
|
|
|
|
if tt.path != "" {
|
|
// ValidatePath quotes the path with %q; assert against the
|
|
// same rendering so escaped characters (e.g. a backslash)
|
|
// still match.
|
|
assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path),
|
|
"error must name the offending path")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A manifest that lists a path twice is refused as it is loaded, naming the
|
|
// path. Paths are compared byte for byte: two that differ only in letter case
|
|
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
|
|
// writes do; entries of a path alone would take too much memory to decode.
|
|
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
paths []string
|
|
refused bool
|
|
}{
|
|
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
|
|
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
|
|
|
|
for _, p := range tt.paths {
|
|
inner.Files = append(inner.Files, &MFFilePath{
|
|
Path: p,
|
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
})
|
|
}
|
|
|
|
innerData, err := proto.Marshal(inner)
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDuplicatePath)
|
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), len(tt.paths))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A manifest holding a file entry with no hash is refused as it is loaded,
|
|
// naming the path: there is nothing to check the file against.
|
|
func TestDeserializeRefusesEntryWithNoHash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner, err := proto.Marshal(&MFFile{
|
|
Version: MFFile_VERSION_ONE,
|
|
Files: []*MFFilePath{{Path: "x", Size: 1}},
|
|
Uuid: id[:],
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, err = NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
require.ErrorIs(t, err, errEntryMissingHash)
|
|
require.EqualError(t, err, `file entry has no hash: "x"`)
|
|
}
|
|
|
|
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
|
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
|
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
|
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
|
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
|
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
|
|
// padded with zeros to that length, since a manifest lists a path only once.
|
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
pathLen int
|
|
refused bool
|
|
}{
|
|
{37, true},
|
|
{39, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
for i := range 1000 {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
}
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDecodedTooLarge)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Fields the decoder does not know are dropped, in the outer message, the inner
|
|
// message and a file entry, so that they take no memory once loaded.
|
|
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
|
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, "a")
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = append(entry, unknown...)
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
inner = append(inner, unknown...)
|
|
|
|
data := wrapInner(t, id, inner)
|
|
data = append(data, unknown...) // the outer message ends the file
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
}
|
|
|
|
// Many empty files with names of at most three characters and modification
|
|
// times at the epoch make about the densest manifest mfer writes: it takes
|
|
// about 7 times its size to decode, and still loads. A signature would not
|
|
// change the inner message, so none is added.
|
|
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
b.SetIncludeTimestamps(true)
|
|
|
|
const files = 10000
|
|
for i := range files {
|
|
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
|
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), 0, hash))
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), files)
|
|
}
|
|
|
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, 0, hash))
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
|
require.NoError(t, err)
|
|
|
|
files := m.Files()
|
|
require.Len(t, files, 1)
|
|
assert.Equal(t, "dir/file.txt", files[0].GetPath())
|
|
assert.Equal(t, int64(123), files[0].GetSize())
|
|
}
|
|
|
|
// TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest
|
|
// path must be valid UTF-8, independent of the proto decoder that also enforces
|
|
// it on the wire.
|
|
func TestValidatePathRejectsInvalidUTF8(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := ValidatePath("abc\xff")
|
|
require.ErrorIs(t, err, errPathNotUTF8)
|
|
assert.Contains(t, err.Error(), "UTF-8")
|
|
}
|