clawbot
31e22a9337
check / check (push) Waiting to run
Loading accepted an entry recording a mode above 0777, such as 04755. check then reported a file with exactly the recorded permission bits as a mode mismatch, list and export printed the out-of-range mode, and freshen silently rewrote it as the permission bits alone. Loading now refuses such an entry, naming the path and the mode, so every command that reads a manifest refuses it with one message. fetch's own check for this could no longer be reached and is gone; its test moves to the loading tests in package mfer. docs/FORMAT.md says readers reject a mode above 0777. Model: opus-5-5
460 lines
14 KiB
Go
460 lines
14 KiB
Go
//nolint:testpackage // white-box tests exercise unexported internals
|
|
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/sha512"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
"uuid"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/multiformats/go-multihash"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/protobuf/encoding/protowire"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
// craftInnerBytes builds the wire bytes of an inner MFFile holding a single
|
|
// file entry whose path is exactly pathBytes. It writes the wire form by hand
|
|
// so a hostile path — including one that is not valid UTF-8 — can be embedded
|
|
// without proto.Marshal's own UTF-8 enforcement rejecting it first.
|
|
func craftInnerBytes(id uuid.UUID, pathBytes string) []byte {
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, pathBytes)
|
|
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
return inner
|
|
}
|
|
|
|
// wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf
|
|
// envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so
|
|
// that deserialization reaches path validation rather than failing earlier on
|
|
// an integrity check.
|
|
func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte {
|
|
t.Helper()
|
|
|
|
var cbuf bytes.Buffer
|
|
|
|
zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
|
require.NoError(t, err)
|
|
|
|
_, err = zw.Write(innerData)
|
|
require.NoError(t, err)
|
|
require.NoError(t, zw.Close())
|
|
|
|
compressed := cbuf.Bytes()
|
|
sum := sha256.Sum256(compressed)
|
|
|
|
outer := &MFFileOuter{
|
|
InnerMessage: compressed,
|
|
Size: int64(len(innerData)),
|
|
Sha256: sum[:],
|
|
Uuid: id[:],
|
|
Version: MFFileOuter_VERSION_ONE,
|
|
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
|
|
}
|
|
|
|
ob, err := proto.Marshal(outer)
|
|
require.NoError(t, err)
|
|
|
|
return append([]byte(MAGIC), ob...)
|
|
}
|
|
|
|
func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
path string
|
|
}{
|
|
{"parent traversal", "../escape"},
|
|
{"interior traversal", "a/../../escape"},
|
|
{"leading dot segment", "./a.txt"},
|
|
{"interior dot segment", "dir/./a.txt"},
|
|
{"absolute path", "/etc/passwd"},
|
|
{"backslash path", `a\b`},
|
|
{"double slash", "a//b"},
|
|
{"empty path", ""},
|
|
{"invalid utf-8", "abc\xff"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
data := wrapInner(t, id, craftInnerBytes(id, tt.path))
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.Error(t, err)
|
|
|
|
if tt.path == "abc\xff" {
|
|
// A path that is not valid UTF-8 cannot survive the proto3
|
|
// string decoder, which rejects it before path validation
|
|
// runs; the manifest is still refused at load time.
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errInvalidManifestPath)
|
|
|
|
if tt.path != "" {
|
|
// ValidatePath quotes the path with %q; assert against the
|
|
// same rendering so escaped characters (e.g. a backslash)
|
|
// still match.
|
|
assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path),
|
|
"error must name the offending path")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A manifest that lists a path twice is refused as it is loaded, naming the
|
|
// path. Paths are compared byte for byte: two that differ only in letter case
|
|
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
|
|
// writes do; entries of a path alone would take too much memory to decode.
|
|
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
paths []string
|
|
refused bool
|
|
}{
|
|
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
|
|
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
|
|
|
|
for _, p := range tt.paths {
|
|
inner.Files = append(inner.Files, &MFFilePath{
|
|
Path: p,
|
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
})
|
|
}
|
|
|
|
innerData, err := proto.Marshal(inner)
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDuplicatePath)
|
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), len(tt.paths))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A file entry must list exactly one SHA-256 hash, and may list hashes of
|
|
// other algorithms beside it. An entry with none, even one that lists a
|
|
// SHA-512 hash, is refused as it is loaded, and so is an entry with two;
|
|
// each refusal names the path. An entry listing a SHA-512 hash and then a
|
|
// SHA-256 one loads with both.
|
|
func TestDeserializeRequiresExactlyOneSHA256Hash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
otherSHA256Hash, err := multihash.Encode(
|
|
bytes.Repeat([]byte{1}, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
sha512Hash, err := multihash.Encode(make([]byte, sha512.Size), multihash.SHA2_512)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
hashes [][]byte
|
|
want error // nil for an entry that loads
|
|
}{
|
|
{"no hash", nil, errEntryMissingHash},
|
|
{"SHA-512 hash only", [][]byte{sha512Hash}, errEntryMissingHash},
|
|
{
|
|
"two SHA-256 hashes",
|
|
[][]byte{sha256Hash, otherSHA256Hash},
|
|
errEntryExtraSHA256Hash,
|
|
},
|
|
{"SHA-512 hash then SHA-256 hash", [][]byte{sha512Hash, sha256Hash}, nil},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
entry := &MFFilePath{Path: "x", Size: 1}
|
|
for _, h := range tt.hashes {
|
|
entry.Hashes = append(entry.Hashes, &MFFileChecksum{MultiHash: h})
|
|
}
|
|
|
|
id := uuid.NewV4()
|
|
inner, err := proto.Marshal(&MFFile{
|
|
Version: MFFile_VERSION_ONE,
|
|
Files: []*MFFilePath{entry},
|
|
Uuid: id[:],
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.want == nil {
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Len(t, m.Files()[0].GetHashes(), len(tt.hashes))
|
|
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, tt.want)
|
|
assert.Contains(t, err.Error(), `"x"`)
|
|
})
|
|
}
|
|
}
|
|
|
|
// A file entry that records a mode with more than the permission bits is
|
|
// refused as it is loaded, naming the path and the mode: once with setuid as
|
|
// Unix writes it (04755) and once as Go keeps it in os.FileMode. An entry
|
|
// recording 0777 loads.
|
|
func TestDeserializeRefusesModeOutsidePermissionBits(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
mode uint32
|
|
want string // "" for an entry that loads
|
|
}{
|
|
{
|
|
"setuid as Unix writes it",
|
|
0o4755,
|
|
`file entry has a mode above 0777: "x" (04755)`,
|
|
},
|
|
{
|
|
"setuid as Go keeps it",
|
|
uint32(os.ModeSetuid | 0o755),
|
|
`file entry has a mode above 0777: "x" (040000755)`,
|
|
},
|
|
{"permission bits only", 0o777, ""},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner, err := proto.Marshal(&MFFile{
|
|
Version: MFFile_VERSION_ONE,
|
|
Files: []*MFFilePath{{
|
|
Path: "x",
|
|
Size: 1,
|
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
Mode: tt.mode,
|
|
}},
|
|
Uuid: id[:],
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.want == "" {
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Equal(t, tt.mode, m.Files()[0].GetMode())
|
|
|
|
return
|
|
}
|
|
|
|
require.ErrorIs(t, err, errModeOutOfRange)
|
|
require.EqualError(t, err, tt.want)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Entries of a path, a SHA-256 hash, an empty MIME type and empty
|
|
// modification and change times are counted at 432 bytes each (176 + 112 +
|
|
// 16 + 64 + 64) and take 52 bytes plus the path to encode. A one-character
|
|
// path makes that 53 bytes, about 8.2 times: refused, and leaving any one of
|
|
// the five uncounted, even the MIME type, brings it under 8. A two-character
|
|
// path makes it 54 bytes, 8 times exactly: loaded. There is one entry for
|
|
// each base-36 digit, and each entry's path is its number in base 36, padded
|
|
// with zeros to that length, since a manifest lists a path only once.
|
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
pathLen int
|
|
refused bool
|
|
}{
|
|
{1, true},
|
|
{2, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
|
|
for i := range 36 {
|
|
path := strconv.FormatInt(int64(i), 36)
|
|
path = strings.Repeat("0", tt.pathLen-len(path)) + path
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, path)
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, checksum)
|
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
|
entry = protowire.AppendBytes(entry, nil)
|
|
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
}
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
if tt.refused {
|
|
require.ErrorIs(t, err, errDecodedTooLarge)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Fields the decoder does not know are dropped, in the outer message, the inner
|
|
// message and a file entry, so that they take no memory once loaded.
|
|
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
|
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
checksum, err := proto.Marshal(&MFFileChecksum{MultiHash: hash})
|
|
require.NoError(t, err)
|
|
|
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
entry = protowire.AppendString(entry, "a")
|
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
entry = protowire.AppendBytes(entry, checksum)
|
|
entry = append(entry, unknown...)
|
|
|
|
id := uuid.NewV4()
|
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
inner = protowire.AppendBytes(inner, entry)
|
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
inner = protowire.AppendBytes(inner, id[:])
|
|
inner = append(inner, unknown...)
|
|
|
|
data := wrapInner(t, id, inner)
|
|
data = append(data, unknown...) // the outer message ends the file
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
require.NoError(t, err)
|
|
require.Len(t, m.Files(), 1)
|
|
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
|
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
}
|
|
|
|
// Many empty files with names of at most three characters and modification
|
|
// times at the epoch make about the densest manifest mfer writes: it takes
|
|
// about 7 times its size to decode, and still loads. A signature would not
|
|
// change the inner message, so none is added.
|
|
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
b.SetIncludeTimestamps(true)
|
|
|
|
const files = 10000
|
|
for i := range files {
|
|
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
|
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), 0, hash))
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), files)
|
|
}
|
|
|
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, 0, hash))
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
|
require.NoError(t, err)
|
|
|
|
files := m.Files()
|
|
require.Len(t, files, 1)
|
|
assert.Equal(t, "dir/file.txt", files[0].GetPath())
|
|
assert.Equal(t, int64(123), files[0].GetSize())
|
|
}
|
|
|
|
// TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest
|
|
// path must be valid UTF-8, independent of the proto decoder that also enforces
|
|
// it on the wire.
|
|
func TestValidatePathRejectsInvalidUTF8(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := ValidatePath("abc\xff")
|
|
require.ErrorIs(t, err, errPathNotUTF8)
|
|
assert.Contains(t, err.Error(), "UTF-8")
|
|
}
|