//nolint:testpackage // white-box tests exercise unexported internals package mfer import ( "bytes" "context" "crypto/sha256" "fmt" "strconv" "testing" "time" "uuid" "github.com/klauspost/compress/zstd" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" ) // craftInnerBytes builds the wire bytes of an inner MFFile holding a single // file entry whose path is exactly pathBytes. It writes the wire form by hand // so a hostile path — including one that is not valid UTF-8 — can be embedded // without proto.Marshal's own UTF-8 enforcement rejecting it first. func craftInnerBytes(id uuid.UUID, pathBytes string) []byte { entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path entry = protowire.AppendString(entry, pathBytes) inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE)) inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files inner = protowire.AppendBytes(inner, entry) inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid inner = protowire.AppendBytes(inner, id[:]) return inner } // wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf // envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so // that deserialization reaches path validation rather than failing earlier on // an integrity check. func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte { t.Helper() var cbuf bytes.Buffer zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) require.NoError(t, err) _, err = zw.Write(innerData) require.NoError(t, err) require.NoError(t, zw.Close()) compressed := cbuf.Bytes() sum := sha256.Sum256(compressed) outer := &MFFileOuter{ InnerMessage: compressed, Size: int64(len(innerData)), Sha256: sum[:], Uuid: id[:], Version: MFFileOuter_VERSION_ONE, CompressionType: MFFileOuter_COMPRESSION_ZSTD, } ob, err := proto.Marshal(outer) require.NoError(t, err) return append([]byte(MAGIC), ob...) } func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) { t.Parallel() tests := []struct { name string path string }{ {"parent traversal", "../escape"}, {"interior traversal", "a/../../escape"}, {"absolute path", "/etc/passwd"}, {"backslash path", `a\b`}, {"double slash", "a//b"}, {"empty path", ""}, {"invalid utf-8", "abc\xff"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() id := uuid.NewV4() data := wrapInner(t, id, craftInnerBytes(id, tt.path)) _, err := NewManifestFromReader(bytes.NewReader(data)) require.Error(t, err) if tt.path == "abc\xff" { // A path that is not valid UTF-8 cannot survive the proto3 // string decoder, which rejects it before path validation // runs; the manifest is still refused at load time. return } require.ErrorIs(t, err, errInvalidManifestPath) if tt.path != "" { // ValidatePath quotes the path with %q; assert against the // same rendering so escaped characters (e.g. a backslash) // still match. assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path), "error must name the offending path") } }) } } // A manifest that lists a path twice is refused as it is loaded, naming the // path. Paths are compared byte for byte: two that differ only in letter case // load, and fetch refuses those itself. Each entry has a hash, as entries mfer // writes do; entries of a path alone would take too much memory to decode. func TestDeserializeRefusesPathListedTwice(t *testing.T) { t.Parallel() hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256) require.NoError(t, err) tests := []struct { name string paths []string refused bool }{ {"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true}, {"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() id := uuid.NewV4() inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]} for _, p := range tt.paths { inner.Files = append(inner.Files, &MFFilePath{ Path: p, Hashes: []*MFFileChecksum{{MultiHash: hash}}, }) } innerData, err := proto.Marshal(inner) require.NoError(t, err) m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData))) if tt.refused { require.ErrorIs(t, err, errDuplicatePath) require.EqualError(t, err, `duplicate path "dir/a.txt"`) } else { require.NoError(t, err) assert.Len(t, m.Files(), len(tt.paths)) } }) } } // Entries of a path, an empty hash, an empty MIME type and empty modification // and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64) // and take 16 bytes plus the path to encode. A 37-character path makes that // 53 bytes, about 8.2 times: refused, and leaving any one of the five // uncounted, even the MIME type, brings it under 8. A 39-character path makes // it 55 bytes, about 7.9 times: loaded. Each entry's path is its number, // padded with zeros to that length, since a manifest lists a path only once. func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) { t.Parallel() tests := []struct { pathLen int refused bool }{ {37, true}, {39, false}, } for _, tt := range tests { t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) { t.Parallel() id := uuid.NewV4() inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE)) inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid inner = protowire.AppendBytes(inner, id[:]) for i := range 1000 { entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i)) entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes entry = protowire.AppendBytes(entry, nil) entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType entry = protowire.AppendBytes(entry, nil) entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime entry = protowire.AppendBytes(entry, nil) entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime entry = protowire.AppendBytes(entry, nil) inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files inner = protowire.AppendBytes(inner, entry) } _, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner))) if tt.refused { require.ErrorIs(t, err, errDecodedTooLarge) } else { require.NoError(t, err) } }) } } // Fields the decoder does not know are dropped, in the outer message, the inner // message and a file entry, so that they take no memory once loaded. func TestDeserializeDropsUnknownFields(t *testing.T) { t.Parallel() unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message unknown = protowire.AppendBytes(unknown, []byte("not known")) entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path entry = protowire.AppendString(entry, "a") entry = append(entry, unknown...) id := uuid.NewV4() inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE)) inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files inner = protowire.AppendBytes(inner, entry) inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid inner = protowire.AppendBytes(inner, id[:]) inner = append(inner, unknown...) data := wrapInner(t, id, inner) data = append(data, unknown...) // the outer message ends the file m, err := NewManifestFromReader(bytes.NewReader(data)) require.NoError(t, err) require.Len(t, m.Files(), 1) assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown()) assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown()) assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown()) } // Many empty files with names of at most three characters and modification // times at the epoch make about the densest manifest mfer writes: it takes // about 7 times its size to decode, and still loads. A signature would not // change the inner message, so none is added. func TestDeserializeLoadsDensestManifest(t *testing.T) { t.Parallel() hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256) require.NoError(t, err) b := NewBuilder() b.SetIncludeTimestamps(true) const files = 10000 for i := range files { name := RelFilePath(strconv.FormatInt(int64(i), 36)) require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), 0, hash)) } var buf bytes.Buffer require.NoError(t, b.Build(context.Background(), &buf)) m, err := NewManifestFromReader(&buf) require.NoError(t, err) assert.Len(t, m.Files(), files) } func TestDeserializeValidManifestRoundTrips(t *testing.T) { t.Parallel() hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256) require.NoError(t, err) b := NewBuilder() require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, 0, hash)) var buf bytes.Buffer require.NoError(t, b.Build(context.Background(), &buf)) m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes())) require.NoError(t, err) files := m.Files() require.Len(t, files, 1) assert.Equal(t, "dir/file.txt", files[0].GetPath()) assert.Equal(t, int64(123), files[0].GetSize()) } // TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest // path must be valid UTF-8, independent of the proto decoder that also enforces // it on the wire. func TestValidatePathRejectsInvalidUTF8(t *testing.T) { t.Parallel() err := ValidatePath("abc\xff") require.ErrorIs(t, err, errPathNotUTF8) assert.Contains(t, err.Error(), "UTF-8") }