clawbot
4bf87d1ccf
check / check (push) Failing after 2s
AddFileWithHash took any non-empty bytes as a hash, so the builder could write a manifest that mfer refuses to load. It now decodes the hash with go-multihash and also requires a digest of at least 32 bytes, the SHA-256 length the reader's decoding-cost limit assumes: a valid but shorter multihash, such as an empty identity hash or SHA-1, still makes a manifest of one-character paths too costly to load. When mfer freshen meets such a hash in an existing manifest, its error names the manifest entry and says to regenerate the manifest with mfer generate. Test fixtures whose stand-in hashes were not valid multihashes now use a SHA-256 multihash. Model: opus-5-5
562 lines
14 KiB
Go
562 lines
14 KiB
Go
//nolint:testpackage // white-box tests exercise unexported internals
|
|
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/multiformats/go-multihash"
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const testFileName = "file.txt"
|
|
|
|
func TestNewBuilder(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
assert.NotNil(t, b)
|
|
assert.Equal(t, 0, b.FileCount())
|
|
}
|
|
|
|
func TestBuilderAddFile(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := []byte("test content")
|
|
reader := bytes.NewReader(content)
|
|
|
|
bytesRead, err := b.AddFile(
|
|
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, FileSize(len(content)), bytesRead)
|
|
assert.Equal(t, 1, b.FileCount())
|
|
}
|
|
|
|
func TestBuilderAddFileWithHash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, b.FileCount())
|
|
}
|
|
|
|
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
t.Run("empty path", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
err := b.AddFileWithHash("", 100, ModTime(time.Now()), sha256Hash)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "path")
|
|
})
|
|
|
|
t.Run("negative size", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), sha256Hash)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "size")
|
|
})
|
|
|
|
t.Run("valid inputs", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), sha256Hash)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, b.FileCount())
|
|
})
|
|
}
|
|
|
|
func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
|
require.NoError(t, err)
|
|
|
|
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
hash Multihash
|
|
want error
|
|
}{
|
|
{"nil hash", nil, errHashNotMultihash},
|
|
{"empty hash", []byte{}, errHashNotMultihash},
|
|
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
|
|
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
|
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
|
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
|
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
|
// A valid multihash whose 31-byte digest is one byte short.
|
|
{"31-byte digest", truncatedHash, errHashTooShort},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), tt.hash)
|
|
require.ErrorIs(t, err, tt.want)
|
|
assert.Equal(t, 0, b.FileCount())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuilderBuild(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := []byte("test content")
|
|
reader := bytes.NewReader(content)
|
|
|
|
_, err := b.AddFile(
|
|
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err = b.Build(context.Background(), &buf)
|
|
require.NoError(t, err)
|
|
|
|
// Should have magic bytes
|
|
assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
|
|
}
|
|
|
|
func TestNewTimestampFromTimeExtremeDate(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Regression test: newTimestampFromTime used UnixNano() which panics
|
|
// for dates outside ~1678-2262. Now uses Nanosecond() which is safe.
|
|
tests := []struct {
|
|
name string
|
|
time time.Time
|
|
}{
|
|
{"zero time", time.Time{}},
|
|
{"year 1000", time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC)},
|
|
{"year 3000", time.Date(3000, 1, 1, 0, 0, 0, 123456789, time.UTC)},
|
|
{"unix epoch", time.Unix(0, 0)},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Should not panic
|
|
ts := newTimestampFromTime(tt.time)
|
|
assert.Equal(t, tt.time.Unix(), ts.GetSeconds())
|
|
assert.Equal(t, tt.time.Nanosecond(), int(ts.GetNanos()))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuilderDeterministicOutput(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
buildManifest := func() []byte {
|
|
b := NewBuilder()
|
|
// Use a fixed createdAt and UUID so output is reproducible
|
|
b.createdAt = time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
b.fixedUUID = make([]byte, 16) // all zeros
|
|
|
|
mtime := ModTime(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC))
|
|
|
|
// Add files in reverse order to test sorting
|
|
files := []struct {
|
|
path string
|
|
content string
|
|
}{
|
|
{"c/file.txt", "content c"},
|
|
{"a/file.txt", "content a"},
|
|
{"b/file.txt", "content b"},
|
|
}
|
|
for _, f := range files {
|
|
r := bytes.NewReader([]byte(f.content))
|
|
_, err := b.AddFile(
|
|
RelFilePath(f.path), FileSize(len(f.content)), mtime, r, nil,
|
|
)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := b.Build(context.Background(), &buf)
|
|
require.NoError(t, err)
|
|
|
|
return buf.Bytes()
|
|
}
|
|
|
|
out1 := buildManifest()
|
|
out2 := buildManifest()
|
|
assert.Equal(t, out1, out2,
|
|
"two builds with same input should produce byte-identical output")
|
|
}
|
|
|
|
func TestSetSeedDeterministic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b1 := NewBuilder()
|
|
b1.SetSeed("test-seed-value")
|
|
|
|
b2 := NewBuilder()
|
|
b2.SetSeed("test-seed-value")
|
|
assert.Equal(t, b1.fixedUUID, b2.fixedUUID, "same seed should produce same UUID")
|
|
assert.Len(t, b1.fixedUUID, 16, "UUID should be 16 bytes")
|
|
|
|
b3 := NewBuilder()
|
|
b3.SetSeed("different-seed")
|
|
assert.NotEqual(t, b1.fixedUUID, b3.fixedUUID,
|
|
"different seeds should produce different UUIDs")
|
|
}
|
|
|
|
func TestValidatePath(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
valid := []string{
|
|
testFileName,
|
|
"dir/file.txt",
|
|
"a/b/c/d.txt",
|
|
"file with spaces.txt",
|
|
"日本語.txt", //nolint:gosmopolitan // deliberately tests non-ASCII UTF-8 paths
|
|
}
|
|
for _, p := range valid {
|
|
t.Run("valid:"+p, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
assert.NoError(t, ValidatePath(p))
|
|
})
|
|
}
|
|
|
|
invalid := []struct {
|
|
path string
|
|
desc string
|
|
}{
|
|
{"", "empty"},
|
|
{"/absolute", "absolute path"},
|
|
{"has\\backslash", "backslash"},
|
|
{"has/../traversal", "dot-dot segment"},
|
|
{"has//double", "empty segment"},
|
|
{"..", "just dot-dot"},
|
|
{string([]byte{0xff, 0xfe}), "invalid UTF-8"},
|
|
}
|
|
for _, tt := range invalid {
|
|
t.Run("invalid:"+tt.desc, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
assert.Error(t, ValidatePath(tt.path))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuilderAddFileSizeMismatch(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := []byte("short")
|
|
reader := bytes.NewReader(content)
|
|
|
|
// Declare wrong size
|
|
_, err := b.AddFile("test.txt", FileSize(100), ModTime(time.Now()), reader, nil)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "size mismatch")
|
|
}
|
|
|
|
func TestBuilderAddFileInvalidPath(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := []byte("data")
|
|
reader := bytes.NewReader(content)
|
|
|
|
_, err := b.AddFile("", FileSize(len(content)), ModTime(time.Now()), reader, nil)
|
|
require.Error(t, err)
|
|
|
|
reader.Reset(content)
|
|
_, err = b.AddFile(
|
|
"/absolute", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
)
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestBuilderAddFileWithProgress(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := bytes.Repeat([]byte("x"), 1000)
|
|
reader := bytes.NewReader(content)
|
|
progress := make(chan FileHashProgress, 100)
|
|
|
|
bytesRead, err := b.AddFile(
|
|
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, progress,
|
|
)
|
|
close(progress)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, FileSize(1000), bytesRead)
|
|
|
|
var updates []FileHashProgress
|
|
for p := range progress {
|
|
updates = append(updates, p)
|
|
}
|
|
|
|
assert.NotEmpty(t, updates)
|
|
// Last update should show all bytes
|
|
assert.Equal(t, FileSize(1000), updates[len(updates)-1].BytesRead)
|
|
}
|
|
|
|
func TestBuilderBuildRoundTrip(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Build a manifest, deserialize it, verify all fields survive round-trip
|
|
b := NewBuilder()
|
|
now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
|
|
|
|
files := []struct {
|
|
path string
|
|
content []byte
|
|
}{
|
|
{"alpha.txt", []byte("alpha content")},
|
|
{"beta/gamma.txt", []byte("gamma content")},
|
|
{"beta/delta.txt", []byte("delta content")},
|
|
}
|
|
|
|
for _, f := range files {
|
|
reader := bytes.NewReader(f.content)
|
|
_, err := b.AddFile(
|
|
RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), reader, nil,
|
|
)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
|
|
mfiles := m.Files()
|
|
require.Len(t, mfiles, 3)
|
|
|
|
// Verify sorted order
|
|
assert.Equal(t, "alpha.txt", mfiles[0].GetPath())
|
|
assert.Equal(t, "beta/delta.txt", mfiles[1].GetPath())
|
|
assert.Equal(t, "beta/gamma.txt", mfiles[2].GetPath())
|
|
|
|
// Verify sizes
|
|
assert.Equal(t, int64(len("alpha content")), mfiles[0].GetSize())
|
|
|
|
// Verify hashes are present
|
|
for _, f := range mfiles {
|
|
require.NotEmpty(t, f.GetHashes(), "file %s should have hashes", f.GetPath())
|
|
assert.NotEmpty(t, f.GetHashes()[0].GetMultiHash())
|
|
}
|
|
}
|
|
|
|
// A manifest whose payload is larger than one zstd block (128 KiB) is
|
|
// written as a frame asking for the writer's whole window, zstdWindowSize,
|
|
// which is the most the parser accepts.
|
|
func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
require.NoError(t, err)
|
|
|
|
b := NewBuilder()
|
|
|
|
for i := range 4000 {
|
|
path := RelFilePath(fmt.Sprintf("dir/file-%05d.txt", i))
|
|
require.NoError(t, b.AddFileWithHash(path, FileSize(i), ModTime{}, hash))
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
require.Greater(t, m.pbOuter.GetSize(), int64(128<<10))
|
|
assert.Len(t, m.Files(), 4000)
|
|
}
|
|
|
|
func TestNewManifestFromReaderInvalidMagic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader([]byte("NOT_VALID")))
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "invalid file format")
|
|
}
|
|
|
|
func TestNewManifestFromReaderEmpty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
_, err := NewManifestFromReader(bytes.NewReader([]byte{}))
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestNewManifestFromReaderTruncated(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Just the magic with nothing after
|
|
_, err := NewManifestFromReader(bytes.NewReader([]byte(MAGIC)))
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestNewManifestFromFileRequiresPath(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
_, err := NewManifestFromFile(nil)
|
|
require.ErrorIs(t, err, errManifestPathEmpty)
|
|
|
|
_, err = NewManifestFromFile(&ManifestFromFileOptions{Fs: afero.NewMemMapFs()})
|
|
require.ErrorIs(t, err, errManifestPathEmpty)
|
|
}
|
|
|
|
func TestNewManifestFromFileNilFsUsesOsFs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
path := filepath.Join(t.TempDir(), "index.mf")
|
|
createTestManifest(t, afero.NewOsFs(), path, map[string][]byte{
|
|
testFileName: []byte("hello"),
|
|
})
|
|
|
|
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
|
|
require.NoError(t, err)
|
|
assert.Len(t, m.Files(), 1)
|
|
}
|
|
|
|
func TestManifestString(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := []byte("test")
|
|
reader := bytes.NewReader(content)
|
|
_, err := b.AddFile(
|
|
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Contains(t, m.String(), "count=1")
|
|
}
|
|
|
|
func TestBuilderBuildEmpty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
|
|
var buf bytes.Buffer
|
|
|
|
err := b.Build(context.Background(), &buf)
|
|
require.NoError(t, err)
|
|
|
|
// Should still produce valid manifest with 0 files
|
|
assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
|
|
}
|
|
|
|
func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
content := []byte("hello")
|
|
_, err := b.AddFile(
|
|
"test.txt", FileSize(len(content)), ModTime(time.Now()),
|
|
bytes.NewReader(content), nil,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.Nil(t, m.pbInner.GetCreatedAt(),
|
|
"createdAt should be nil by default for deterministic output")
|
|
}
|
|
|
|
func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
b := NewBuilder()
|
|
b.SetIncludeTimestamps(true)
|
|
|
|
content := []byte("hello")
|
|
_, err := b.AddFile(
|
|
"test.txt", FileSize(len(content)), ModTime(time.Now()),
|
|
bytes.NewReader(content), nil,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
assert.NotNil(t, m.pbInner.GetCreatedAt(),
|
|
"createdAt should be set when IncludeTimestamps is true")
|
|
}
|
|
|
|
func TestBuilderDeterministicFileOrder(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Two builds with same files in different order should produce same file ordering.
|
|
// Note: UUIDs differ per build, so we compare parsed file lists, not raw bytes.
|
|
buildAndParse := func(order []string) []*MFFilePath {
|
|
b := NewBuilder()
|
|
|
|
for _, name := range order {
|
|
content := []byte("content of " + name)
|
|
_, err := b.AddFile(
|
|
RelFilePath(name), FileSize(len(content)),
|
|
ModTime(time.Unix(1000, 0)), bytes.NewReader(content), nil,
|
|
)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
require.NoError(t, b.Build(context.Background(), &buf))
|
|
m, err := NewManifestFromReader(&buf)
|
|
require.NoError(t, err)
|
|
|
|
return m.Files()
|
|
}
|
|
|
|
files1 := buildAndParse([]string{"b.txt", "a.txt"})
|
|
files2 := buildAndParse([]string{"a.txt", "b.txt"})
|
|
|
|
require.Len(t, files1, 2)
|
|
require.Len(t, files2, 2)
|
|
|
|
for i := range files1 {
|
|
assert.Equal(t, files1[i].GetPath(), files2[i].GetPath())
|
|
assert.Equal(t, files1[i].GetSize(), files2[i].GetSize())
|
|
}
|
|
|
|
assert.Equal(t, "a.txt", files1[0].GetPath())
|
|
assert.Equal(t, "b.txt", files1[1].GetPath())
|
|
}
|