2 Commits
Author SHA1 Message Date
sneak c74318145a AddFileWithHash rejects hashes that are not multihashes (closes #129)
check / check (push) Failing after 2s
AddFileWithHash took any non-empty bytes as a hash, so the builder could
write a manifest that mfer refuses to load. It now decodes the hash with
go-multihash and also requires a digest of at least 32 bytes, the SHA-256
length the reader's decoding-cost limit assumes: a valid but shorter
multihash, such as an empty identity hash or SHA-1, still makes a
manifest of one-character paths too costly to load. Test fixtures that
used 34 zero bytes, which is not a valid multihash, now use a SHA-256
multihash.

Model: opus-5-5
2026-10-04 10:20:58 +00:00
clawbot 0a9963002c NewChecker takes CheckerOptions instead of positional arguments (closes #78)
check / check (push) Failing after 1s
NewChecker now takes *CheckerOptions (ManifestPath, BasePath, Fs), named
like ScannerOptions. A nil Fs still means the OS filesystem, as before and
as in ScannerOptions; nil options or an empty path return an error naming
the missing path.

Audit of the other exported constructors in mfer: NewManifestFromFile took
a filesystem and a path positionally; it now takes
*ManifestFromFileOptions (Path, Fs) with the same nil and empty rules.
NewBuilder and NewScanner take no arguments, NewScannerWithOptions already
takes options, and NewManifestFromReader takes one reader, which the style
guide exempts; these are unchanged.

Model: opus-5-5
2026-10-04 12:19:31 +02:00
12 changed files with 396 additions and 100 deletions

No files matched your search

+5 -1
View File
@@ -298,7 +298,11 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
// Create checker
chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: manifestPath,
BasePath: basePath,
Fs: mfa.Fs,
})
if err != nil {
return fmt.Errorf("failed to load manifest: %w", err)
}
+8 -2
View File
@@ -504,7 +504,10 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
assert.True(t, exists)
// Verify manifest only has 1 file (the non-dotfile)
manifest, err := mfer.NewManifestFromFile(fs, testMF)
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: testMF,
Fs: fs,
})
require.NoError(t, err)
assert.Len(t, manifest.Files(), 1)
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
@@ -528,7 +531,10 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
require.Equal(t, 0, exitCode)
// Verify manifest has 2 files (including dotfile)
manifest, err := mfer.NewManifestFromFile(fs, testMF)
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: testMF,
Fs: fs,
})
require.NoError(t, err)
assert.Len(t, manifest.Files(), 2)
}
+10 -2
View File
@@ -61,7 +61,11 @@ func unsignedChecker(t *testing.T) *mfer.Checker {
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker("/d/index.mf", "/d", fs)
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: "/d/index.mf",
BasePath: "/d",
Fs: fs,
})
require.NoError(t, err)
require.False(t, chk.IsSigned())
@@ -167,7 +171,11 @@ func signedChecker(t *testing.T) *mfer.Checker {
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker("/index.mf", "/", fs)
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: "/index.mf",
BasePath: "/",
Fs: fs,
})
require.NoError(t, err)
require.True(t, chk.IsSigned())
+4 -1
View File
@@ -445,7 +445,10 @@ func (mfa *CLIApp) loadExistingEntries(
log.Infof("loading manifest from %s", manifestPath)
// Load existing manifest
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: manifestPath,
Fs: mfa.Fs,
})
if err != nil {
return nil, fmt.Errorf("failed to load manifest: %w", err)
}
+8 -2
View File
@@ -58,7 +58,10 @@ func TestFreshenUnchanged(t *testing.T) {
setupFreshenDir(t, fs)
// Parse manifest to verify
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: "/testdir/.index.mf",
Fs: fs,
})
require.NoError(t, err)
assert.Len(t, manifest.Files(), 2)
}
@@ -70,7 +73,10 @@ func TestFreshenWithChanges(t *testing.T) {
setupFreshenDir(t, fs)
// Verify initial manifest has 2 files
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
Path: "/testdir/.index.mf",
Fs: fs,
})
require.NoError(t, err)
assert.Len(t, manifest.Files(), 2)
+17 -4
View File
@@ -35,7 +35,8 @@ var (
errPathDotDot = errors.New("contains '..' segment")
errSizeMismatch = errors.New("size mismatch")
errNegativeSize = errors.New("size cannot be negative")
errEmptyHash = errors.New("hash cannot be nil or empty")
errHashNotMultihash = errors.New("hash is not a valid multihash")
errHashTooShort = errors.New("hash digest is too short")
)
// ValidatePath checks that a file path conforms to manifest path invariants:
@@ -228,7 +229,8 @@ func (b *Builder) FileCount() int {
// AddFileWithHash adds a file entry with a pre-computed hash.
// This is useful when the hash is already known (e.g., from an existing manifest).
// Returns an error if path is empty, size is negative, or hash is nil/empty.
// Returns an error if path is invalid, size is negative, or hash is not a
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
func (b *Builder) AddFileWithHash(
path RelFilePath,
size FileSize,
@@ -244,8 +246,19 @@ func (b *Builder) AddFileWithHash(
return errNegativeSize
}
if len(hash) == 0 {
return errEmptyHash
decoded, err := multihash.Decode(hash)
if err != nil {
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
}
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
// hash is at least as long as a SHA-256 multihash, so a manifest of
// shorter ones could fail to load.
if len(decoded.Digest) < sha256.Size {
return fmt.Errorf(
"%w: %d bytes, at least %d needed",
errHashTooShort, len(decoded.Digest), sha256.Size,
)
}
entry := &MFFilePath{
+96 -27
View File
@@ -4,11 +4,15 @@ package mfer
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"testing"
"time"
"github.com/multiformats/go-multihash"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -42,9 +46,10 @@ func TestBuilderAddFileWithHash(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34) // SHA256 multihash is 34 bytes
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
require.NoError(t, err)
assert.Equal(t, 1, b.FileCount())
}
@@ -52,12 +57,14 @@ func TestBuilderAddFileWithHash(t *testing.T) {
func TestBuilderAddFileWithHashValidation(t *testing.T) {
t.Parallel()
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
t.Run("empty path", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34)
err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
err := b.AddFileWithHash("", 100, ModTime(time.Now()), sha256Hash)
require.Error(t, err)
assert.Contains(t, err.Error(), "path")
})
@@ -66,41 +73,79 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34)
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), sha256Hash)
require.Error(t, err)
assert.Contains(t, err.Error(), "size")
})
t.Run("nil hash", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "hash")
})
t.Run("empty hash", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
require.Error(t, err)
assert.Contains(t, err.Error(), "hash")
})
t.Run("valid inputs", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34)
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), sha256Hash)
require.NoError(t, err)
assert.Equal(t, 1, b.FileCount())
})
}
func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
t.Parallel()
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
require.NoError(t, err)
tests := []struct {
name string
hash Multihash
want error
}{
{"nil hash", nil, errHashNotMultihash},
{"empty hash", []byte{}, errHashNotMultihash},
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
// A SHA-256 code and 32-byte length, then only two bytes of digest.
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
// A valid multihash, but its 20-byte SHA-1 digest is too short.
{"SHA-1 multihash", sha1Hash, errHashTooShort},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
b := NewBuilder()
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), tt.hash)
require.ErrorIs(t, err, tt.want)
assert.Equal(t, 0, b.FileCount())
})
}
}
// Entries made from the shortest inputs AddFileWithHash accepts (a
// one-character path, an empty file, a modification time at the epoch and
// a SHA-256 multihash) are the most costly to decode for their size, and a
// manifest of them still loads.
func TestBuilderShortestEntriesLoad(t *testing.T) {
t.Parallel()
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
const names = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
b := NewBuilder()
for _, name := range strings.Split(names, "") {
err = b.AddFileWithHash(RelFilePath(name), 0, ModTime(time.Unix(0, 0)), hash)
require.NoError(t, err)
}
var buf bytes.Buffer
require.NoError(t, b.Build(context.Background(), &buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.Len(t, m.Files(), len(names))
}
func TestBuilderBuild(t *testing.T) {
t.Parallel()
@@ -356,7 +401,8 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
t.Parallel()
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
b := NewBuilder()
@@ -397,6 +443,29 @@ func TestNewManifestFromReaderTruncated(t *testing.T) {
assert.Error(t, err)
}
func TestNewManifestFromFileRequiresPath(t *testing.T) {
t.Parallel()
_, err := NewManifestFromFile(nil)
require.ErrorIs(t, err, errManifestPathEmpty)
_, err = NewManifestFromFile(&ManifestFromFileOptions{Fs: afero.NewMemMapFs()})
require.ErrorIs(t, err, errManifestPathEmpty)
}
func TestNewManifestFromFileNilFsUsesOsFs(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "index.mf")
createTestManifest(t, afero.NewOsFs(), path, map[string][]byte{
testFileName: []byte("hello"),
})
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
require.NoError(t, err)
assert.Len(t, m.Files(), 1)
}
func TestManifestString(t *testing.T) {
t.Parallel()
+34 -8
View File
@@ -14,7 +14,11 @@ import (
"github.com/spf13/afero"
)
var errNoSigningPubKey = errors.New("manifest has no signing public key")
var (
errNoSigningPubKey = errors.New("manifest has no signing public key")
errManifestPathEmpty = errors.New("manifest path cannot be empty")
errBasePathEmpty = errors.New("base path cannot be empty")
)
// Result represents the outcome of checking a single file.
type Result struct {
@@ -82,20 +86,42 @@ type Checker struct {
signingPubKey []byte
}
// NewChecker creates a new Checker for the given manifest, base path, and filesystem.
// The basePath is the directory relative to which manifest paths are resolved.
// If fs is nil, the real filesystem (OsFs) is used.
func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, error) {
// CheckerOptions configures a Checker.
type CheckerOptions struct {
// ManifestPath is the manifest file to check against (required).
ManifestPath string
// BasePath is the directory relative to which manifest paths are
// resolved (required).
BasePath string
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
}
// NewChecker creates a new Checker with the given options. It returns an
// error if opts is nil or either path is empty.
func NewChecker(opts *CheckerOptions) (*Checker, error) {
if opts == nil || opts.ManifestPath == "" {
return nil, errManifestPathEmpty
}
if opts.BasePath == "" {
return nil, errBasePathEmpty
}
fs := opts.Fs
if fs == nil {
fs = afero.NewOsFs()
}
m, err := NewManifestFromFile(fs, manifestPath)
m, err := NewManifestFromFile(&ManifestFromFileOptions{
Path: opts.ManifestPath,
Fs: fs,
})
if err != nil {
return nil, err
}
abs, err := filepath.Abs(basePath)
abs, err := filepath.Abs(opts.BasePath)
if err != nil {
return nil, err
}
@@ -108,7 +134,7 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
}
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
absManifest, err := filepath.Abs(manifestPath)
absManifest, err := filepath.Abs(opts.ManifestPath)
if err != nil {
return nil, err
}
+190 -46
View File
@@ -5,6 +5,8 @@ import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
@@ -14,9 +16,11 @@ import (
)
const (
testFile1 = "file1.txt"
testFile2 = "file2.txt"
testExistsFile = "exists.txt"
testFile1 = "file1.txt"
testFile2 = "file2.txt"
testExistsFile = "exists.txt"
testManifestPath = "/manifest.mf"
testDataDir = "/data"
)
func TestStatusString(t *testing.T) {
@@ -66,15 +70,13 @@ func createTestManifest(
}
// createFilesOnDisk creates the given files on the filesystem under
// /data.
// testDataDir.
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
t.Helper()
basePath := "/data"
for path, content := range files {
fullPath := basePath + "/" + path
require.NoError(t, fs.MkdirAll(basePath, 0o755))
fullPath := testDataDir + "/" + path
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
}
}
@@ -90,9 +92,13 @@ func TestNewChecker(t *testing.T) {
testFile1: []byte("hello"),
testFile2: []byte("world"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
chk, err := NewChecker("/manifest.mf", "/", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: "/",
Fs: fs,
})
require.NoError(t, err)
assert.NotNil(t, chk)
assert.Equal(t, FileCount(2), chk.FileCount())
@@ -102,7 +108,11 @@ func TestNewChecker(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
_, err := NewChecker("/nonexistent.mf", "/", fs)
_, err := NewChecker(&CheckerOptions{
ManifestPath: "/nonexistent.mf",
BasePath: "/",
Fs: fs,
})
assert.Error(t, err)
})
@@ -111,11 +121,73 @@ func TestNewChecker(t *testing.T) {
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
_, err := NewChecker("/bad.mf", "/", fs)
_, err := NewChecker(&CheckerOptions{
ManifestPath: "/bad.mf",
BasePath: "/",
Fs: fs,
})
assert.Error(t, err)
})
}
func TestNewCheckerRequiredPaths(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
opts *CheckerOptions
want string
is error
}{
{
name: "nil options",
opts: nil,
want: "manifest path cannot be empty",
is: errManifestPathEmpty,
},
{
name: "empty manifest path",
opts: &CheckerOptions{BasePath: testDataDir},
want: "manifest path cannot be empty",
is: errManifestPathEmpty,
},
{
name: "empty base path",
opts: &CheckerOptions{ManifestPath: testManifestPath},
want: "base path cannot be empty",
is: errBasePathEmpty,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
chk, err := NewChecker(tc.opts)
require.ErrorIs(t, err, tc.is)
require.EqualError(t, err, tc.want)
assert.Nil(t, chk)
})
}
}
func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
t.Parallel()
dir := t.TempDir()
manifestPath := filepath.Join(dir, "index.mf")
content := []byte("hello")
createTestManifest(t, afero.NewOsFs(), manifestPath, map[string][]byte{
testFile1: content,
})
require.NoError(t, os.WriteFile(filepath.Join(dir, testFile1), content, 0o600))
chk, err := NewChecker(&CheckerOptions{ManifestPath: manifestPath, BasePath: dir})
require.NoError(t, err)
results := make(chan Result, 1)
require.NoError(t, chk.Check(context.Background(), results, nil))
assert.Equal(t, StatusOK, (<-results).Status)
}
func TestCheckerFileCountAndTotalBytes(t *testing.T) {
t.Parallel()
@@ -125,9 +197,13 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
"medium.txt": []byte("hello world"),
"large.txt": bytes.Repeat([]byte("x"), 1000),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
chk, err := NewChecker("/manifest.mf", "/", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: "/",
Fs: fs,
})
require.NoError(t, err)
assert.Equal(t, FileCount(3), chk.FileCount())
@@ -142,10 +218,14 @@ func TestCheckAllFilesOK(t *testing.T) {
testFile1: []byte("content one"),
testFile2: []byte("content two"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -172,13 +252,17 @@ func TestCheckMissingFile(t *testing.T) {
testExistsFile: []byte("I exist"),
"missing.txt": []byte("I don't exist on disk"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
// Only create one file
createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("I exist"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -211,13 +295,17 @@ func TestCheckSizeMismatch(t *testing.T) {
files := map[string][]byte{
testFileName: []byte("original content"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
// Create file with different size
createFilesOnDisk(t, fs, map[string][]byte{
testFileName: []byte("short"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -237,7 +325,7 @@ func TestCheckHashMismatch(t *testing.T) {
files := map[string][]byte{
testFileName: originalContent,
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
// Create file with same size but different content
differentContent := []byte("different contnt") // same length (16 bytes) but different
require.Len(t, differentContent, len(originalContent), "test requires same length")
@@ -245,7 +333,11 @@ func TestCheckHashMismatch(t *testing.T) {
testFileName: differentContent,
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -265,10 +357,14 @@ func TestCheckWithProgress(t *testing.T) {
testFile1: bytes.Repeat([]byte("a"), 100),
testFile2: bytes.Repeat([]byte("b"), 200),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -305,10 +401,14 @@ func TestCheckContextCancellation(t *testing.T) {
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
@@ -327,7 +427,7 @@ func TestFindExtraFiles(t *testing.T) {
manifestFiles := map[string][]byte{
testFile1: []byte("in manifest"),
}
createTestManifest(t, fs, "/manifest.mf", manifestFiles)
createTestManifest(t, fs, testManifestPath, manifestFiles)
// Disk has file1 and file2
createFilesOnDisk(t, fs, map[string][]byte{
@@ -335,7 +435,11 @@ func TestFindExtraFiles(t *testing.T) {
testFile2: []byte("extra file"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -368,10 +472,14 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
// Create a real extra file
require.NoError(t, fs.MkdirAll("/data", 0o755))
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: "/data/.index.mf",
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -400,10 +508,14 @@ func TestFindExtraFilesContextCancellation(t *testing.T) {
fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
@@ -419,10 +531,14 @@ func TestCheckNilChannels(t *testing.T) {
fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
// Should not panic with nil channels
@@ -435,10 +551,14 @@ func TestFindExtraFilesNilChannel(t *testing.T) {
fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
// Should not panic with nil channel
@@ -455,7 +575,7 @@ func TestCheckSubdirectories(t *testing.T) {
"dir1/dir2/file2.txt": []byte("content2"),
"dir1/dir2/dir3/deep.txt": []byte("deep content"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
// Create files with full directory structure
for path, content := range files {
@@ -465,7 +585,11 @@ func TestCheckSubdirectories(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
}
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -493,13 +617,17 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
testExistsFile: []byte("here"),
"missing.txt": []byte("not on disk"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
// Only create one file on disk
createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("here"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -537,7 +665,11 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
require.NoError(t,
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: "/data/.index.mf",
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -566,7 +698,11 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
createTestManifest(t, fs, "/data/index.mf", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/data/index.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: "/data/index.mf",
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 10)
@@ -587,9 +723,13 @@ func TestCheckEmptyManifest(t *testing.T) {
fs := afero.NewMemMapFs()
// Create manifest with no files
createTestManifest(t, fs, "/manifest.mf", map[string][]byte{})
createTestManifest(t, fs, testManifestPath, map[string][]byte{})
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
assert.Equal(t, FileCount(0), chk.FileCount())
@@ -621,10 +761,14 @@ func TestCheckProgressRateLimited(t *testing.T) {
files[name] = []byte("content")
}
createTestManifest(t, fs, "/manifest.mf", files)
createTestManifest(t, fs, testManifestPath, files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
chk, err := NewChecker(&CheckerOptions{
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err)
results := make(chan Result, 200)
+2 -1
View File
@@ -36,7 +36,8 @@ const (
decodedMIMETypeSize = 16
// Each file entry mfer writes holds a path of at least one byte, a
// 34-byte SHA-256 multihash and a modification time: at least 47 bytes,
// multihash at least as long as SHA-256's 34 bytes (AddFileWithHash
// refuses shorter ones) and a modification time: at least 47 bytes,
// counted at 336. So its manifests add up to at most about 7.15 times
// their size, and this limit is about 12% above that.
maxDecodedGrowth = 8
+17 -4
View File
@@ -352,16 +352,29 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
return m, nil
}
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
// If fs is nil, the real filesystem (OsFs) is used.
// ManifestFromFileOptions configures NewManifestFromFile.
type ManifestFromFileOptions struct {
// Path is the manifest file to read (required).
Path string
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
}
// NewManifestFromFile reads a manifest from a file. It returns an error if
// opts is nil or its path is empty.
//
//nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
func NewManifestFromFile(opts *ManifestFromFileOptions) (*manifest, error) {
if opts == nil || opts.Path == "" {
return nil, errManifestPathEmpty
}
fs := opts.Fs
if fs == nil {
fs = afero.NewOsFs()
}
f, err := fs.Open(path)
f, err := fs.Open(opts.Path)
if err != nil {
return nil, err
}
+5 -2
View File
@@ -13,6 +13,7 @@ import (
"github.com/google/uuid"
"github.com/klauspost/compress/zstd"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protowire"
@@ -205,7 +206,8 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
func TestDeserializeLoadsDensestManifest(t *testing.T) {
t.Parallel()
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
b := NewBuilder()
b.SetIncludeTimestamps(true)
@@ -227,7 +229,8 @@ func TestDeserializeLoadsDensestManifest(t *testing.T) {
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
t.Parallel()
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
b := NewBuilder()
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))