2 Commits
Author SHA1 Message Date
clawbot ae0599c0cb Refuse a path listed twice in a manifest (closes #170)
check / check (push) Canceled after 0s
gen given arguments whose files share a path, such as gen a b with a.txt
in both, or gen . ., now fails while listing the files, before hashing
any, naming the path and both files. Builder.AddFile and
Builder.AddFileWithHash refuse a path already added. Loading refuses a
manifest that lists a path twice, compared byte for byte; fetch keeps
its own letter-case check. The Path Rules in docs/FORMAT.md say each
path appears at most once. The decode-size test listed one path 1000
times; each entry now has its own path of the same length.

Model: opus-5-5
2026-10-07 12:35:14 +00:00
clawbot f663f4242d Limit how much fetch and check read for a manifest or a file (closes #168)
check / check (push) Canceled after 0s
NewManifestFromReader reads at most one byte past MaxManifestSize, a new
constant of 258 MiB: the 256 MiB decompressed limit grown by zstd's worst
case of 1/256, plus 1 MiB for the signature, the signing key and the
other outer fields. It refuses a larger manifest. fetch, and check given
a URL, stop downloading a manifest one byte past the same size and report
it as too large; tests lower that size to keep their memory small. fetch
stops reading a file one byte past its listed size, so a longer body ends
in the size mismatch at once instead of filling the disk. docs/FORMAT.md
states the limit and gives the decompressed limit as 256 MiB, the size
the code uses.

Model: opus-5-5
2026-10-07 14:25:45 +02:00
15 changed files with 342 additions and 34 deletions

No files matched your search

+9 -3
View File
@@ -50,11 +50,14 @@ allows verifying data integrity before decompression.
The `innerMessage` field is compressed with The `innerMessage` field is compressed with
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must [Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
enforce a decompression size limit to prevent decompression bombs. The reference enforce a decompression size limit to prevent decompression bombs. The reference
implementation limits decompressed size to 256 MB. It writes zstd frames with a implementation limits decompressed size to 256 MiB. It writes zstd frames with a
window of at most 8 MiB, the largest window the zstd format recommends decoders window of at most 8 MiB, the largest window the zstd format recommends decoders
support, and refuses frames that ask for a larger one. It also refuses an inner support, and refuses frames that ask for a larger one. It also refuses an inner
message whose file entries, hashes, timestamps and MIME types, counted at 176, message whose file entries, hashes, timestamps and MIME types, counted at 176,
112, 64 and 16 bytes each, add up to more than 8 times its size. 112, 64 and 16 bytes each, add up to more than 8 times its size. It refuses a
manifest file larger than 258 MiB without reading the rest of it: zstd's worst
case grows a 256 MiB inner message by 1/256 to 257 MiB, and the last MiB is room
for the signature, the signing key and the other outer fields.
## Inner Message (`MFFile`) ## Inner Message (`MFFile`)
@@ -106,9 +109,12 @@ All `path` values must satisfy these invariants:
- **No parent traversal**: no `..` path segments - **No parent traversal**: no `..` path segments
- **No empty segments**: no `//` sequences - **No empty segments**: no `//` sequences
- **No trailing slash**: paths refer to files, not directories - **No trailing slash**: paths refer to files, not directories
- **Listed once**: each path appears at most once in a manifest, compared byte
for byte, so `A.txt` and `a.txt` are two paths
Implementations must validate these invariants when reading and writing Implementations must validate these invariants when reading and writing
manifests. Paths that violate these rules must be rejected. manifests. Paths that violate these rules must be rejected, and a reader must
reject a manifest that lists a path more than once.
## Hash Format (`MFFileChecksum`) ## Hash Format (`MFFileChecksum`)
+8 -1
View File
@@ -112,10 +112,17 @@ func (mfa *CLIApp) fetchManifestToTemp(
} }
tmpPath := tmpFile.Name() tmpPath := tmpFile.Name()
_, cpErr := io.Copy(tmpFile, rc)
// Copying stops one byte past mfa.maxManifestSize, which is enough to
// tell that the manifest is too large.
written, cpErr := io.Copy(tmpFile, io.LimitReader(rc, mfa.maxManifestSize+1))
_ = rc.Close() _ = rc.Close()
_ = tmpFile.Close() _ = tmpFile.Close()
if cpErr == nil && written > mfa.maxManifestSize {
cpErr = fmt.Errorf("%w of %d bytes", errManifestTooLarge, mfa.maxManifestSize)
}
if cpErr != nil { if cpErr != nil {
_ = mfa.Fs.Remove(tmpPath) _ = mfa.Fs.Remove(tmpPath)
+2
View File
@@ -5,6 +5,7 @@ import (
"os" "os"
"github.com/spf13/afero" "github.com/spf13/afero"
"sneak.berlin/go/mfer/mfer"
) )
// NoColor disables colored output when set. Automatically true if the // NoColor disables colored output when set. Automatically true if the
@@ -60,6 +61,7 @@ func RunWithOptions(opts *RunOptions) int {
version: opts.Version, version: opts.Version,
gitrev: opts.Gitrev, gitrev: opts.Gitrev,
exitCode: 0, exitCode: 0,
maxManifestSize: mfer.MaxManifestSize,
Stdin: opts.Stdin, Stdin: opts.Stdin,
Stdout: opts.Stdout, Stdout: opts.Stdout,
Stderr: opts.Stderr, Stderr: opts.Stderr,
+38
View File
@@ -354,6 +354,44 @@ func TestGenerateCommand(t *testing.T) {
assert.True(t, exists) assert.True(t, exists)
} }
// TestGenerateRefusesTwoFilesAtOnePath runs gen on arguments whose files
// would share a path in the manifest: two directories that each hold a.txt,
// and one directory given twice. gen must fail while it lists the files,
// before it hashes any, naming the path, and write no manifest.
func TestGenerateRefusesTwoFilesAtOnePath(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
first, second string
}{
{"two directories", testDir, "/other"},
{"one directory twice", testDir, testDir},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll(testDir, 0o755))
require.NoError(t, fs.MkdirAll("/other", 0o755))
writeTestFile(t, fs, "/testdir/a.txt", "first")
writeTestFile(t, fs, "/other/a.txt", "second")
opts := testOpts([]string{
testApp, cmdGenerate, "-q", "-o", testOutput, tc.first, tc.second,
}, fs)
assert.Equal(t, 1, runCLI(opts))
assert.Contains(t, testStderr(t, opts),
`generate: failed to enumerate paths: duplicate path "a.txt": `+
tc.first+"/a.txt and "+tc.second+"/a.txt")
exists, err := afero.Exists(fs, testOutput)
require.NoError(t, err)
assert.False(t, exists)
})
}
}
// TestGenerateSeededManifestBytes pins the exact bytes `gen --seed` writes // TestGenerateSeededManifestBytes pins the exact bytes `gen --seed` writes
// for a fixed tree, so that a Go or dependency update that changes what // for a fixed tree, so that a Go or dependency update that changes what
// mfer writes fails here. testdata/seeded.mf was written by an mfer built // mfer writes fails here. testdata/seeded.mf was written by an mfer built
+15 -6
View File
@@ -361,7 +361,7 @@ func (mfa *CLIApp) fetchManifestOperation(
firstDelay: firstRetryDelay, firstDelay: firstRetryDelay,
} }
manifestData, files, err := fetchManifest(ctx, cmd, client, manifestURL) manifestData, files, err := mfa.fetchManifest(ctx, cmd, client, manifestURL)
if err != nil { if err != nil {
return err return err
} }
@@ -426,20 +426,22 @@ func (mfa *CLIApp) fetchManifestOperation(
// that lists a file where fetch writes another or a mode outside 0777. It // that lists a file where fetch writes another or a mode outside 0777. It
// returns the manifest as downloaded, to be saved once the files are in // returns the manifest as downloaded, to be saved once the files are in
// place, and the files it lists. // place, and the files it lists.
func fetchManifest( func (mfa *CLIApp) fetchManifest(
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string, ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
) ([]byte, []*mfer.MFFilePath, error) { ) ([]byte, []*mfer.MFFilePath, error) {
log.Infof("fetching manifest from %s", manifestURL) log.Infof("fetching manifest from %s", manifestURL)
// Read the whole manifest before parsing it, so that a connection // Read the whole manifest before parsing it, so that a connection
// lost partway through is retried rather than reported as a bad // lost partway through is retried rather than reported as a bad
// manifest. // manifest. Reading stops one byte past mfa.maxManifestSize, which is
// enough to tell that the manifest is too large.
var manifestData []byte var manifestData []byte
err := client.get(ctx, manifestURL, func(resp *http.Response) error { err := client.get(ctx, manifestURL, func(resp *http.Response) error {
var readErr error var readErr error
manifestData, readErr = io.ReadAll(resp.Body) manifestData, readErr = io.ReadAll(
io.LimitReader(resp.Body, mfa.maxManifestSize+1))
return readErr return readErr
}) })
@@ -447,6 +449,11 @@ func fetchManifest(
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err) return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
} }
if int64(len(manifestData)) > mfa.maxManifestSize {
return nil, nil, fmt.Errorf("failed to fetch manifest: %w of %d bytes",
errManifestTooLarge, mfa.maxManifestSize)
}
// Parse manifest // Parse manifest
//nolint:contextcheck // mfer loads a manifest without a context //nolint:contextcheck // mfer loads a manifest without a context
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData)) manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
@@ -908,8 +915,10 @@ func saveResponse(
progress: progress, progress: progress,
} }
// Copy content while hashing and reporting progress // Copy content while hashing and reporting progress. One byte past
written, copyErr := io.Copy(pw, resp.Body) // the listed size is enough for finishDownload to report a size
// mismatch.
written, copyErr := io.Copy(pw, io.LimitReader(resp.Body, expectedSize+1))
// Close file before checking errors (to flush writes) // Close file before checking errors (to flush writes)
closeErr := out.Close() closeErr := out.Close()
+70
View File
@@ -26,6 +26,7 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
urfcli "github.com/urfave/cli/v3"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
"sneak.berlin/go/mfer/mfer" "sneak.berlin/go/mfer/mfer"
) )
@@ -441,6 +442,75 @@ func TestFetchSizeMismatch(t *testing.T) {
"temp file should be cleaned up on size mismatch") "temp file should be cleaned up on size mismatch")
} }
// zeros is an io.Reader of zero bytes without end.
type zeros struct{}
func (zeros) Read(p []byte) (int, error) {
clear(p)
return len(p), nil
}
// TestFetchStopsReadingFilePastListedSize serves a body that never ends
// for a file listed at 16 bytes. fetch must stop reading one byte past
// the listed size, report the size mismatch and remove its temp file.
//
//nolint:paralleltest // changes the process-global working directory
func TestFetchStopsReadingFilePastListedSize(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.Copy(w, zeros{})
}))
defer server.Close()
chdirTemp(t)
err := downloadFile(context.Background(), testClient(),
server.URL+"/"+testFileTxt, ".", testFileTxt,
&mfer.MFFilePath{Path: testFileTxt, Size: 16}, nil)
require.ErrorIs(t, err, errSizeMismatch)
require.EqualError(t, err, "size mismatch: expected 16 bytes, got 17")
assert.NoFileExists(t, tempPathFor(testFileTxt))
}
// TestManifestDownloadStopsAtLimit serves a manifest that never ends to
// fetch and to check, with the most they download of a manifest lowered
// to 64 KiB. Each must stop reading at that limit, fail with an error
// naming it and leave no temp file.
func TestManifestDownloadStopsAtLimit(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.Copy(w, zeros{})
}))
defer server.Close()
tmpDir := t.TempDir()
t.Setenv("TMPDIR", tmpDir)
mfa := &CLIApp{Fs: afero.NewOsFs(), maxManifestSize: 64 << 10}
fetch := mfa.fetchCommand()
fetch.Action = mfa.fetchManifestOperation
check := mfa.checkCommand()
check.Action = mfa.checkManifestOperation
for _, cmd := range []*urfcli.Command{fetch, check} {
// Both operations log to the process-global logger.
err := runLocked(func() error {
return cmd.Run(context.Background(),
[]string{cmd.Name, server.URL + "/index.mf"})
})
require.ErrorIs(t, err, errManifestTooLarge, cmd.Name)
require.ErrorContains(t, err,
"maximum allowed size of 65536 bytes", cmd.Name)
}
leftover, err := os.ReadDir(tmpDir)
require.NoError(t, err)
assert.Empty(t, leftover)
}
//nolint:paralleltest // changes the process-global working directory //nolint:paralleltest // changes the process-global working directory
func TestFetchProgress(t *testing.T) { func TestFetchProgress(t *testing.T) {
// Create source filesystem with a larger test file // Create source filesystem with a larger test file
+4
View File
@@ -23,6 +23,10 @@ const manifestFetchTimeout = 30 * time.Second
// its message. // its message.
var errHTTPStatus = errors.New("HTTP") var errHTTPStatus = errors.New("HTTP")
// errManifestTooLarge indicates a manifest download that passed
// CLIApp.maxManifestSize.
var errManifestTooLarge = errors.New("manifest exceeds maximum allowed size")
// isHTTPURL returns true if the string starts with http:// or https://. // isHTTPURL returns true if the string starts with http:// or https://.
func isHTTPURL(s string) bool { func isHTTPURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
+4
View File
@@ -59,6 +59,10 @@ type CLIApp struct {
exitCode int exitCode int
app *cli.Command app *cli.Command
// maxManifestSize is the most of a manifest that fetch, and check
// given a URL, download: mfer.MaxManifestSize, which tests lower.
maxManifestSize int64
Stdin io.Reader // Standard input stream Stdin io.Reader // Standard input stream
Stdout io.Writer // Standard output stream for normal output Stdout io.Writer // Standard output stream for normal output
Stderr io.Writer // Standard error stream for diagnostics Stderr io.Writer // Standard error stream for diagnostics
+25 -12
View File
@@ -38,6 +38,7 @@ var (
errNegativeSize = errors.New("size cannot be negative") errNegativeSize = errors.New("size cannot be negative")
errHashNotMultihash = errors.New("hash is not a valid multihash") errHashNotMultihash = errors.New("hash is not a valid multihash")
errHashTooShort = errors.New("hash digest is too short") errHashTooShort = errors.New("hash digest is too short")
errDuplicatePath = errors.New("duplicate path")
) )
// ValidatePath checks that a file path conforms to manifest path invariants: // ValidatePath checks that a file path conforms to manifest path invariants:
@@ -115,6 +116,7 @@ type FileHashProgress struct {
type Builder struct { type Builder struct {
mu sync.Mutex mu sync.Mutex
files []*MFFilePath files []*MFFilePath
paths map[string]bool // the path of each entry in files
createdAt time.Time createdAt time.Time
includeTimestamps bool includeTimestamps bool
signingOptions *SigningOptions signingOptions *SigningOptions
@@ -125,6 +127,7 @@ type Builder struct {
func NewBuilder() *Builder { func NewBuilder() *Builder {
return &Builder{ return &Builder{
files: make([]*MFFilePath, 0), files: make([]*MFFilePath, 0),
paths: make(map[string]bool),
createdAt: time.Now(), createdAt: time.Now(),
} }
} }
@@ -138,6 +141,7 @@ func (b *Builder) SetSeed(seed string) {
} }
// AddFile reads file content from reader, computes hashes, and adds to manifest. // AddFile reads file content from reader, computes hashes, and adds to manifest.
// A path already added is refused once the file is read.
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none. // Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
// Progress updates are sent to the progress channel (if non-nil) without blocking. // Progress updates are sent to the progress channel (if non-nil) without blocking.
// Returns the number of bytes read. // Returns the number of bytes read.
@@ -204,11 +208,7 @@ func (b *Builder) AddFile(
Mode: uint32(mode.Perm()), Mode: uint32(mode.Perm()),
} }
b.mu.Lock() return totalRead, b.addEntry(entry)
b.files = append(b.files, entry)
b.mu.Unlock()
return totalRead, nil
} }
// sendFileHashProgress sends a progress update without blocking. // sendFileHashProgress sends a progress update without blocking.
@@ -234,8 +234,9 @@ func (b *Builder) FileCount() int {
// AddFileWithHash adds a file entry with a pre-computed hash. // AddFileWithHash adds a file entry with a pre-computed hash.
// This is useful when the hash is already known (e.g., from an existing manifest). // This is useful when the hash is already known (e.g., from an existing manifest).
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none. // Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
// Returns an error if path is invalid, size is negative, or hash is not a // Returns an error if path is invalid or already added, size is negative,
// multihash with a digest of at least 32 bytes, as long as SHA-256's. // or hash is not a multihash with a digest of at least 32 bytes, as long
// as SHA-256's.
func (b *Builder) AddFileWithHash( func (b *Builder) AddFileWithHash(
path RelFilePath, path RelFilePath,
size FileSize, size FileSize,
@@ -277,11 +278,7 @@ func (b *Builder) AddFileWithHash(
Mode: uint32(mode.Perm()), Mode: uint32(mode.Perm()),
} }
b.mu.Lock() return b.addEntry(entry)
b.files = append(b.files, entry)
b.mu.Unlock()
return nil
} }
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp. // SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
@@ -349,3 +346,19 @@ func (b *Builder) Build(ctx context.Context, w io.Writer) error {
return nil return nil
} }
// addEntry adds entry to the manifest unless an entry with its path is
// already there.
func (b *Builder) addEntry(entry *MFFilePath) error {
b.mu.Lock()
defer b.mu.Unlock()
if b.paths[entry.GetPath()] {
return fmt.Errorf("%w %q", errDuplicatePath, entry.GetPath())
}
b.paths[entry.GetPath()] = true
b.files = append(b.files, entry)
return nil
}
+26
View File
@@ -125,6 +125,32 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
} }
} }
// TestBuilderRefusesPathAlreadyAdded adds a path, then adds it again with
// AddFile and with AddFileWithHash. Each must refuse it, naming it, and
// keep the one entry already added.
func TestBuilderRefusesPathAlreadyAdded(t *testing.T) {
t.Parallel()
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
b := NewBuilder()
require.NoError(t, b.AddFileWithHash("dir/a.txt", 4, ModTime{}, 0, hash))
content := []byte("data")
_, err = b.AddFile(
"dir/a.txt", FileSize(len(content)), ModTime{}, 0, bytes.NewReader(content), nil,
)
require.ErrorIs(t, err, errDuplicatePath)
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
err = b.AddFileWithHash("dir/a.txt", 4, ModTime{}, 0, hash)
require.ErrorIs(t, err, errDuplicatePath)
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
assert.Equal(t, 1, b.FileCount())
}
func TestBuilderBuild(t *testing.T) { func TestBuilderBuild(t *testing.T) {
t.Parallel() t.Parallel()
+8 -1
View File
@@ -8,10 +8,17 @@ const (
ReleaseDate = "2025-12-17" ReleaseDate = "2025-12-17"
// MaxDecompressedSize is the maximum allowed size of decompressed manifest // MaxDecompressedSize is the maximum allowed size of decompressed manifest
// data (256 MB). This prevents decompression bombs from consuming excessive // data (256 MiB). This prevents decompression bombs from consuming excessive
// memory. // memory.
MaxDecompressedSize int64 = 256 * 1024 * 1024 MaxDecompressedSize int64 = 256 * 1024 * 1024
// MaxManifestSize is the largest manifest file mfer reads (258 MiB).
// zstd's worst case grows data it cannot compress by 1/256, so an inner
// message of MaxDecompressedSize compresses to at most 257 MiB; the
// last MiB is room for the signature, the signing key and the other
// outer fields.
MaxManifestSize = MaxDecompressedSize + MaxDecompressedSize/256 + 1<<20
// zstdWindowSize is the zstd window zstd.SpeedBestCompression gives mfer's writer. // zstdWindowSize is the zstd window zstd.SpeedBestCompression gives mfer's writer.
zstdWindowSize = 8 << 20 zstdWindowSize = 8 << 20
+29 -3
View File
@@ -24,6 +24,7 @@ var (
errCompressedHashWrong = errors.New("compressed data hash mismatch") errCompressedHashWrong = errors.New("compressed data hash mismatch")
errSignatureNoPubKey = errors.New("signature present but no public key") errSignatureNoPubKey = errors.New("signature present but no public key")
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size") errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
errManifestTooLarge = errors.New("manifest exceeds maximum allowed size")
errUUIDMismatch = errors.New("outer and inner UUID mismatch") errUUIDMismatch = errors.New("outer and inner UUID mismatch")
errInvalidFileFormat = errors.New("invalid file format") errInvalidFileFormat = errors.New("invalid file format")
errInvalidManifestPath = errors.New("manifest contains invalid path") errInvalidManifestPath = errors.New("manifest contains invalid path")
@@ -293,12 +294,21 @@ func (m *manifest) deserializeInner() error {
// extract path tomorrow — acts on a traversal or absolute path from an // extract path tomorrow — acts on a traversal or absolute path from an
// untrusted .mf. Reject loudly on the first offender rather than // untrusted .mf. Reject loudly on the first offender rather than
// dropping entries, which would let a hostile manifest hide files from a // dropping entries, which would let a hostile manifest hide files from a
// check. // check. A path listed twice is refused too: check would check the one
// file against both entries.
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
for _, f := range m.pbInner.GetFiles() { for _, f := range m.pbInner.GetFiles() {
err = ValidatePath(f.GetPath()) err = ValidatePath(f.GetPath())
if err != nil { if err != nil {
return fmt.Errorf("%w: %w", errInvalidManifestPath, err) return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
} }
if seen[f.GetPath()] {
return fmt.Errorf("%w %q", errDuplicatePath, f.GetPath())
}
seen[f.GetPath()] = true
} }
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles())) log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
@@ -318,13 +328,14 @@ func validateMagic(dat []byte) bool {
return bytes.Equal(got, expected) return bytes.Equal(got, expected)
} }
// NewManifestFromReader reads a manifest from an io.Reader. // NewManifestFromReader reads a manifest from an io.Reader. It refuses a
// manifest larger than MaxManifestSize, reading at most one byte past it.
// //
//nolint:revive // unexported-return: exporting manifest is owner question 13 //nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromReader(input io.Reader) (*manifest, error) { func NewManifestFromReader(input io.Reader) (*manifest, error) {
m := &manifest{} m := &manifest{}
dat, err := io.ReadAll(input) dat, err := readAtMost(input, MaxManifestSize)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -356,6 +367,21 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
return m, nil return m, nil
} }
// readAtMost reads all of input, or refuses it with errManifestTooLarge
// once it passes maxSize bytes, after reading one byte past maxSize.
func readAtMost(input io.Reader, maxSize int64) ([]byte, error) {
dat, err := io.ReadAll(io.LimitReader(input, maxSize+1))
if err != nil {
return nil, err
}
if int64(len(dat)) > maxSize {
return nil, fmt.Errorf("%w of %d bytes", errManifestTooLarge, maxSize)
}
return dat, nil
}
// ManifestFromFileOptions configures NewManifestFromFile. // ManifestFromFileOptions configures NewManifestFromFile.
type ManifestFromFileOptions struct { type ManifestFromFileOptions struct {
// Path is the manifest file to read (required). // Path is the manifest file to read (required).
+56 -8
View File
@@ -7,7 +7,6 @@ import (
"crypto/sha256" "crypto/sha256"
"fmt" "fmt"
"strconv" "strconv"
"strings"
"testing" "testing"
"time" "time"
"uuid" "uuid"
@@ -118,12 +117,61 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
} }
} }
// A manifest that lists a path twice is refused as it is loaded, naming the
// path. Paths are compared byte for byte: two that differ only in letter case
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
// writes do; entries of a path alone would take too much memory to decode.
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
t.Parallel()
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
require.NoError(t, err)
tests := []struct {
name string
paths []string
refused bool
}{
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
id := uuid.NewV4()
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
for _, p := range tt.paths {
inner.Files = append(inner.Files, &MFFilePath{
Path: p,
Hashes: []*MFFileChecksum{{MultiHash: hash}},
})
}
innerData, err := proto.Marshal(inner)
require.NoError(t, err)
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
if tt.refused {
require.ErrorIs(t, err, errDuplicatePath)
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
} else {
require.NoError(t, err)
assert.Len(t, m.Files(), len(tt.paths))
}
})
}
}
// Entries of a path, an empty hash, an empty MIME type and empty modification // Entries of a path, an empty hash, an empty MIME type and empty modification
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64) // and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
// and take 16 bytes plus the path to encode. A 37-character path makes that // and take 16 bytes plus the path to encode. A 37-character path makes that
// 53 bytes, about 8.2 times: refused, and leaving any one of the five // 53 bytes, about 8.2 times: refused, and leaving any one of the five
// uncounted, even the MIME type, brings it under 8. A 39-character path makes // uncounted, even the MIME type, brings it under 8. A 39-character path makes
// it 55 bytes, about 7.9 times: loaded. // it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
// padded with zeros to that length, since a manifest lists a path only once.
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) { func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
t.Parallel() t.Parallel()
@@ -139,8 +187,13 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) { t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
t.Parallel() t.Parallel()
id := uuid.NewV4()
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
inner = protowire.AppendBytes(inner, id[:])
for i := range 1000 {
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen)) entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
entry = protowire.AppendBytes(entry, nil) entry = protowire.AppendBytes(entry, nil)
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
@@ -150,11 +203,6 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
entry = protowire.AppendBytes(entry, nil) entry = protowire.AppendBytes(entry, nil)
id := uuid.NewV4()
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
inner = protowire.AppendBytes(inner, id[:])
for range 1000 {
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
inner = protowire.AppendBytes(inner, entry) inner = protowire.AppendBytes(inner, entry)
} }
+33
View File
@@ -0,0 +1,33 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestReadAtMost gives readAtMost exactly its maximum, which it must
// return whole, and twice its maximum, which it must refuse after reading
// one byte past the maximum, and no more. NewManifestFromReader reads
// through it with MaxManifestSize; the test uses 64 KiB, since reading
// MaxManifestSize under the race detector takes gigabytes of memory.
func TestReadAtMost(t *testing.T) {
t.Parallel()
const maxSize = 64 << 10
dat, err := readAtMost(bytes.NewReader(make([]byte, maxSize)), maxSize)
require.NoError(t, err)
assert.Len(t, dat, maxSize)
input := bytes.NewReader(make([]byte, 2*maxSize))
_, err = readAtMost(input, maxSize)
require.ErrorIs(t, err, errManifestTooLarge)
require.EqualError(t, err,
"manifest exceeds maximum allowed size of 65536 bytes")
assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
}
+15
View File
@@ -2,6 +2,7 @@ package mfer
import ( import (
"context" "context"
"fmt"
"io" "io"
"io/fs" "io/fs"
"os" "os"
@@ -79,6 +80,7 @@ type FileEntry struct {
type Scanner struct { type Scanner struct {
mu sync.RWMutex mu sync.RWMutex
files []*FileEntry files []*FileEntry
paths map[RelFilePath]AbsFilePath // the file at each path in files
totalBytes FileSize // cached sum of all file sizes totalBytes FileSize // cached sum of all file sizes
options *ScannerOptions options *ScannerOptions
fs afero.Fs fs afero.Fs
@@ -103,6 +105,7 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
s := &Scanner{ s := &Scanner{
files: make([]*FileEntry, 0), files: make([]*FileEntry, 0),
paths: make(map[RelFilePath]AbsFilePath),
options: opts, options: opts,
fs: fs, fs: fs,
} }
@@ -478,6 +481,18 @@ func (s *Scanner) enumerateFileWithInfo(
} }
s.mu.Lock() s.mu.Lock()
// Each path is relative to the input path it was found under, so files
// under two input paths can share one.
first, ok := s.paths[entry.Path]
if ok {
s.mu.Unlock()
return fmt.Errorf("%w %q: %s and %s",
errDuplicatePath, entry.Path, first, entry.AbsPath)
}
s.paths[entry.Path] = entry.AbsPath
s.files = append(s.files, entry) s.files = append(s.files, entry)
s.totalBytes += entry.Size s.totalBytes += entry.Size
filesFound := FileCount(len(s.files)) filesFound := FileCount(len(s.files))