Compare commits
2
Commits
e5d29aa149
...
ae0599c0cb
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae0599c0cb | ||
|
|
f663f4242d |
No files matched your search
+9
-3
@@ -50,11 +50,14 @@ allows verifying data integrity before decompression.
|
|||||||
The `innerMessage` field is compressed with
|
The `innerMessage` field is compressed with
|
||||||
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
|
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
|
||||||
enforce a decompression size limit to prevent decompression bombs. The reference
|
enforce a decompression size limit to prevent decompression bombs. The reference
|
||||||
implementation limits decompressed size to 256 MB. It writes zstd frames with a
|
implementation limits decompressed size to 256 MiB. It writes zstd frames with a
|
||||||
window of at most 8 MiB, the largest window the zstd format recommends decoders
|
window of at most 8 MiB, the largest window the zstd format recommends decoders
|
||||||
support, and refuses frames that ask for a larger one. It also refuses an inner
|
support, and refuses frames that ask for a larger one. It also refuses an inner
|
||||||
message whose file entries, hashes, timestamps and MIME types, counted at 176,
|
message whose file entries, hashes, timestamps and MIME types, counted at 176,
|
||||||
112, 64 and 16 bytes each, add up to more than 8 times its size.
|
112, 64 and 16 bytes each, add up to more than 8 times its size. It refuses a
|
||||||
|
manifest file larger than 258 MiB without reading the rest of it: zstd's worst
|
||||||
|
case grows a 256 MiB inner message by 1/256 to 257 MiB, and the last MiB is room
|
||||||
|
for the signature, the signing key and the other outer fields.
|
||||||
|
|
||||||
## Inner Message (`MFFile`)
|
## Inner Message (`MFFile`)
|
||||||
|
|
||||||
@@ -106,9 +109,12 @@ All `path` values must satisfy these invariants:
|
|||||||
- **No parent traversal**: no `..` path segments
|
- **No parent traversal**: no `..` path segments
|
||||||
- **No empty segments**: no `//` sequences
|
- **No empty segments**: no `//` sequences
|
||||||
- **No trailing slash**: paths refer to files, not directories
|
- **No trailing slash**: paths refer to files, not directories
|
||||||
|
- **Listed once**: each path appears at most once in a manifest, compared byte
|
||||||
|
for byte, so `A.txt` and `a.txt` are two paths
|
||||||
|
|
||||||
Implementations must validate these invariants when reading and writing
|
Implementations must validate these invariants when reading and writing
|
||||||
manifests. Paths that violate these rules must be rejected.
|
manifests. Paths that violate these rules must be rejected, and a reader must
|
||||||
|
reject a manifest that lists a path more than once.
|
||||||
|
|
||||||
## Hash Format (`MFFileChecksum`)
|
## Hash Format (`MFFileChecksum`)
|
||||||
|
|
||||||
|
|||||||
@@ -112,10 +112,17 @@ func (mfa *CLIApp) fetchManifestToTemp(
|
|||||||
}
|
}
|
||||||
|
|
||||||
tmpPath := tmpFile.Name()
|
tmpPath := tmpFile.Name()
|
||||||
_, cpErr := io.Copy(tmpFile, rc)
|
|
||||||
|
// Copying stops one byte past mfa.maxManifestSize, which is enough to
|
||||||
|
// tell that the manifest is too large.
|
||||||
|
written, cpErr := io.Copy(tmpFile, io.LimitReader(rc, mfa.maxManifestSize+1))
|
||||||
_ = rc.Close()
|
_ = rc.Close()
|
||||||
_ = tmpFile.Close()
|
_ = tmpFile.Close()
|
||||||
|
|
||||||
|
if cpErr == nil && written > mfa.maxManifestSize {
|
||||||
|
cpErr = fmt.Errorf("%w of %d bytes", errManifestTooLarge, mfa.maxManifestSize)
|
||||||
|
}
|
||||||
|
|
||||||
if cpErr != nil {
|
if cpErr != nil {
|
||||||
_ = mfa.Fs.Remove(tmpPath)
|
_ = mfa.Fs.Remove(tmpPath)
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
|
"sneak.berlin/go/mfer/mfer"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NoColor disables colored output when set. Automatically true if the
|
// NoColor disables colored output when set. Automatically true if the
|
||||||
@@ -60,6 +61,7 @@ func RunWithOptions(opts *RunOptions) int {
|
|||||||
version: opts.Version,
|
version: opts.Version,
|
||||||
gitrev: opts.Gitrev,
|
gitrev: opts.Gitrev,
|
||||||
exitCode: 0,
|
exitCode: 0,
|
||||||
|
maxManifestSize: mfer.MaxManifestSize,
|
||||||
Stdin: opts.Stdin,
|
Stdin: opts.Stdin,
|
||||||
Stdout: opts.Stdout,
|
Stdout: opts.Stdout,
|
||||||
Stderr: opts.Stderr,
|
Stderr: opts.Stderr,
|
||||||
|
|||||||
@@ -354,6 +354,44 @@ func TestGenerateCommand(t *testing.T) {
|
|||||||
assert.True(t, exists)
|
assert.True(t, exists)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestGenerateRefusesTwoFilesAtOnePath runs gen on arguments whose files
|
||||||
|
// would share a path in the manifest: two directories that each hold a.txt,
|
||||||
|
// and one directory given twice. gen must fail while it lists the files,
|
||||||
|
// before it hashes any, naming the path, and write no manifest.
|
||||||
|
func TestGenerateRefusesTwoFilesAtOnePath(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
first, second string
|
||||||
|
}{
|
||||||
|
{"two directories", testDir, "/other"},
|
||||||
|
{"one directory twice", testDir, testDir},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewMemMapFs()
|
||||||
|
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||||
|
require.NoError(t, fs.MkdirAll("/other", 0o755))
|
||||||
|
writeTestFile(t, fs, "/testdir/a.txt", "first")
|
||||||
|
writeTestFile(t, fs, "/other/a.txt", "second")
|
||||||
|
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdGenerate, "-q", "-o", testOutput, tc.first, tc.second,
|
||||||
|
}, fs)
|
||||||
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
|
assert.Contains(t, testStderr(t, opts),
|
||||||
|
`generate: failed to enumerate paths: duplicate path "a.txt": `+
|
||||||
|
tc.first+"/a.txt and "+tc.second+"/a.txt")
|
||||||
|
|
||||||
|
exists, err := afero.Exists(fs, testOutput)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, exists)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestGenerateSeededManifestBytes pins the exact bytes `gen --seed` writes
|
// TestGenerateSeededManifestBytes pins the exact bytes `gen --seed` writes
|
||||||
// for a fixed tree, so that a Go or dependency update that changes what
|
// for a fixed tree, so that a Go or dependency update that changes what
|
||||||
// mfer writes fails here. testdata/seeded.mf was written by an mfer built
|
// mfer writes fails here. testdata/seeded.mf was written by an mfer built
|
||||||
|
|||||||
+15
-6
@@ -361,7 +361,7 @@ func (mfa *CLIApp) fetchManifestOperation(
|
|||||||
firstDelay: firstRetryDelay,
|
firstDelay: firstRetryDelay,
|
||||||
}
|
}
|
||||||
|
|
||||||
manifestData, files, err := fetchManifest(ctx, cmd, client, manifestURL)
|
manifestData, files, err := mfa.fetchManifest(ctx, cmd, client, manifestURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -426,20 +426,22 @@ func (mfa *CLIApp) fetchManifestOperation(
|
|||||||
// that lists a file where fetch writes another or a mode outside 0777. It
|
// that lists a file where fetch writes another or a mode outside 0777. It
|
||||||
// returns the manifest as downloaded, to be saved once the files are in
|
// returns the manifest as downloaded, to be saved once the files are in
|
||||||
// place, and the files it lists.
|
// place, and the files it lists.
|
||||||
func fetchManifest(
|
func (mfa *CLIApp) fetchManifest(
|
||||||
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
||||||
) ([]byte, []*mfer.MFFilePath, error) {
|
) ([]byte, []*mfer.MFFilePath, error) {
|
||||||
log.Infof("fetching manifest from %s", manifestURL)
|
log.Infof("fetching manifest from %s", manifestURL)
|
||||||
|
|
||||||
// Read the whole manifest before parsing it, so that a connection
|
// Read the whole manifest before parsing it, so that a connection
|
||||||
// lost partway through is retried rather than reported as a bad
|
// lost partway through is retried rather than reported as a bad
|
||||||
// manifest.
|
// manifest. Reading stops one byte past mfa.maxManifestSize, which is
|
||||||
|
// enough to tell that the manifest is too large.
|
||||||
var manifestData []byte
|
var manifestData []byte
|
||||||
|
|
||||||
err := client.get(ctx, manifestURL, func(resp *http.Response) error {
|
err := client.get(ctx, manifestURL, func(resp *http.Response) error {
|
||||||
var readErr error
|
var readErr error
|
||||||
|
|
||||||
manifestData, readErr = io.ReadAll(resp.Body)
|
manifestData, readErr = io.ReadAll(
|
||||||
|
io.LimitReader(resp.Body, mfa.maxManifestSize+1))
|
||||||
|
|
||||||
return readErr
|
return readErr
|
||||||
})
|
})
|
||||||
@@ -447,6 +449,11 @@ func fetchManifest(
|
|||||||
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
|
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if int64(len(manifestData)) > mfa.maxManifestSize {
|
||||||
|
return nil, nil, fmt.Errorf("failed to fetch manifest: %w of %d bytes",
|
||||||
|
errManifestTooLarge, mfa.maxManifestSize)
|
||||||
|
}
|
||||||
|
|
||||||
// Parse manifest
|
// Parse manifest
|
||||||
//nolint:contextcheck // mfer loads a manifest without a context
|
//nolint:contextcheck // mfer loads a manifest without a context
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
@@ -908,8 +915,10 @@ func saveResponse(
|
|||||||
progress: progress,
|
progress: progress,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy content while hashing and reporting progress
|
// Copy content while hashing and reporting progress. One byte past
|
||||||
written, copyErr := io.Copy(pw, resp.Body)
|
// the listed size is enough for finishDownload to report a size
|
||||||
|
// mismatch.
|
||||||
|
written, copyErr := io.Copy(pw, io.LimitReader(resp.Body, expectedSize+1))
|
||||||
|
|
||||||
// Close file before checking errors (to flush writes)
|
// Close file before checking errors (to flush writes)
|
||||||
closeErr := out.Close()
|
closeErr := out.Close()
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
urfcli "github.com/urfave/cli/v3"
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
"sneak.berlin/go/mfer/mfer"
|
"sneak.berlin/go/mfer/mfer"
|
||||||
)
|
)
|
||||||
@@ -441,6 +442,75 @@ func TestFetchSizeMismatch(t *testing.T) {
|
|||||||
"temp file should be cleaned up on size mismatch")
|
"temp file should be cleaned up on size mismatch")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// zeros is an io.Reader of zero bytes without end.
|
||||||
|
type zeros struct{}
|
||||||
|
|
||||||
|
func (zeros) Read(p []byte) (int, error) {
|
||||||
|
clear(p)
|
||||||
|
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFetchStopsReadingFilePastListedSize serves a body that never ends
|
||||||
|
// for a file listed at 16 bytes. fetch must stop reading one byte past
|
||||||
|
// the listed size, report the size mismatch and remove its temp file.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // changes the process-global working directory
|
||||||
|
func TestFetchStopsReadingFilePastListedSize(t *testing.T) {
|
||||||
|
server := httptest.NewServer(
|
||||||
|
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = io.Copy(w, zeros{})
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
chdirTemp(t)
|
||||||
|
|
||||||
|
err := downloadFile(context.Background(), testClient(),
|
||||||
|
server.URL+"/"+testFileTxt, ".", testFileTxt,
|
||||||
|
&mfer.MFFilePath{Path: testFileTxt, Size: 16}, nil)
|
||||||
|
require.ErrorIs(t, err, errSizeMismatch)
|
||||||
|
require.EqualError(t, err, "size mismatch: expected 16 bytes, got 17")
|
||||||
|
assert.NoFileExists(t, tempPathFor(testFileTxt))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestManifestDownloadStopsAtLimit serves a manifest that never ends to
|
||||||
|
// fetch and to check, with the most they download of a manifest lowered
|
||||||
|
// to 64 KiB. Each must stop reading at that limit, fail with an error
|
||||||
|
// naming it and leave no temp file.
|
||||||
|
func TestManifestDownloadStopsAtLimit(t *testing.T) {
|
||||||
|
server := httptest.NewServer(
|
||||||
|
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = io.Copy(w, zeros{})
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
t.Setenv("TMPDIR", tmpDir)
|
||||||
|
|
||||||
|
mfa := &CLIApp{Fs: afero.NewOsFs(), maxManifestSize: 64 << 10}
|
||||||
|
|
||||||
|
fetch := mfa.fetchCommand()
|
||||||
|
fetch.Action = mfa.fetchManifestOperation
|
||||||
|
|
||||||
|
check := mfa.checkCommand()
|
||||||
|
check.Action = mfa.checkManifestOperation
|
||||||
|
|
||||||
|
for _, cmd := range []*urfcli.Command{fetch, check} {
|
||||||
|
// Both operations log to the process-global logger.
|
||||||
|
err := runLocked(func() error {
|
||||||
|
return cmd.Run(context.Background(),
|
||||||
|
[]string{cmd.Name, server.URL + "/index.mf"})
|
||||||
|
})
|
||||||
|
require.ErrorIs(t, err, errManifestTooLarge, cmd.Name)
|
||||||
|
require.ErrorContains(t, err,
|
||||||
|
"maximum allowed size of 65536 bytes", cmd.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
leftover, err := os.ReadDir(tmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, leftover)
|
||||||
|
}
|
||||||
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
//nolint:paralleltest // changes the process-global working directory
|
||||||
func TestFetchProgress(t *testing.T) {
|
func TestFetchProgress(t *testing.T) {
|
||||||
// Create source filesystem with a larger test file
|
// Create source filesystem with a larger test file
|
||||||
|
|||||||
@@ -23,6 +23,10 @@ const manifestFetchTimeout = 30 * time.Second
|
|||||||
// its message.
|
// its message.
|
||||||
var errHTTPStatus = errors.New("HTTP")
|
var errHTTPStatus = errors.New("HTTP")
|
||||||
|
|
||||||
|
// errManifestTooLarge indicates a manifest download that passed
|
||||||
|
// CLIApp.maxManifestSize.
|
||||||
|
var errManifestTooLarge = errors.New("manifest exceeds maximum allowed size")
|
||||||
|
|
||||||
// isHTTPURL returns true if the string starts with http:// or https://.
|
// isHTTPURL returns true if the string starts with http:// or https://.
|
||||||
func isHTTPURL(s string) bool {
|
func isHTTPURL(s string) bool {
|
||||||
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
|
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ type CLIApp struct {
|
|||||||
exitCode int
|
exitCode int
|
||||||
app *cli.Command
|
app *cli.Command
|
||||||
|
|
||||||
|
// maxManifestSize is the most of a manifest that fetch, and check
|
||||||
|
// given a URL, download: mfer.MaxManifestSize, which tests lower.
|
||||||
|
maxManifestSize int64
|
||||||
|
|
||||||
Stdin io.Reader // Standard input stream
|
Stdin io.Reader // Standard input stream
|
||||||
Stdout io.Writer // Standard output stream for normal output
|
Stdout io.Writer // Standard output stream for normal output
|
||||||
Stderr io.Writer // Standard error stream for diagnostics
|
Stderr io.Writer // Standard error stream for diagnostics
|
||||||
|
|||||||
+25
-12
@@ -38,6 +38,7 @@ var (
|
|||||||
errNegativeSize = errors.New("size cannot be negative")
|
errNegativeSize = errors.New("size cannot be negative")
|
||||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||||
errHashTooShort = errors.New("hash digest is too short")
|
errHashTooShort = errors.New("hash digest is too short")
|
||||||
|
errDuplicatePath = errors.New("duplicate path")
|
||||||
)
|
)
|
||||||
|
|
||||||
// ValidatePath checks that a file path conforms to manifest path invariants:
|
// ValidatePath checks that a file path conforms to manifest path invariants:
|
||||||
@@ -115,6 +116,7 @@ type FileHashProgress struct {
|
|||||||
type Builder struct {
|
type Builder struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
files []*MFFilePath
|
files []*MFFilePath
|
||||||
|
paths map[string]bool // the path of each entry in files
|
||||||
createdAt time.Time
|
createdAt time.Time
|
||||||
includeTimestamps bool
|
includeTimestamps bool
|
||||||
signingOptions *SigningOptions
|
signingOptions *SigningOptions
|
||||||
@@ -125,6 +127,7 @@ type Builder struct {
|
|||||||
func NewBuilder() *Builder {
|
func NewBuilder() *Builder {
|
||||||
return &Builder{
|
return &Builder{
|
||||||
files: make([]*MFFilePath, 0),
|
files: make([]*MFFilePath, 0),
|
||||||
|
paths: make(map[string]bool),
|
||||||
createdAt: time.Now(),
|
createdAt: time.Now(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -138,6 +141,7 @@ func (b *Builder) SetSeed(seed string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
||||||
|
// A path already added is refused once the file is read.
|
||||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||||
// Progress updates are sent to the progress channel (if non-nil) without blocking.
|
// Progress updates are sent to the progress channel (if non-nil) without blocking.
|
||||||
// Returns the number of bytes read.
|
// Returns the number of bytes read.
|
||||||
@@ -204,11 +208,7 @@ func (b *Builder) AddFile(
|
|||||||
Mode: uint32(mode.Perm()),
|
Mode: uint32(mode.Perm()),
|
||||||
}
|
}
|
||||||
|
|
||||||
b.mu.Lock()
|
return totalRead, b.addEntry(entry)
|
||||||
b.files = append(b.files, entry)
|
|
||||||
b.mu.Unlock()
|
|
||||||
|
|
||||||
return totalRead, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendFileHashProgress sends a progress update without blocking.
|
// sendFileHashProgress sends a progress update without blocking.
|
||||||
@@ -234,8 +234,9 @@ func (b *Builder) FileCount() int {
|
|||||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
// AddFileWithHash adds a file entry with a pre-computed hash.
|
||||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||||
// Returns an error if path is invalid, size is negative, or hash is not a
|
// Returns an error if path is invalid or already added, size is negative,
|
||||||
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
|
// or hash is not a multihash with a digest of at least 32 bytes, as long
|
||||||
|
// as SHA-256's.
|
||||||
func (b *Builder) AddFileWithHash(
|
func (b *Builder) AddFileWithHash(
|
||||||
path RelFilePath,
|
path RelFilePath,
|
||||||
size FileSize,
|
size FileSize,
|
||||||
@@ -277,11 +278,7 @@ func (b *Builder) AddFileWithHash(
|
|||||||
Mode: uint32(mode.Perm()),
|
Mode: uint32(mode.Perm()),
|
||||||
}
|
}
|
||||||
|
|
||||||
b.mu.Lock()
|
return b.addEntry(entry)
|
||||||
b.files = append(b.files, entry)
|
|
||||||
b.mu.Unlock()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
|
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
|
||||||
@@ -349,3 +346,19 @@ func (b *Builder) Build(ctx context.Context, w io.Writer) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// addEntry adds entry to the manifest unless an entry with its path is
|
||||||
|
// already there.
|
||||||
|
func (b *Builder) addEntry(entry *MFFilePath) error {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
|
||||||
|
if b.paths[entry.GetPath()] {
|
||||||
|
return fmt.Errorf("%w %q", errDuplicatePath, entry.GetPath())
|
||||||
|
}
|
||||||
|
|
||||||
|
b.paths[entry.GetPath()] = true
|
||||||
|
b.files = append(b.files, entry)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -125,6 +125,32 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestBuilderRefusesPathAlreadyAdded adds a path, then adds it again with
|
||||||
|
// AddFile and with AddFileWithHash. Each must refuse it, naming it, and
|
||||||
|
// keep the one entry already added.
|
||||||
|
func TestBuilderRefusesPathAlreadyAdded(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
b := NewBuilder()
|
||||||
|
require.NoError(t, b.AddFileWithHash("dir/a.txt", 4, ModTime{}, 0, hash))
|
||||||
|
|
||||||
|
content := []byte("data")
|
||||||
|
_, err = b.AddFile(
|
||||||
|
"dir/a.txt", FileSize(len(content)), ModTime{}, 0, bytes.NewReader(content), nil,
|
||||||
|
)
|
||||||
|
require.ErrorIs(t, err, errDuplicatePath)
|
||||||
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
||||||
|
|
||||||
|
err = b.AddFileWithHash("dir/a.txt", 4, ModTime{}, 0, hash)
|
||||||
|
require.ErrorIs(t, err, errDuplicatePath)
|
||||||
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
||||||
|
|
||||||
|
assert.Equal(t, 1, b.FileCount())
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuilderBuild(t *testing.T) {
|
func TestBuilderBuild(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+8
-1
@@ -8,10 +8,17 @@ const (
|
|||||||
ReleaseDate = "2025-12-17"
|
ReleaseDate = "2025-12-17"
|
||||||
|
|
||||||
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
|
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
|
||||||
// data (256 MB). This prevents decompression bombs from consuming excessive
|
// data (256 MiB). This prevents decompression bombs from consuming excessive
|
||||||
// memory.
|
// memory.
|
||||||
MaxDecompressedSize int64 = 256 * 1024 * 1024
|
MaxDecompressedSize int64 = 256 * 1024 * 1024
|
||||||
|
|
||||||
|
// MaxManifestSize is the largest manifest file mfer reads (258 MiB).
|
||||||
|
// zstd's worst case grows data it cannot compress by 1/256, so an inner
|
||||||
|
// message of MaxDecompressedSize compresses to at most 257 MiB; the
|
||||||
|
// last MiB is room for the signature, the signing key and the other
|
||||||
|
// outer fields.
|
||||||
|
MaxManifestSize = MaxDecompressedSize + MaxDecompressedSize/256 + 1<<20
|
||||||
|
|
||||||
// zstdWindowSize is the zstd window zstd.SpeedBestCompression gives mfer's writer.
|
// zstdWindowSize is the zstd window zstd.SpeedBestCompression gives mfer's writer.
|
||||||
zstdWindowSize = 8 << 20
|
zstdWindowSize = 8 << 20
|
||||||
|
|
||||||
|
|||||||
+29
-3
@@ -24,6 +24,7 @@ var (
|
|||||||
errCompressedHashWrong = errors.New("compressed data hash mismatch")
|
errCompressedHashWrong = errors.New("compressed data hash mismatch")
|
||||||
errSignatureNoPubKey = errors.New("signature present but no public key")
|
errSignatureNoPubKey = errors.New("signature present but no public key")
|
||||||
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
|
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
|
||||||
|
errManifestTooLarge = errors.New("manifest exceeds maximum allowed size")
|
||||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
||||||
errInvalidFileFormat = errors.New("invalid file format")
|
errInvalidFileFormat = errors.New("invalid file format")
|
||||||
errInvalidManifestPath = errors.New("manifest contains invalid path")
|
errInvalidManifestPath = errors.New("manifest contains invalid path")
|
||||||
@@ -293,12 +294,21 @@ func (m *manifest) deserializeInner() error {
|
|||||||
// extract path tomorrow — acts on a traversal or absolute path from an
|
// extract path tomorrow — acts on a traversal or absolute path from an
|
||||||
// untrusted .mf. Reject loudly on the first offender rather than
|
// untrusted .mf. Reject loudly on the first offender rather than
|
||||||
// dropping entries, which would let a hostile manifest hide files from a
|
// dropping entries, which would let a hostile manifest hide files from a
|
||||||
// check.
|
// check. A path listed twice is refused too: check would check the one
|
||||||
|
// file against both entries.
|
||||||
|
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
|
||||||
|
|
||||||
for _, f := range m.pbInner.GetFiles() {
|
for _, f := range m.pbInner.GetFiles() {
|
||||||
err = ValidatePath(f.GetPath())
|
err = ValidatePath(f.GetPath())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if seen[f.GetPath()] {
|
||||||
|
return fmt.Errorf("%w %q", errDuplicatePath, f.GetPath())
|
||||||
|
}
|
||||||
|
|
||||||
|
seen[f.GetPath()] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
|
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
|
||||||
@@ -318,13 +328,14 @@ func validateMagic(dat []byte) bool {
|
|||||||
return bytes.Equal(got, expected)
|
return bytes.Equal(got, expected)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManifestFromReader reads a manifest from an io.Reader.
|
// NewManifestFromReader reads a manifest from an io.Reader. It refuses a
|
||||||
|
// manifest larger than MaxManifestSize, reading at most one byte past it.
|
||||||
//
|
//
|
||||||
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
||||||
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
||||||
m := &manifest{}
|
m := &manifest{}
|
||||||
|
|
||||||
dat, err := io.ReadAll(input)
|
dat, err := readAtMost(input, MaxManifestSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -356,6 +367,21 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// readAtMost reads all of input, or refuses it with errManifestTooLarge
|
||||||
|
// once it passes maxSize bytes, after reading one byte past maxSize.
|
||||||
|
func readAtMost(input io.Reader, maxSize int64) ([]byte, error) {
|
||||||
|
dat, err := io.ReadAll(io.LimitReader(input, maxSize+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if int64(len(dat)) > maxSize {
|
||||||
|
return nil, fmt.Errorf("%w of %d bytes", errManifestTooLarge, maxSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dat, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ManifestFromFileOptions configures NewManifestFromFile.
|
// ManifestFromFileOptions configures NewManifestFromFile.
|
||||||
type ManifestFromFileOptions struct {
|
type ManifestFromFileOptions struct {
|
||||||
// Path is the manifest file to read (required).
|
// Path is the manifest file to read (required).
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
"uuid"
|
"uuid"
|
||||||
@@ -118,12 +117,61 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A manifest that lists a path twice is refused as it is loaded, naming the
|
||||||
|
// path. Paths are compared byte for byte: two that differ only in letter case
|
||||||
|
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
|
||||||
|
// writes do; entries of a path alone would take too much memory to decode.
|
||||||
|
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
paths []string
|
||||||
|
refused bool
|
||||||
|
}{
|
||||||
|
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
|
||||||
|
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
id := uuid.NewV4()
|
||||||
|
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
|
||||||
|
|
||||||
|
for _, p := range tt.paths {
|
||||||
|
inner.Files = append(inner.Files, &MFFilePath{
|
||||||
|
Path: p,
|
||||||
|
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
innerData, err := proto.Marshal(inner)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
|
||||||
|
if tt.refused {
|
||||||
|
require.ErrorIs(t, err, errDuplicatePath)
|
||||||
|
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, m.Files(), len(tt.paths))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||||
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
||||||
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
||||||
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||||
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
||||||
// it 55 bytes, about 7.9 times: loaded.
|
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
|
||||||
|
// padded with zeros to that length, since a manifest lists a path only once.
|
||||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -139,8 +187,13 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|||||||
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
id := uuid.NewV4()
|
||||||
|
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
|
||||||
|
inner = protowire.AppendBytes(inner, id[:])
|
||||||
|
|
||||||
|
for i := range 1000 {
|
||||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||||
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen))
|
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
|
||||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
||||||
@@ -150,11 +203,6 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|||||||
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
|
|
||||||
id := uuid.NewV4()
|
|
||||||
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
|
|
||||||
inner = protowire.AppendBytes(inner, id[:])
|
|
||||||
|
|
||||||
for range 1000 {
|
|
||||||
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
||||||
inner = protowire.AppendBytes(inner, entry)
|
inner = protowire.AppendBytes(inner, entry)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
//nolint:testpackage // white-box tests exercise unexported internals
|
||||||
|
package mfer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestReadAtMost gives readAtMost exactly its maximum, which it must
|
||||||
|
// return whole, and twice its maximum, which it must refuse after reading
|
||||||
|
// one byte past the maximum, and no more. NewManifestFromReader reads
|
||||||
|
// through it with MaxManifestSize; the test uses 64 KiB, since reading
|
||||||
|
// MaxManifestSize under the race detector takes gigabytes of memory.
|
||||||
|
func TestReadAtMost(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const maxSize = 64 << 10
|
||||||
|
|
||||||
|
dat, err := readAtMost(bytes.NewReader(make([]byte, maxSize)), maxSize)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, dat, maxSize)
|
||||||
|
|
||||||
|
input := bytes.NewReader(make([]byte, 2*maxSize))
|
||||||
|
|
||||||
|
_, err = readAtMost(input, maxSize)
|
||||||
|
require.ErrorIs(t, err, errManifestTooLarge)
|
||||||
|
require.EqualError(t, err,
|
||||||
|
"manifest exceeds maximum allowed size of 65536 bytes")
|
||||||
|
assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
@@ -79,6 +80,7 @@ type FileEntry struct {
|
|||||||
type Scanner struct {
|
type Scanner struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
files []*FileEntry
|
files []*FileEntry
|
||||||
|
paths map[RelFilePath]AbsFilePath // the file at each path in files
|
||||||
totalBytes FileSize // cached sum of all file sizes
|
totalBytes FileSize // cached sum of all file sizes
|
||||||
options *ScannerOptions
|
options *ScannerOptions
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
@@ -103,6 +105,7 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|||||||
|
|
||||||
s := &Scanner{
|
s := &Scanner{
|
||||||
files: make([]*FileEntry, 0),
|
files: make([]*FileEntry, 0),
|
||||||
|
paths: make(map[RelFilePath]AbsFilePath),
|
||||||
options: opts,
|
options: opts,
|
||||||
fs: fs,
|
fs: fs,
|
||||||
}
|
}
|
||||||
@@ -478,6 +481,18 @@ func (s *Scanner) enumerateFileWithInfo(
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
|
|
||||||
|
// Each path is relative to the input path it was found under, so files
|
||||||
|
// under two input paths can share one.
|
||||||
|
first, ok := s.paths[entry.Path]
|
||||||
|
if ok {
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
return fmt.Errorf("%w %q: %s and %s",
|
||||||
|
errDuplicatePath, entry.Path, first, entry.AbsPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.paths[entry.Path] = entry.AbsPath
|
||||||
s.files = append(s.files, entry)
|
s.files = append(s.files, entry)
|
||||||
s.totalBytes += entry.Size
|
s.totalBytes += entry.Size
|
||||||
filesFound := FileCount(len(s.files))
|
filesFound := FileCount(len(s.files))
|
||||||
|
|||||||
Reference in new issue
Block a user