Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
64e3e17a09 |
No files matched your search
+1
-4
@@ -109,12 +109,9 @@ All `path` values must satisfy these invariants:
|
|||||||
- **No parent traversal**: no `..` path segments
|
- **No parent traversal**: no `..` path segments
|
||||||
- **No empty segments**: no `//` sequences
|
- **No empty segments**: no `//` sequences
|
||||||
- **No trailing slash**: paths refer to files, not directories
|
- **No trailing slash**: paths refer to files, not directories
|
||||||
- **Listed once**: each path appears at most once in a manifest, compared byte
|
|
||||||
for byte, so `A.txt` and `a.txt` are two paths
|
|
||||||
|
|
||||||
Implementations must validate these invariants when reading and writing
|
Implementations must validate these invariants when reading and writing
|
||||||
manifests. Paths that violate these rules must be rejected, and a reader must
|
manifests. Paths that violate these rules must be rejected.
|
||||||
reject a manifest that lists a path more than once.
|
|
||||||
|
|
||||||
## Hash Format (`MFFileChecksum`)
|
## Hash Format (`MFFileChecksum`)
|
||||||
|
|
||||||
|
|||||||
@@ -354,44 +354,6 @@ func TestGenerateCommand(t *testing.T) {
|
|||||||
assert.True(t, exists)
|
assert.True(t, exists)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestGenerateRefusesTwoFilesAtOnePath runs gen on arguments whose files
|
|
||||||
// would share a path in the manifest: two directories that each hold a.txt,
|
|
||||||
// and one directory given twice. gen must fail while it lists the files,
|
|
||||||
// before it hashes any, naming the path, and write no manifest.
|
|
||||||
func TestGenerateRefusesTwoFilesAtOnePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
first, second string
|
|
||||||
}{
|
|
||||||
{"two directories", testDir, "/other"},
|
|
||||||
{"one directory twice", testDir, testDir},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
require.NoError(t, fs.MkdirAll("/other", 0o755))
|
|
||||||
writeTestFile(t, fs, "/testdir/a.txt", "first")
|
|
||||||
writeTestFile(t, fs, "/other/a.txt", "second")
|
|
||||||
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "-o", testOutput, tc.first, tc.second,
|
|
||||||
}, fs)
|
|
||||||
assert.Equal(t, 1, runCLI(opts))
|
|
||||||
assert.Contains(t, testStderr(t, opts),
|
|
||||||
`generate: failed to enumerate paths: duplicate path "a.txt": `+
|
|
||||||
tc.first+"/a.txt and "+tc.second+"/a.txt")
|
|
||||||
|
|
||||||
exists, err := afero.Exists(fs, testOutput)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, exists)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateSeededManifestBytes pins the exact bytes `gen --seed` writes
|
// TestGenerateSeededManifestBytes pins the exact bytes `gen --seed` writes
|
||||||
// for a fixed tree, so that a Go or dependency update that changes what
|
// for a fixed tree, so that a Go or dependency update that changes what
|
||||||
// mfer writes fails here. testdata/seeded.mf was written by an mfer built
|
// mfer writes fails here. testdata/seeded.mf was written by an mfer built
|
||||||
|
|||||||
+12
-25
@@ -38,7 +38,6 @@ var (
|
|||||||
errNegativeSize = errors.New("size cannot be negative")
|
errNegativeSize = errors.New("size cannot be negative")
|
||||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||||
errHashTooShort = errors.New("hash digest is too short")
|
errHashTooShort = errors.New("hash digest is too short")
|
||||||
errDuplicatePath = errors.New("duplicate path")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ValidatePath checks that a file path conforms to manifest path invariants:
|
// ValidatePath checks that a file path conforms to manifest path invariants:
|
||||||
@@ -116,7 +115,6 @@ type FileHashProgress struct {
|
|||||||
type Builder struct {
|
type Builder struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
files []*MFFilePath
|
files []*MFFilePath
|
||||||
paths map[string]bool // the path of each entry in files
|
|
||||||
createdAt time.Time
|
createdAt time.Time
|
||||||
includeTimestamps bool
|
includeTimestamps bool
|
||||||
signingOptions *SigningOptions
|
signingOptions *SigningOptions
|
||||||
@@ -127,7 +125,6 @@ type Builder struct {
|
|||||||
func NewBuilder() *Builder {
|
func NewBuilder() *Builder {
|
||||||
return &Builder{
|
return &Builder{
|
||||||
files: make([]*MFFilePath, 0),
|
files: make([]*MFFilePath, 0),
|
||||||
paths: make(map[string]bool),
|
|
||||||
createdAt: time.Now(),
|
createdAt: time.Now(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -141,7 +138,6 @@ func (b *Builder) SetSeed(seed string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
||||||
// A path already added is refused once the file is read.
|
|
||||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||||
// Progress updates are sent to the progress channel (if non-nil) without blocking.
|
// Progress updates are sent to the progress channel (if non-nil) without blocking.
|
||||||
// Returns the number of bytes read.
|
// Returns the number of bytes read.
|
||||||
@@ -208,7 +204,11 @@ func (b *Builder) AddFile(
|
|||||||
Mode: uint32(mode.Perm()),
|
Mode: uint32(mode.Perm()),
|
||||||
}
|
}
|
||||||
|
|
||||||
return totalRead, b.addEntry(entry)
|
b.mu.Lock()
|
||||||
|
b.files = append(b.files, entry)
|
||||||
|
b.mu.Unlock()
|
||||||
|
|
||||||
|
return totalRead, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendFileHashProgress sends a progress update without blocking.
|
// sendFileHashProgress sends a progress update without blocking.
|
||||||
@@ -234,9 +234,8 @@ func (b *Builder) FileCount() int {
|
|||||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
// AddFileWithHash adds a file entry with a pre-computed hash.
|
||||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||||
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
// Only mode's permission bits (mode.Perm()) are recorded; 0 records none.
|
||||||
// Returns an error if path is invalid or already added, size is negative,
|
// Returns an error if path is invalid, size is negative, or hash is not a
|
||||||
// or hash is not a multihash with a digest of at least 32 bytes, as long
|
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
|
||||||
// as SHA-256's.
|
|
||||||
func (b *Builder) AddFileWithHash(
|
func (b *Builder) AddFileWithHash(
|
||||||
path RelFilePath,
|
path RelFilePath,
|
||||||
size FileSize,
|
size FileSize,
|
||||||
@@ -278,7 +277,11 @@ func (b *Builder) AddFileWithHash(
|
|||||||
Mode: uint32(mode.Perm()),
|
Mode: uint32(mode.Perm()),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.addEntry(entry)
|
b.mu.Lock()
|
||||||
|
b.files = append(b.files, entry)
|
||||||
|
b.mu.Unlock()
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
|
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
|
||||||
@@ -346,19 +349,3 @@ func (b *Builder) Build(ctx context.Context, w io.Writer) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// addEntry adds entry to the manifest unless an entry with its path is
|
|
||||||
// already there.
|
|
||||||
func (b *Builder) addEntry(entry *MFFilePath) error {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
if b.paths[entry.GetPath()] {
|
|
||||||
return fmt.Errorf("%w %q", errDuplicatePath, entry.GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
b.paths[entry.GetPath()] = true
|
|
||||||
b.files = append(b.files, entry)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -125,32 +125,6 @@ func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestBuilderRefusesPathAlreadyAdded adds a path, then adds it again with
|
|
||||||
// AddFile and with AddFileWithHash. Each must refuse it, naming it, and
|
|
||||||
// keep the one entry already added.
|
|
||||||
func TestBuilderRefusesPathAlreadyAdded(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
require.NoError(t, b.AddFileWithHash("dir/a.txt", 4, ModTime{}, 0, hash))
|
|
||||||
|
|
||||||
content := []byte("data")
|
|
||||||
_, err = b.AddFile(
|
|
||||||
"dir/a.txt", FileSize(len(content)), ModTime{}, 0, bytes.NewReader(content), nil,
|
|
||||||
)
|
|
||||||
require.ErrorIs(t, err, errDuplicatePath)
|
|
||||||
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
||||||
|
|
||||||
err = b.AddFileWithHash("dir/a.txt", 4, ModTime{}, 0, hash)
|
|
||||||
require.ErrorIs(t, err, errDuplicatePath)
|
|
||||||
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
||||||
|
|
||||||
assert.Equal(t, 1, b.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderBuild(t *testing.T) {
|
func TestBuilderBuild(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+1
-10
@@ -298,21 +298,12 @@ func (m *manifest) deserializeInner() error {
|
|||||||
// extract path tomorrow — acts on a traversal or absolute path from an
|
// extract path tomorrow — acts on a traversal or absolute path from an
|
||||||
// untrusted .mf. Reject loudly on the first offender rather than
|
// untrusted .mf. Reject loudly on the first offender rather than
|
||||||
// dropping entries, which would let a hostile manifest hide files from a
|
// dropping entries, which would let a hostile manifest hide files from a
|
||||||
// check. A path listed twice is refused too: check would check the one
|
// check.
|
||||||
// file against both entries.
|
|
||||||
seen := make(map[string]bool, len(m.pbInner.GetFiles()))
|
|
||||||
|
|
||||||
for _, f := range m.pbInner.GetFiles() {
|
for _, f := range m.pbInner.GetFiles() {
|
||||||
err = ValidatePath(f.GetPath())
|
err = ValidatePath(f.GetPath())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if seen[f.GetPath()] {
|
|
||||||
return fmt.Errorf("%w %q", errDuplicatePath, f.GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
seen[f.GetPath()] = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
|
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
"uuid"
|
"uuid"
|
||||||
@@ -117,61 +118,12 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A manifest that lists a path twice is refused as it is loaded, naming the
|
|
||||||
// path. Paths are compared byte for byte: two that differ only in letter case
|
|
||||||
// load, and fetch refuses those itself. Each entry has a hash, as entries mfer
|
|
||||||
// writes do; entries of a path alone would take too much memory to decode.
|
|
||||||
func TestDeserializeRefusesPathListedTwice(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
paths []string
|
|
||||||
refused bool
|
|
||||||
}{
|
|
||||||
{"same path twice", []string{"dir/a.txt", "other.txt", "dir/a.txt"}, true},
|
|
||||||
{"paths differing in letter case", []string{"dir/b.txt", "dir/B.txt"}, false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
id := uuid.NewV4()
|
|
||||||
inner := &MFFile{Version: MFFile_VERSION_ONE, Uuid: id[:]}
|
|
||||||
|
|
||||||
for _, p := range tt.paths {
|
|
||||||
inner.Files = append(inner.Files, &MFFilePath{
|
|
||||||
Path: p,
|
|
||||||
Hashes: []*MFFileChecksum{{MultiHash: hash}},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
innerData, err := proto.Marshal(inner)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, innerData)))
|
|
||||||
if tt.refused {
|
|
||||||
require.ErrorIs(t, err, errDuplicatePath)
|
|
||||||
require.EqualError(t, err, `duplicate path "dir/a.txt"`)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, m.Files(), len(tt.paths))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||||
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
// and change times are counted at 432 bytes each (176 + 112 + 16 + 64 + 64)
|
||||||
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
// and take 16 bytes plus the path to encode. A 37-character path makes that
|
||||||
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
// 53 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||||
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
// uncounted, even the MIME type, brings it under 8. A 39-character path makes
|
||||||
// it 55 bytes, about 7.9 times: loaded. Each entry's path is its number,
|
// it 55 bytes, about 7.9 times: loaded.
|
||||||
// padded with zeros to that length, since a manifest lists a path only once.
|
|
||||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -187,24 +139,24 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|||||||
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||||
|
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen))
|
||||||
|
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||||
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
|
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
||||||
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
|
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
||||||
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
|
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
||||||
|
entry = protowire.AppendBytes(entry, nil)
|
||||||
|
|
||||||
id := uuid.NewV4()
|
id := uuid.NewV4()
|
||||||
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
|
||||||
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
|
||||||
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
||||||
inner = protowire.AppendBytes(inner, id[:])
|
inner = protowire.AppendBytes(inner, id[:])
|
||||||
|
|
||||||
for i := range 1000 {
|
for range 1000 {
|
||||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
||||||
entry = protowire.AppendString(entry, fmt.Sprintf("%0*d", tt.pathLen, i))
|
|
||||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
|
|
||||||
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
||||||
inner = protowire.AppendBytes(inner, entry)
|
inner = protowire.AppendBytes(inner, entry)
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-16
@@ -2,7 +2,6 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
@@ -80,8 +79,7 @@ type FileEntry struct {
|
|||||||
type Scanner struct {
|
type Scanner struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
files []*FileEntry
|
files []*FileEntry
|
||||||
paths map[RelFilePath]AbsFilePath // the file at each path in files
|
totalBytes FileSize // cached sum of all file sizes
|
||||||
totalBytes FileSize // cached sum of all file sizes
|
|
||||||
options *ScannerOptions
|
options *ScannerOptions
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
excluded []fs.FileInfo // the files named in ExcludePaths that exist
|
excluded []fs.FileInfo // the files named in ExcludePaths that exist
|
||||||
@@ -105,7 +103,6 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|||||||
|
|
||||||
s := &Scanner{
|
s := &Scanner{
|
||||||
files: make([]*FileEntry, 0),
|
files: make([]*FileEntry, 0),
|
||||||
paths: make(map[RelFilePath]AbsFilePath),
|
|
||||||
options: opts,
|
options: opts,
|
||||||
fs: fs,
|
fs: fs,
|
||||||
}
|
}
|
||||||
@@ -481,18 +478,6 @@ func (s *Scanner) enumerateFileWithInfo(
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
|
|
||||||
// Each path is relative to the input path it was found under, so files
|
|
||||||
// under two input paths can share one.
|
|
||||||
first, ok := s.paths[entry.Path]
|
|
||||||
if ok {
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
return fmt.Errorf("%w %q: %s and %s",
|
|
||||||
errDuplicatePath, entry.Path, first, entry.AbsPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.paths[entry.Path] = entry.AbsPath
|
|
||||||
s.files = append(s.files, entry)
|
s.files = append(s.files, entry)
|
||||||
s.totalBytes += entry.Size
|
s.totalBytes += entry.Size
|
||||||
filesFound := FileCount(len(s.files))
|
filesFound := FileCount(len(s.files))
|
||||||
|
|||||||
Reference in new issue
Block a user