Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2e415004a8 |
@@ -629,15 +629,7 @@ func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
return b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||||
entry.GetHashes()[0].GetMultiHash())
|
entry.GetHashes()[0].GetMultiHash())
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
|
||||||
entry.GetPath(), err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,13 +4,11 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -236,50 +234,21 @@ func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
|||||||
func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
b := mfer.NewBuilder()
|
||||||
entry := &mfer.MFFilePath{
|
entry := &mfer.MFFilePath{
|
||||||
Path: "file1.txt",
|
Path: "file1.txt",
|
||||||
Size: 8,
|
Size: 8,
|
||||||
Mtime: nil,
|
Mtime: nil,
|
||||||
Hashes: []*mfer.MFFileChecksum{
|
Hashes: []*mfer.MFFileChecksum{
|
||||||
{MultiHash: hash},
|
{MultiHash: []byte{0x12, 0x20}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err = addExistingToBuilder(b, entry)
|
err := addExistingToBuilder(b, entry)
|
||||||
require.ErrorIs(t, err, errEntryMissingMtime)
|
require.ErrorIs(t, err, errEntryMissingMtime)
|
||||||
assert.Contains(t, err.Error(), "file1.txt")
|
assert.Contains(t, err.Error(), "file1.txt")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
|
|
||||||
// entry whose hash the builder refuses is reported as a problem with the
|
|
||||||
// manifest, with the command that fixes it.
|
|
||||||
func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
|
||||||
entry := &mfer.MFFilePath{
|
|
||||||
Path: "old.txt",
|
|
||||||
Size: 8,
|
|
||||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
|
||||||
Hashes: []*mfer.MFFileChecksum{
|
|
||||||
{MultiHash: sha1Hash},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
err = addExistingToBuilder(b, entry)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "manifest entry old.txt")
|
|
||||||
assert.Contains(t, err.Error(), "mfer generate")
|
|
||||||
assert.Zero(t, b.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntryMtime pins the presence semantics the callers depend on.
|
// TestEntryMtime pins the presence semantics the callers depend on.
|
||||||
func TestEntryMtime(t *testing.T) {
|
func TestEntryMtime(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
@@ -143,44 +143,6 @@ func TestColoredLine(t *testing.T) {
|
|||||||
assert.Equal(t, want, buf.String())
|
assert.Equal(t, want, buf.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRecordLevels checks the slog level of the record each helper logs,
|
|
||||||
// through slog's text handler with the time left out. slog has no verbose
|
|
||||||
// level, so verbose messages are info records.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the package's process-wide logger and level
|
|
||||||
func TestRecordLevels(t *testing.T) {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
capture(t, DebugLevel)
|
|
||||||
|
|
||||||
opts := &slog.HandlerOptions{
|
|
||||||
Level: slog.LevelDebug,
|
|
||||||
ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr {
|
|
||||||
if a.Key == slog.TimeKey {
|
|
||||||
return slog.Attr{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return a
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
mu.Lock()
|
|
||||||
logger = slog.New(slog.NewTextHandler(&buf, opts))
|
|
||||||
mu.Unlock()
|
|
||||||
|
|
||||||
Debugf("debug")
|
|
||||||
Verbosef("verbose")
|
|
||||||
Infof("info")
|
|
||||||
Warnf("warn")
|
|
||||||
Errorf("error")
|
|
||||||
|
|
||||||
assert.Regexp(t, `^level=DEBUG msg="log_test\.go:\d+: debug"\n`+
|
|
||||||
"level=INFO msg=verbose\n"+
|
|
||||||
"level=INFO msg=info\n"+
|
|
||||||
"level=WARN msg=warn\n"+
|
|
||||||
"level=ERROR msg=error\n$", buf.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestProgress checks that progress lines go to the stdout writer whatever
|
// TestProgress checks that progress lines go to the stdout writer whatever
|
||||||
// the log level, each starting with a carriage return so it overwrites the
|
// the log level, each starting with a carriage return so it overwrites the
|
||||||
// last, and that ProgressDone erases the line.
|
// last, and that ProgressDone erases the line.
|
||||||
|
|||||||
+4
-17
@@ -35,8 +35,7 @@ var (
|
|||||||
errPathDotDot = errors.New("contains '..' segment")
|
errPathDotDot = errors.New("contains '..' segment")
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
errSizeMismatch = errors.New("size mismatch")
|
||||||
errNegativeSize = errors.New("size cannot be negative")
|
errNegativeSize = errors.New("size cannot be negative")
|
||||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
errEmptyHash = errors.New("hash cannot be nil or empty")
|
||||||
errHashTooShort = errors.New("hash digest is too short")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ValidatePath checks that a file path conforms to manifest path invariants:
|
// ValidatePath checks that a file path conforms to manifest path invariants:
|
||||||
@@ -229,8 +228,7 @@ func (b *Builder) FileCount() int {
|
|||||||
|
|
||||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
// AddFileWithHash adds a file entry with a pre-computed hash.
|
||||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||||
// Returns an error if path is invalid, size is negative, or hash is not a
|
// Returns an error if path is empty, size is negative, or hash is nil/empty.
|
||||||
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
|
|
||||||
func (b *Builder) AddFileWithHash(
|
func (b *Builder) AddFileWithHash(
|
||||||
path RelFilePath,
|
path RelFilePath,
|
||||||
size FileSize,
|
size FileSize,
|
||||||
@@ -246,19 +244,8 @@ func (b *Builder) AddFileWithHash(
|
|||||||
return errNegativeSize
|
return errNegativeSize
|
||||||
}
|
}
|
||||||
|
|
||||||
decoded, err := multihash.Decode(hash)
|
if len(hash) == 0 {
|
||||||
if err != nil {
|
return errEmptyHash
|
||||||
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
|
|
||||||
// hash is at least as long as a SHA-256 multihash, so a manifest of
|
|
||||||
// shorter ones could fail to load.
|
|
||||||
if len(decoded.Digest) < sha256.Size {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %d bytes, at least %d needed",
|
|
||||||
errHashTooShort, len(decoded.Digest), sha256.Size,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
entry := &MFFilePath{
|
entry := &MFFilePath{
|
||||||
|
|||||||
+27
-50
@@ -4,14 +4,12 @@ package mfer
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -46,10 +44,9 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
hash := make([]byte, 34) // SHA256 multihash is 34 bytes
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 1, b.FileCount())
|
assert.Equal(t, 1, b.FileCount())
|
||||||
}
|
}
|
||||||
@@ -57,14 +54,12 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
|||||||
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Run("empty path", func(t *testing.T) {
|
t.Run("empty path", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), sha256Hash)
|
hash := make([]byte, 34)
|
||||||
|
err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "path")
|
assert.Contains(t, err.Error(), "path")
|
||||||
})
|
})
|
||||||
@@ -73,58 +68,41 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), sha256Hash)
|
hash := make([]byte, 34)
|
||||||
|
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "size")
|
assert.Contains(t, err.Error(), "size")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("nil hash", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
b := NewBuilder()
|
||||||
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "hash")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty hash", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
b := NewBuilder()
|
||||||
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "hash")
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid inputs", func(t *testing.T) {
|
t.Run("valid inputs", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), sha256Hash)
|
hash := make([]byte, 34)
|
||||||
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 1, b.FileCount())
|
assert.Equal(t, 1, b.FileCount())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
hash Multihash
|
|
||||||
want error
|
|
||||||
}{
|
|
||||||
{"nil hash", nil, errHashNotMultihash},
|
|
||||||
{"empty hash", []byte{}, errHashNotMultihash},
|
|
||||||
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
|
|
||||||
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
|
||||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
|
||||||
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
|
||||||
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
|
||||||
// A valid multihash whose 31-byte digest is one byte short.
|
|
||||||
{"31-byte digest", truncatedHash, errHashTooShort},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), tt.hash)
|
|
||||||
require.ErrorIs(t, err, tt.want)
|
|
||||||
assert.Equal(t, 0, b.FileCount())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderBuild(t *testing.T) {
|
func TestBuilderBuild(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -380,8 +358,7 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
|
|||||||
func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
|
func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
|
|
||||||
|
|||||||
+1
-2
@@ -36,8 +36,7 @@ const (
|
|||||||
decodedMIMETypeSize = 16
|
decodedMIMETypeSize = 16
|
||||||
|
|
||||||
// Each file entry mfer writes holds a path of at least one byte, a
|
// Each file entry mfer writes holds a path of at least one byte, a
|
||||||
// multihash at least as long as SHA-256's 34 bytes (AddFileWithHash
|
// 34-byte SHA-256 multihash and a modification time: at least 47 bytes,
|
||||||
// refuses shorter ones) and a modification time: at least 47 bytes,
|
|
||||||
// counted at 336. So its manifests add up to at most about 7.15 times
|
// counted at 336. So its manifests add up to at most about 7.15 times
|
||||||
// their size, and this limit is about 12% above that.
|
// their size, and this limit is about 12% above that.
|
||||||
maxDecodedGrowth = 8
|
maxDecodedGrowth = 8
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ import (
|
|||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"google.golang.org/protobuf/encoding/protowire"
|
"google.golang.org/protobuf/encoding/protowire"
|
||||||
@@ -206,8 +205,7 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|||||||
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
b.SetIncludeTimestamps(true)
|
b.SetIncludeTimestamps(true)
|
||||||
@@ -229,8 +227,7 @@ func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|||||||
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
||||||
|
|||||||
Reference in New Issue
Block a user