Update golangci-lint to v2.12.2 with canonical config (closes #60)
check / check (push) Has been cancelled

Adopts golangci-lint v2.12.2 and the canonical .golangci.yml (default: all), and fixes all resulting findings across the tree.

Two intended behavior changes: absent MFFilePath.Mtime is handled explicitly in freshen, list and export rather than dereferenced (main panicked); gpg positional key IDs now follow an explicit -- end-of-options marker.

All twelve reworded user-visible error messages restored to byte-identical parity with main and pinned by tests.
This commit was merged in pull request #59.
This commit is contained in:
2026-08-10 16:06:12 +02:00
parent 6d19de74e7
commit de476708e9
40 changed files with 4012 additions and 1798 deletions
+78 -30
View File
@@ -1,3 +1,5 @@
// Package mfer implements the mfer manifest file format: building,
// serializing, verifying, and checking manifests of file trees.
package mfer
import (
@@ -14,6 +16,27 @@ import (
"github.com/multiformats/go-multihash"
)
// readChunkSize is the buffer size used when reading file contents for
// hashing.
const readChunkSize = 64 * 1024
// The errPath* sentinels below are worded as the trailing fragment of the
// message ValidatePath renders, because the offending path is quoted
// before them (`path %q ...`). Wrapping them mid-sentence keeps the
// rendered text exactly as mfer has always printed it. Match them with
// errors.Is rather than by reading their messages.
var (
errPathEmpty = errors.New("path cannot be empty")
errPathNotUTF8 = errors.New("is not valid UTF-8")
errPathBackslash = errors.New("contains backslash; use forward slashes only")
errPathAbsolute = errors.New("is absolute; must be relative")
errPathEmptySegment = errors.New("contains empty segment")
errPathDotDot = errors.New("contains '..' segment")
errSizeMismatch = errors.New("size mismatch")
errNegativeSize = errors.New("size cannot be negative")
errEmptyHash = errors.New("hash cannot be nil or empty")
)
// ValidatePath checks that a file path conforms to manifest path invariants:
// - Must be valid UTF-8
// - Must use forward slashes only (no backslashes)
@@ -23,25 +46,31 @@ import (
// - Must not be empty
func ValidatePath(p string) error {
if p == "" {
return errors.New("path cannot be empty")
return errPathEmpty
}
if !utf8.ValidString(p) {
return fmt.Errorf("path %q is not valid UTF-8", p)
return fmt.Errorf("path %q %w", p, errPathNotUTF8)
}
if strings.ContainsRune(p, '\\') {
return fmt.Errorf("path %q contains backslash; use forward slashes only", p)
return fmt.Errorf("path %q %w", p, errPathBackslash)
}
if strings.HasPrefix(p, "/") {
return fmt.Errorf("path %q is absolute; must be relative", p)
return fmt.Errorf("path %q %w", p, errPathAbsolute)
}
for _, seg := range strings.Split(p, "/") {
if seg == "" {
return fmt.Errorf("path %q contains empty segment", p)
return fmt.Errorf("path %q %w", p, errPathEmptySegment)
}
if seg == ".." {
return fmt.Errorf("path %q contains '..' segment", p)
return fmt.Errorf("path %q %w", p, errPathDotDot)
}
}
return nil
}
@@ -68,11 +97,7 @@ type UnixNanos int32
// Timestamp converts ModTime to a protobuf Timestamp.
func (m ModTime) Timestamp() *Timestamp {
t := time.Time(m)
return &Timestamp{
Seconds: t.Unix(),
Nanos: int32(t.Nanosecond()),
}
return newTimestampFromTime(time.Time(m))
}
// Multihash represents a multihash-encoded file hash (typically SHA2-256).
@@ -93,14 +118,6 @@ type Builder struct {
fixedUUID []byte // if set, use this UUID instead of generating one
}
// SetSeed derives a deterministic UUID from the given seed string.
// The seed is hashed once with SHA-256 and the first 16 bytes are used
// as a fixed UUID for the manifest.
func (b *Builder) SetSeed(seed string) {
hash := sha256.Sum256([]byte(seed))
b.fixedUUID = hash[:16]
}
// NewBuilder creates a new Builder.
func NewBuilder() *Builder {
return &Builder{
@@ -109,6 +126,14 @@ func NewBuilder() *Builder {
}
}
// SetSeed derives a deterministic UUID from the given seed string.
// The seed is hashed once with SHA-256 and the first 16 bytes are used
// as a fixed UUID for the manifest.
func (b *Builder) SetSeed(seed string) {
hash := sha256.Sum256([]byte(seed))
b.fixedUUID = hash[:uuidLength]
}
// AddFile reads file content from reader, computes hashes, and adds to manifest.
// Progress updates are sent to the progress channel (if non-nil) without blocking.
// Returns the number of bytes read.
@@ -119,7 +144,8 @@ func (b *Builder) AddFile(
reader io.Reader,
progress chan<- FileHashProgress,
) (FileSize, error) {
if err := ValidatePath(string(path)); err != nil {
err := ValidatePath(string(path))
if err != nil {
return 0, err
}
@@ -128,7 +154,8 @@ func (b *Builder) AddFile(
// Read file in chunks, updating hash and progress
var totalRead FileSize
buf := make([]byte, 64*1024) // 64KB chunks
buf := make([]byte, readChunkSize)
for {
n, err := reader.Read(buf)
@@ -137,9 +164,11 @@ func (b *Builder) AddFile(
totalRead += FileSize(n)
sendFileHashProgress(progress, FileHashProgress{BytesRead: totalRead})
}
if err == io.EOF {
break
}
if err != nil {
return totalRead, err
}
@@ -147,7 +176,10 @@ func (b *Builder) AddFile(
// Verify actual bytes read matches declared size
if totalRead != size {
return totalRead, fmt.Errorf("size mismatch for %q: declared %d bytes but read %d bytes", path, size, totalRead)
return totalRead, fmt.Errorf(
"%w for %q: declared %d bytes but read %d bytes",
errSizeMismatch, path, size, totalRead,
)
}
// Encode hash as multihash (SHA2-256)
@@ -178,6 +210,7 @@ func sendFileHashProgress(ch chan<- FileHashProgress, p FileHashProgress) {
if ch == nil {
return
}
select {
case ch <- p:
default:
@@ -188,21 +221,30 @@ func sendFileHashProgress(ch chan<- FileHashProgress, p FileHashProgress) {
func (b *Builder) FileCount() int {
b.mu.Lock()
defer b.mu.Unlock()
return len(b.files)
}
// AddFileWithHash adds a file entry with a pre-computed hash.
// This is useful when the hash is already known (e.g., from an existing manifest).
// Returns an error if path is empty, size is negative, or hash is nil/empty.
func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime, hash Multihash) error {
if err := ValidatePath(string(path)); err != nil {
func (b *Builder) AddFileWithHash(
path RelFilePath,
size FileSize,
mtime ModTime,
hash Multihash,
) error {
err := ValidatePath(string(path))
if err != nil {
return fmt.Errorf("add file: %w", err)
}
if size < 0 {
return errors.New("size cannot be negative")
return errNegativeSize
}
if len(hash) == 0 {
return errors.New("hash cannot be nil or empty")
return errEmptyHash
}
entry := &MFFilePath{
@@ -217,6 +259,7 @@ func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime
b.mu.Lock()
b.files = append(b.files, entry)
b.mu.Unlock()
return nil
}
@@ -225,6 +268,7 @@ func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime
func (b *Builder) SetIncludeTimestamps(include bool) {
b.mu.Lock()
defer b.mu.Unlock()
b.includeTimestamps = include
}
@@ -233,6 +277,7 @@ func (b *Builder) SetIncludeTimestamps(include bool) {
func (b *Builder) SetSigningOptions(opts *SigningOptions) {
b.mu.Lock()
defer b.mu.Unlock()
b.signingOptions = opts
}
@@ -243,7 +288,7 @@ func (b *Builder) Build(w io.Writer) error {
// Sort files by path for deterministic output
sort.Slice(b.files, func(i, j int) bool {
return b.files[i].Path < b.files[j].Path
return b.files[i].GetPath() < b.files[j].GetPath()
})
// Create inner manifest
@@ -263,19 +308,22 @@ func (b *Builder) Build(w io.Writer) error {
}
// Generate outer wrapper
if err := m.generateOuter(); err != nil {
err := m.generateOuter()
if err != nil {
return fmt.Errorf("build: generate outer: %w", err)
}
// Generate final output
if err := m.generate(); err != nil {
err = m.generate()
if err != nil {
return fmt.Errorf("build: generate: %w", err)
}
// Write to output
_, err := w.Write(m.output.Bytes())
_, err = w.Write(m.output.Bytes())
if err != nil {
return fmt.Errorf("build: write output: %w", err)
}
return nil
}
+136 -36
View File
@@ -1,3 +1,4 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
@@ -10,24 +11,34 @@ import (
"github.com/stretchr/testify/require"
)
const testFileName = "file.txt"
func TestNewBuilder(t *testing.T) {
t.Parallel()
b := NewBuilder()
assert.NotNil(t, b)
assert.Equal(t, 0, b.FileCount())
}
func TestBuilderAddFile(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := []byte("test content")
reader := bytes.NewReader(content)
bytesRead, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil)
bytesRead, err := b.AddFile(
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
)
require.NoError(t, err)
assert.Equal(t, FileSize(len(content)), bytesRead)
assert.Equal(t, 1, b.FileCount())
}
func TestBuilderAddFileWithHash(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34) // SHA256 multihash is 34 bytes
@@ -37,54 +48,71 @@ func TestBuilderAddFileWithHash(t *testing.T) {
}
func TestBuilderAddFileWithHashValidation(t *testing.T) {
t.Parallel()
t.Run("empty path", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34)
err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
assert.Error(t, err)
require.Error(t, err)
assert.Contains(t, err.Error(), "path")
})
t.Run("negative size", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34)
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
assert.Error(t, err)
require.Error(t, err)
assert.Contains(t, err.Error(), "size")
})
t.Run("nil hash", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
assert.Error(t, err)
require.Error(t, err)
assert.Contains(t, err.Error(), "hash")
})
t.Run("empty hash", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
assert.Error(t, err)
require.Error(t, err)
assert.Contains(t, err.Error(), "hash")
})
t.Run("valid inputs", func(t *testing.T) {
t.Parallel()
b := NewBuilder()
hash := make([]byte, 34)
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, 1, b.FileCount())
})
}
func TestBuilderBuild(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := []byte("test content")
reader := bytes.NewReader(content)
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil)
_, err := b.AddFile(
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
)
require.NoError(t, err)
var buf bytes.Buffer
err = b.Build(&buf)
require.NoError(t, err)
@@ -93,6 +121,8 @@ func TestBuilderBuild(t *testing.T) {
}
func TestNewTimestampFromTimeExtremeDate(t *testing.T) {
t.Parallel()
// Regression test: newTimestampFromTime used UnixNano() which panics
// for dates outside ~1678-2262. Now uses Nanosecond() which is safe.
tests := []struct {
@@ -107,15 +137,19 @@ func TestNewTimestampFromTimeExtremeDate(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
// Should not panic
ts := newTimestampFromTime(tt.time)
assert.Equal(t, tt.time.Unix(), ts.Seconds)
assert.Equal(t, int32(tt.time.Nanosecond()), ts.Nanos)
assert.Equal(t, tt.time.Unix(), ts.GetSeconds())
assert.Equal(t, tt.time.Nanosecond(), int(ts.GetNanos()))
})
}
}
func TestBuilderDeterministicOutput(t *testing.T) {
t.Parallel()
buildManifest := func() []byte {
b := NewBuilder()
// Use a fixed createdAt and UUID so output is reproducible
@@ -135,24 +169,32 @@ func TestBuilderDeterministicOutput(t *testing.T) {
}
for _, f := range files {
r := bytes.NewReader([]byte(f.content))
_, err := b.AddFile(RelFilePath(f.path), FileSize(len(f.content)), mtime, r, nil)
_, err := b.AddFile(
RelFilePath(f.path), FileSize(len(f.content)), mtime, r, nil,
)
require.NoError(t, err)
}
var buf bytes.Buffer
err := b.Build(&buf)
require.NoError(t, err)
return buf.Bytes()
}
out1 := buildManifest()
out2 := buildManifest()
assert.Equal(t, out1, out2, "two builds with same input should produce byte-identical output")
assert.Equal(t, out1, out2,
"two builds with same input should produce byte-identical output")
}
func TestSetSeedDeterministic(t *testing.T) {
t.Parallel()
b1 := NewBuilder()
b1.SetSeed("test-seed-value")
b2 := NewBuilder()
b2.SetSeed("test-seed-value")
assert.Equal(t, b1.fixedUUID, b2.fixedUUID, "same seed should produce same UUID")
@@ -160,19 +202,24 @@ func TestSetSeedDeterministic(t *testing.T) {
b3 := NewBuilder()
b3.SetSeed("different-seed")
assert.NotEqual(t, b1.fixedUUID, b3.fixedUUID, "different seeds should produce different UUIDs")
assert.NotEqual(t, b1.fixedUUID, b3.fixedUUID,
"different seeds should produce different UUIDs")
}
func TestValidatePath(t *testing.T) {
t.Parallel()
valid := []string{
"file.txt",
testFileName,
"dir/file.txt",
"a/b/c/d.txt",
"file with spaces.txt",
"日本語.txt",
"日本語.txt", //nolint:gosmopolitan // deliberately tests non-ASCII UTF-8 paths
}
for _, p := range valid {
t.Run("valid:"+p, func(t *testing.T) {
t.Parallel()
assert.NoError(t, ValidatePath(p))
})
}
@@ -191,42 +238,54 @@ func TestValidatePath(t *testing.T) {
}
for _, tt := range invalid {
t.Run("invalid:"+tt.desc, func(t *testing.T) {
t.Parallel()
assert.Error(t, ValidatePath(tt.path))
})
}
}
func TestBuilderAddFileSizeMismatch(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := []byte("short")
reader := bytes.NewReader(content)
// Declare wrong size
_, err := b.AddFile("test.txt", FileSize(100), ModTime(time.Now()), reader, nil)
assert.Error(t, err)
require.Error(t, err)
assert.Contains(t, err.Error(), "size mismatch")
}
func TestBuilderAddFileInvalidPath(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := []byte("data")
reader := bytes.NewReader(content)
_, err := b.AddFile("", FileSize(len(content)), ModTime(time.Now()), reader, nil)
assert.Error(t, err)
require.Error(t, err)
reader.Reset(content)
_, err = b.AddFile("/absolute", FileSize(len(content)), ModTime(time.Now()), reader, nil)
_, err = b.AddFile(
"/absolute", FileSize(len(content)), ModTime(time.Now()), reader, nil,
)
assert.Error(t, err)
}
func TestBuilderAddFileWithProgress(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := bytes.Repeat([]byte("x"), 1000)
reader := bytes.NewReader(content)
progress := make(chan FileHashProgress, 100)
bytesRead, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, progress)
bytesRead, err := b.AddFile(
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, progress,
)
close(progress)
require.NoError(t, err)
assert.Equal(t, FileSize(1000), bytesRead)
@@ -235,12 +294,15 @@ func TestBuilderAddFileWithProgress(t *testing.T) {
for p := range progress {
updates = append(updates, p)
}
assert.NotEmpty(t, updates)
// Last update should show all bytes
assert.Equal(t, FileSize(1000), updates[len(updates)-1].BytesRead)
}
func TestBuilderBuildRoundTrip(t *testing.T) {
t.Parallel()
// Build a manifest, deserialize it, verify all fields survive round-trip
b := NewBuilder()
now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
@@ -256,7 +318,9 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
for _, f := range files {
reader := bytes.NewReader(f.content)
_, err := b.AddFile(RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), reader, nil)
_, err := b.AddFile(
RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), reader, nil,
)
require.NoError(t, err)
}
@@ -270,42 +334,52 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
require.Len(t, mfiles, 3)
// Verify sorted order
assert.Equal(t, "alpha.txt", mfiles[0].Path)
assert.Equal(t, "beta/delta.txt", mfiles[1].Path)
assert.Equal(t, "beta/gamma.txt", mfiles[2].Path)
assert.Equal(t, "alpha.txt", mfiles[0].GetPath())
assert.Equal(t, "beta/delta.txt", mfiles[1].GetPath())
assert.Equal(t, "beta/gamma.txt", mfiles[2].GetPath())
// Verify sizes
assert.Equal(t, int64(len("alpha content")), mfiles[0].Size)
assert.Equal(t, int64(len("alpha content")), mfiles[0].GetSize())
// Verify hashes are present
for _, f := range mfiles {
require.NotEmpty(t, f.Hashes, "file %s should have hashes", f.Path)
assert.NotEmpty(t, f.Hashes[0].MultiHash)
require.NotEmpty(t, f.GetHashes(), "file %s should have hashes", f.GetPath())
assert.NotEmpty(t, f.GetHashes()[0].GetMultiHash())
}
}
func TestNewManifestFromReaderInvalidMagic(t *testing.T) {
t.Parallel()
_, err := NewManifestFromReader(bytes.NewReader([]byte("NOT_VALID")))
assert.Error(t, err)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid file format")
}
func TestNewManifestFromReaderEmpty(t *testing.T) {
t.Parallel()
_, err := NewManifestFromReader(bytes.NewReader([]byte{}))
assert.Error(t, err)
}
func TestNewManifestFromReaderTruncated(t *testing.T) {
t.Parallel()
// Just the magic with nothing after
_, err := NewManifestFromReader(bytes.NewReader([]byte(MAGIC)))
assert.Error(t, err)
}
func TestManifestString(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := []byte("test")
reader := bytes.NewReader(content)
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil)
_, err := b.AddFile(
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
)
require.NoError(t, err)
var buf bytes.Buffer
@@ -317,9 +391,12 @@ func TestManifestString(t *testing.T) {
}
func TestBuilderBuildEmpty(t *testing.T) {
t.Parallel()
b := NewBuilder()
var buf bytes.Buffer
err := b.Build(&buf)
require.NoError(t, err)
@@ -328,9 +405,14 @@ func TestBuilderBuildEmpty(t *testing.T) {
}
func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
t.Parallel()
b := NewBuilder()
content := []byte("hello")
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), bytes.NewReader(content), nil)
_, err := b.AddFile(
"test.txt", FileSize(len(content)), ModTime(time.Now()),
bytes.NewReader(content), nil,
)
require.NoError(t, err)
var buf bytes.Buffer
@@ -338,14 +420,21 @@ func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.Nil(t, m.pbInner.CreatedAt, "createdAt should be nil by default for deterministic output")
assert.Nil(t, m.pbInner.GetCreatedAt(),
"createdAt should be nil by default for deterministic output")
}
func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
t.Parallel()
b := NewBuilder()
b.SetIncludeTimestamps(true)
content := []byte("hello")
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), bytes.NewReader(content), nil)
_, err := b.AddFile(
"test.txt", FileSize(len(content)), ModTime(time.Now()),
bytes.NewReader(content), nil,
)
require.NoError(t, err)
var buf bytes.Buffer
@@ -353,23 +442,32 @@ func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.NotNil(t, m.pbInner.CreatedAt, "createdAt should be set when IncludeTimestamps is true")
assert.NotNil(t, m.pbInner.GetCreatedAt(),
"createdAt should be set when IncludeTimestamps is true")
}
func TestBuilderDeterministicFileOrder(t *testing.T) {
t.Parallel()
// Two builds with same files in different order should produce same file ordering.
// Note: UUIDs differ per build, so we compare parsed file lists, not raw bytes.
buildAndParse := func(order []string) []*MFFilePath {
b := NewBuilder()
for _, name := range order {
content := []byte("content of " + name)
_, err := b.AddFile(RelFilePath(name), FileSize(len(content)), ModTime(time.Unix(1000, 0)), bytes.NewReader(content), nil)
_, err := b.AddFile(
RelFilePath(name), FileSize(len(content)),
ModTime(time.Unix(1000, 0)), bytes.NewReader(content), nil,
)
require.NoError(t, err)
}
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(&buf)
require.NoError(t, err)
return m.Files()
}
@@ -378,10 +476,12 @@ func TestBuilderDeterministicFileOrder(t *testing.T) {
require.Len(t, files1, 2)
require.Len(t, files2, 2)
for i := range files1 {
assert.Equal(t, files1[i].Path, files2[i].Path)
assert.Equal(t, files1[i].Size, files2[i].Size)
assert.Equal(t, files1[i].GetPath(), files2[i].GetPath())
assert.Equal(t, files1[i].GetSize(), files2[i].GetSize())
}
assert.Equal(t, "a.txt", files1[0].Path)
assert.Equal(t, "b.txt", files1[1].Path)
assert.Equal(t, "a.txt", files1[0].GetPath())
assert.Equal(t, "b.txt", files1[1].GetPath())
}
+106 -78
View File
@@ -14,6 +14,8 @@ import (
"github.com/spf13/afero"
)
var errNoSigningPubKey = errors.New("manifest has no signing public key")
// Result represents the outcome of checking a single file.
type Result struct {
Path RelFilePath // Relative path from manifest
@@ -24,6 +26,7 @@ type Result struct {
// Status represents the verification status of a file.
type Status int
// Verification result statuses reported for each checked file.
const (
StatusOK Status = iota // File matches manifest (size and hash verified)
StatusMissing // File not found on disk
@@ -70,7 +73,8 @@ type Checker struct {
fs afero.Fs
// manifestPaths is a set of paths in the manifest for quick lookup
manifestPaths map[RelFilePath]struct{}
// manifestRelPath is the relative path of the manifest file from basePath (for exclusion)
// manifestRelPath is the relative path of the manifest file from
// basePath (for exclusion)
manifestRelPath RelFilePath
// signature info from the manifest
signature []byte
@@ -97,9 +101,10 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
}
files := m.Files()
manifestPaths := make(map[RelFilePath]struct{}, len(files))
for _, f := range files {
manifestPaths[RelFilePath(f.Path)] = struct{}{}
manifestPaths[RelFilePath(f.GetPath())] = struct{}{}
}
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
@@ -107,6 +112,7 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
if err != nil {
return nil, err
}
manifestRel, err := filepath.Rel(abs, absManifest)
if err != nil {
manifestRel = ""
@@ -118,9 +124,9 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
fs: fs,
manifestPaths: manifestPaths,
manifestRelPath: RelFilePath(manifestRel),
signature: m.pbOuter.Signature,
signer: m.pbOuter.Signer,
signingPubKey: m.pbOuter.SigningPubKey,
signature: m.pbOuter.GetSignature(),
signer: m.pbOuter.GetSigner(),
signingPubKey: m.pbOuter.GetSigningPubKey(),
}, nil
}
@@ -133,8 +139,9 @@ func (c *Checker) FileCount() FileCount {
func (c *Checker) TotalBytes() FileSize {
var total FileSize
for _, f := range c.files {
total += FileSize(f.Size)
total += FileSize(f.GetSize())
}
return total
}
@@ -148,7 +155,8 @@ func (c *Checker) Signer() []byte {
return c.signer
}
// SigningPubKey returns the signing public key if the manifest is signed, nil otherwise.
// SigningPubKey returns the signing public key if the manifest is signed,
// nil otherwise.
func (c *Checker) SigningPubKey() []byte {
return c.signingPubKey
}
@@ -158,8 +166,9 @@ func (c *Checker) SigningPubKey() []byte {
// returns its actual fingerprint from the key material itself.
func (c *Checker) ExtractEmbeddedSigningKeyFP() (string, error) {
if len(c.signingPubKey) == 0 {
return "", errors.New("manifest has no signing public key")
return "", errNoSigningPubKey
}
return gpgExtractPubKeyFingerprint(c.signingPubKey)
}
@@ -167,10 +176,15 @@ func (c *Checker) ExtractEmbeddedSigningKeyFP() (string, error) {
// Results are sent to the results channel as files are checked.
// Progress updates are sent to the progress channel approximately once per second.
// Both channels are closed when the method returns.
func (c *Checker) Check(ctx context.Context, results chan<- Result, progress chan<- CheckStatus) error {
func (c *Checker) Check(
ctx context.Context,
results chan<- Result,
progress chan<- CheckStatus,
) error {
if results != nil {
defer close(results)
}
if progress != nil {
defer close(progress)
}
@@ -178,9 +192,11 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
totalFiles := FileCount(len(c.files))
totalBytes := c.TotalBytes()
var checkedFiles FileCount
var checkedBytes FileSize
var failures FileCount
var (
checkedFiles FileCount
checkedBytes FileSize
failures FileCount
)
startTime := time.Now()
lastProgressTime := time.Now()
@@ -196,6 +212,7 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
if result.Status != StatusOK {
failures++
}
checkedFiles++
if results != nil {
@@ -205,19 +222,12 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
// Send progress at most once per second (rate-limited)
if progress != nil {
now := time.Now()
isLast := checkedFiles == totalFiles
if isLast || now.Sub(lastProgressTime) >= time.Second {
elapsed := time.Since(startTime)
var bytesPerSec float64
var eta time.Duration
if elapsed > 0 && checkedBytes > 0 {
bytesPerSec = float64(checkedBytes) / elapsed.Seconds()
remainingBytes := totalBytes - checkedBytes
if bytesPerSec > 0 {
eta = time.Duration(float64(remainingBytes)/bytesPerSec) * time.Second
}
}
bytesPerSec, eta := computeRateETA(
time.Since(startTime), checkedBytes, totalBytes,
)
sendCheckStatus(progress, CheckStatus{
TotalFiles: totalFiles,
@@ -228,6 +238,7 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
ETA: eta,
Failures: failures,
})
lastProgressTime = now
}
}
@@ -236,59 +247,6 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
return nil
}
func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
absPath := filepath.Join(string(c.basePath), entry.Path)
relPath := RelFilePath(entry.Path)
// Check if file exists
info, err := c.fs.Stat(absPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) || errors.Is(err, afero.ErrFileNotFound) {
return Result{Path: relPath, Status: StatusMissing, Message: "file not found"}
}
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
// Check size
if info.Size() != entry.Size {
*checkedBytes += FileSize(info.Size())
return Result{
Path: relPath,
Status: StatusSizeMismatch,
Message: "size mismatch",
}
}
// Open and hash file
f, err := c.fs.Open(absPath)
if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
defer func() { _ = f.Close() }()
h := sha256.New()
n, err := io.Copy(h, f)
if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
*checkedBytes += FileSize(n)
// Encode as multihash and compare
computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
// Check against all hashes in manifest (at least one must match)
for _, hash := range entry.Hashes {
if bytes.Equal(computed, hash.MultiHash) {
return Result{Path: relPath, Status: StatusOK}
}
}
return Result{Path: relPath, Status: StatusHashMismatch, Message: "hash mismatch"}
}
// FindExtraFiles walks the filesystem and reports files not in the manifest.
// Results are sent to the results channel. The channel is closed when done.
// Hidden files/directories (starting with .) are skipped, as they are excluded
@@ -298,7 +256,7 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
defer close(results)
}
return afero.Walk(c.fs, string(c.basePath), func(walkPath string, info os.FileInfo, err error) error {
walkFn := func(walkPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -320,6 +278,7 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
@@ -347,7 +306,75 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
}
return nil
})
}
return afero.Walk(c.fs, string(c.basePath), walkFn)
}
func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
absPath := filepath.Join(string(c.basePath), entry.GetPath())
relPath := RelFilePath(entry.GetPath())
// Check if file exists
info, err := c.fs.Stat(absPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) || errors.Is(err, afero.ErrFileNotFound) {
return Result{
Path: relPath,
Status: StatusMissing,
Message: "file not found",
}
}
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
// Check size
if info.Size() != entry.GetSize() {
*checkedBytes += FileSize(info.Size())
return Result{
Path: relPath,
Status: StatusSizeMismatch,
Message: "size mismatch",
}
}
// Open and hash file
f, err := c.fs.Open(absPath)
if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
defer func() { _ = f.Close() }()
h := sha256.New()
n, err := io.Copy(h, f)
if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
*checkedBytes += FileSize(n)
// Encode as multihash and compare
computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
// Check against all hashes in manifest (at least one must match)
for _, hash := range entry.GetHashes() {
if bytes.Equal(computed, hash.GetMultiHash()) {
return Result{Path: relPath, Status: StatusOK}
}
}
return Result{
Path: relPath,
Status: StatusHashMismatch,
Message: "hash mismatch",
}
}
// sendCheckStatus sends a status update without blocking.
@@ -355,6 +382,7 @@ func sendCheckStatus(ch chan<- CheckStatus, status CheckStatus) {
if ch == nil {
return
}
select {
case ch <- status:
default:
+143 -62
View File
@@ -1,3 +1,4 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
@@ -12,7 +13,15 @@ import (
"github.com/stretchr/testify/require"
)
const (
testFile1 = "file1.txt"
testFile2 = "file2.txt"
testExistsFile = "exists.txt"
)
func TestStatusString(t *testing.T) {
t.Parallel()
tests := []struct {
status Status
expected string
@@ -28,19 +37,26 @@ func TestStatusString(t *testing.T) {
for _, tt := range tests {
t.Run(tt.expected, func(t *testing.T) {
t.Parallel()
assert.Equal(t, tt.expected, tt.status.String())
})
}
}
// createTestManifest creates a manifest file in the filesystem with the given files.
func createTestManifest(t *testing.T, fs afero.Fs, manifestPath string, files map[string][]byte) {
func createTestManifest(
t *testing.T, fs afero.Fs, manifestPath string, files map[string][]byte,
) {
t.Helper()
builder := NewBuilder()
for path, content := range files {
reader := bytes.NewReader(content)
_, err := builder.AddFile(RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), reader, nil)
_, err := builder.AddFile(
RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), reader, nil,
)
require.NoError(t, err)
}
@@ -49,10 +65,13 @@ func createTestManifest(t *testing.T, fs afero.Fs, manifestPath string, files ma
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0o644))
}
// createFilesOnDisk creates the given files on the filesystem.
func createFilesOnDisk(t *testing.T, fs afero.Fs, basePath string, files map[string][]byte) {
// createFilesOnDisk creates the given files on the filesystem under
// /data.
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
t.Helper()
basePath := "/data"
for path, content := range files {
fullPath := basePath + "/" + path
require.NoError(t, fs.MkdirAll(basePath, 0o755))
@@ -61,11 +80,15 @@ func createFilesOnDisk(t *testing.T, fs afero.Fs, basePath string, files map[str
}
func TestNewChecker(t *testing.T) {
t.Parallel()
t.Run("valid manifest", func(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": []byte("hello"),
"file2.txt": []byte("world"),
testFile1: []byte("hello"),
testFile2: []byte("world"),
}
createTestManifest(t, fs, "/manifest.mf", files)
@@ -76,12 +99,16 @@ func TestNewChecker(t *testing.T) {
})
t.Run("missing manifest", func(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
_, err := NewChecker("/nonexistent.mf", "/", fs)
assert.Error(t, err)
})
t.Run("invalid manifest", func(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
_, err := NewChecker("/bad.mf", "/", fs)
@@ -90,6 +117,8 @@ func TestNewChecker(t *testing.T) {
}
func TestCheckerFileCountAndTotalBytes(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"small.txt": []byte("hi"),
@@ -106,13 +135,15 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
}
func TestCheckAllFilesOK(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": []byte("content one"),
"file2.txt": []byte("content two"),
testFile1: []byte("content one"),
testFile2: []byte("content two"),
}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -127,21 +158,24 @@ func TestCheckAllFilesOK(t *testing.T) {
}
assert.Len(t, resultList, 2)
for _, r := range resultList {
assert.Equal(t, StatusOK, r.Status, "file %s should be OK", r.Path)
}
}
func TestCheckMissingFile(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"exists.txt": []byte("I exist"),
"missing.txt": []byte("I don't exist on disk"),
testExistsFile: []byte("I exist"),
"missing.txt": []byte("I don't exist on disk"),
}
createTestManifest(t, fs, "/manifest.mf", files)
// Only create one file
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"exists.txt": []byte("I exist"),
createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("I exist"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
@@ -152,13 +186,17 @@ func TestCheckMissingFile(t *testing.T) {
require.NoError(t, err)
var okCount, missingCount int
for r := range results {
switch r.Status {
case StatusOK:
okCount++
case StatusMissing:
missingCount++
assert.Equal(t, RelFilePath("missing.txt"), r.Path)
case StatusSizeMismatch, StatusHashMismatch, StatusExtra, StatusError:
// Not expected in this test; counted assertions below will fail.
}
}
@@ -167,14 +205,16 @@ func TestCheckMissingFile(t *testing.T) {
}
func TestCheckSizeMismatch(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file.txt": []byte("original content"),
testFileName: []byte("original content"),
}
createTestManifest(t, fs, "/manifest.mf", files)
// Create file with different size
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"file.txt": []byte("short"),
createFilesOnDisk(t, fs, map[string][]byte{
testFileName: []byte("short"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
@@ -186,21 +226,23 @@ func TestCheckSizeMismatch(t *testing.T) {
r := <-results
assert.Equal(t, StatusSizeMismatch, r.Status)
assert.Equal(t, RelFilePath("file.txt"), r.Path)
assert.Equal(t, RelFilePath(testFileName), r.Path)
}
func TestCheckHashMismatch(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
originalContent := []byte("original content")
files := map[string][]byte{
"file.txt": originalContent,
testFileName: originalContent,
}
createTestManifest(t, fs, "/manifest.mf", files)
// Create file with same size but different content
differentContent := []byte("different contnt") // same length (16 bytes) but different
require.Equal(t, len(originalContent), len(differentContent), "test requires same length")
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"file.txt": differentContent,
require.Len(t, differentContent, len(originalContent), "test requires same length")
createFilesOnDisk(t, fs, map[string][]byte{
testFileName: differentContent,
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
@@ -212,17 +254,19 @@ func TestCheckHashMismatch(t *testing.T) {
r := <-results
assert.Equal(t, StatusHashMismatch, r.Status)
assert.Equal(t, RelFilePath("file.txt"), r.Path)
assert.Equal(t, RelFilePath(testFileName), r.Path)
}
func TestCheckWithProgress(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": bytes.Repeat([]byte("a"), 100),
"file2.txt": bytes.Repeat([]byte("b"), 200),
testFile1: bytes.Repeat([]byte("a"), 100),
testFile2: bytes.Repeat([]byte("b"), 200),
}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -233,9 +277,7 @@ func TestCheckWithProgress(t *testing.T) {
err = chk.Check(context.Background(), results, progress)
require.NoError(t, err)
// Drain results
for range results {
}
// results is fully buffered and closed; no draining needed
// Check progress was sent
var progressUpdates []CheckStatus
@@ -254,14 +296,17 @@ func TestCheckWithProgress(t *testing.T) {
}
func TestCheckContextCancellation(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
// Create many files to ensure we have time to cancel
files := make(map[string][]byte)
for i := 0; i < 100; i++ {
for i := range 100 {
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -275,17 +320,19 @@ func TestCheckContextCancellation(t *testing.T) {
}
func TestFindExtraFiles(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
// Manifest only contains file1
manifestFiles := map[string][]byte{
"file1.txt": []byte("in manifest"),
testFile1: []byte("in manifest"),
}
createTestManifest(t, fs, "/manifest.mf", manifestFiles)
// Disk has file1 and file2
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"file1.txt": []byte("in manifest"),
"file2.txt": []byte("extra file"),
createFilesOnDisk(t, fs, map[string][]byte{
testFile1: []byte("in manifest"),
testFile2: []byte("extra file"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
@@ -301,19 +348,21 @@ func TestFindExtraFiles(t *testing.T) {
}
assert.Len(t, extras, 1)
assert.Equal(t, RelFilePath("file2.txt"), extras[0].Path)
assert.Equal(t, RelFilePath(testFile2), extras[0].Path)
assert.Equal(t, StatusExtra, extras[0].Status)
assert.Equal(t, "not in manifest", extras[0].Message)
}
func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
manifestFiles := map[string][]byte{
"file1.txt": []byte("in manifest"),
testFile1: []byte("in manifest"),
}
createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"file1.txt": []byte("in manifest"),
createFilesOnDisk(t, fs, map[string][]byte{
testFile1: []byte("in manifest"),
})
// Create dotfile and manifest that should be skipped
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
@@ -338,17 +387,21 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
for _, e := range extras {
t.Logf("extra: %s", e.Path)
}
assert.Len(t, extras, 1)
if len(extras) > 0 {
assert.Equal(t, RelFilePath("extra.txt"), extras[0].Path)
}
}
func TestFindExtraFilesContextCancellation(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{"file.txt": []byte("data")}
files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -362,10 +415,12 @@ func TestFindExtraFilesContextCancellation(t *testing.T) {
}
func TestCheckNilChannels(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{"file.txt": []byte("data")}
files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -376,10 +431,12 @@ func TestCheckNilChannels(t *testing.T) {
}
func TestFindExtraFilesNilChannel(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{"file.txt": []byte("data")}
files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -390,6 +447,8 @@ func TestFindExtraFilesNilChannel(t *testing.T) {
}
func TestCheckSubdirectories(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
files := map[string][]byte{
"dir1/file1.txt": []byte("content1"),
@@ -401,6 +460,7 @@ func TestCheckSubdirectories(t *testing.T) {
// Create files with full directory structure
for path, content := range files {
fullPath := "/data/" + path
require.NoError(t, fs.MkdirAll("/data/dir1/dir2/dir3", 0o755))
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
}
@@ -413,25 +473,30 @@ func TestCheckSubdirectories(t *testing.T) {
require.NoError(t, err)
var okCount int
for r := range results {
assert.Equal(t, StatusOK, r.Status, "file %s should be OK", r.Path)
okCount++
}
assert.Equal(t, 3, okCount)
}
func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
t.Parallel()
// Regression test: errors.Is(err, errors.New("...")) never matches because
// errors.New creates a new value each time. The fix uses os.ErrNotExist instead.
fs := afero.NewMemMapFs()
files := map[string][]byte{
"exists.txt": []byte("here"),
"missing.txt": []byte("not on disk"),
testExistsFile: []byte("here"),
"missing.txt": []byte("not on disk"),
}
createTestManifest(t, fs, "/manifest.mf", files)
// Only create one file on disk
createFilesOnDisk(t, fs, "/data", map[string][]byte{
"exists.txt": []byte("here"),
createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("here"),
})
chk, err := NewChecker("/manifest.mf", "/data", fs)
@@ -448,25 +513,29 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
assert.Equal(t, RelFilePath("missing.txt"), r.Path)
}
}
assert.Equal(t, 1, statusCounts[StatusOK], "one file should be OK")
assert.Equal(t, 1, statusCounts[StatusMissing], "one file should be MISSING")
assert.Equal(t, 0, statusCounts[StatusError], "no files should be ERROR")
}
func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
t.Parallel()
// Regression test for #16: FindExtraFiles should not report dotfiles
// or the manifest file itself as extra files.
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": []byte("in manifest"),
testFile1: []byte("in manifest"),
}
createTestManifest(t, fs, "/data/.index.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
// Add dotfiles and manifest file on disk
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("dotfile"), 0o644))
require.NoError(t, fs.MkdirAll("/data/.git", 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
chk, err := NewChecker("/data/.index.mf", "/data", fs)
require.NoError(t, err)
@@ -481,17 +550,21 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
}
// Should report NO extra files — dotfiles and manifest should be skipped
assert.Empty(t, extras, "FindExtraFiles should not report dotfiles or manifest file as extra; got: %v", extras)
assert.Empty(t, extras,
"FindExtraFiles should not report dotfiles or manifest file as extra; got: %v",
extras)
}
func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
t.Parallel()
// The manifest file itself should never be reported as extra
fs := afero.NewMemMapFs()
files := map[string][]byte{
"file1.txt": []byte("content"),
testFile1: []byte("content"),
}
createTestManifest(t, fs, "/data/index.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/data/index.mf", "/data", fs)
require.NoError(t, err)
@@ -505,10 +578,13 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
extras = append(extras, r)
}
assert.Empty(t, extras, "manifest file should not be reported as extra; got: %v", extras)
assert.Empty(t, extras,
"manifest file should not be reported as extra; got: %v", extras)
}
func TestCheckEmptyManifest(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
// Create manifest with no files
createTestManifest(t, fs, "/manifest.mf", map[string][]byte{})
@@ -527,21 +603,26 @@ func TestCheckEmptyManifest(t *testing.T) {
for range results {
count++
}
assert.Equal(t, 0, count)
}
func TestCheckProgressRateLimited(t *testing.T) {
t.Parallel()
// Create many small files - progress should be rate-limited, not one per file.
// With rate-limiting to once per second, we should get far fewer progress
// updates than files (plus one final update).
fs := afero.NewMemMapFs()
files := make(map[string][]byte, 100)
for i := 0; i < 100; i++ {
for i := range 100 {
name := fmt.Sprintf("file%03d.txt", i)
files[name] = []byte("content")
}
createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, "/data", files)
createFilesOnDisk(t, fs, files)
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
@@ -551,9 +632,7 @@ func TestCheckProgressRateLimited(t *testing.T) {
err = chk.Check(context.Background(), results, progress)
require.NoError(t, err)
// Drain results
for range results {
}
// results is fully buffered and closed; no draining needed
// Count progress updates
var progressCount int
@@ -563,6 +642,8 @@ func TestCheckProgressRateLimited(t *testing.T) {
// Should be far fewer than 100 (rate-limited to once per second)
// At minimum we get the final update
assert.GreaterOrEqual(t, progressCount, 1, "should get at least the final progress update")
assert.Less(t, progressCount, 100, "progress should be rate-limited, not one per file")
assert.GreaterOrEqual(t, progressCount, 1,
"should get at least the final progress update")
assert.Less(t, progressCount, 100,
"progress should be rate-limited, not one per file")
}
+7 -1
View File
@@ -1,11 +1,17 @@
package mfer
const (
Version = "0.1.0"
// Version is the current mfer release version.
Version = "0.1.0"
// ReleaseDate is the date on which Version was released.
ReleaseDate = "2025-12-17"
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
// data (256 MB). This prevents decompression bombs from consuming excessive
// memory.
MaxDecompressedSize int64 = 256 * 1024 * 1024
// uuidLength is the length in bytes of a binary UUID.
uuidLength = 16
)
+129 -46
View File
@@ -15,105 +15,174 @@ import (
"sneak.berlin/go/mfer/internal/log"
)
var (
errInvalidUUIDLength = errors.New("invalid UUID length")
errInvalidUUIDFormat = errors.New("invalid UUID format")
errUnknownVersion = errors.New("unknown version")
errUnknownCompression = errors.New("unknown compression type")
errCompressedHashWrong = errors.New("compressed data hash mismatch")
errSignatureNoPubKey = errors.New("signature present but no public key")
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
errInvalidFileFormat = errors.New("invalid file format")
)
// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
func validateUUID(data []byte) error {
if len(data) != 16 {
return errors.New("invalid UUID length")
if len(data) != uuidLength {
return errInvalidUUIDLength
}
// Try to parse as UUID to validate format
_, err := uuid.FromBytes(data)
if err != nil {
return errors.New("invalid UUID format")
return errInvalidUUIDFormat
}
return nil
}
func (m *manifest) deserializeInner() error {
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
return errors.New("unknown version")
// validateOuterHeader checks the outer message's version, compression
// type, and UUID.
func (m *manifest) validateOuterHeader() error {
if m.pbOuter.GetVersion() != MFFileOuter_VERSION_ONE {
return errUnknownVersion
}
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_ZSTD {
return errors.New("unknown compression type")
if m.pbOuter.GetCompressionType() != MFFileOuter_COMPRESSION_ZSTD {
return errUnknownCompression
}
// Validate outer UUID before any decompression
if err := validateUUID(m.pbOuter.Uuid); err != nil {
return errors.New("outer UUID invalid: " + err.Error())
err := validateUUID(m.pbOuter.GetUuid())
if err != nil {
return fmt.Errorf("outer UUID invalid: %w", err)
}
// Verify hash of compressed data before decompression
return nil
}
// verifyOuterIntegrity checks the hash of the compressed payload and,
// if a signature is present, verifies it against the embedded public key.
func (m *manifest) verifyOuterIntegrity() error {
h := sha256.New()
if _, err := h.Write(m.pbOuter.InnerMessage); err != nil {
_, err := h.Write(m.pbOuter.GetInnerMessage())
if err != nil {
return fmt.Errorf("deserialize: hash write: %w", err)
}
sha256Hash := h.Sum(nil)
if !bytes.Equal(sha256Hash, m.pbOuter.Sha256) {
return errors.New("compressed data hash mismatch")
if !bytes.Equal(sha256Hash, m.pbOuter.GetSha256()) {
return errCompressedHashWrong
}
// Verify signature if present
if len(m.pbOuter.Signature) > 0 {
if len(m.pbOuter.SigningPubKey) == 0 {
return errors.New("signature present but no public key")
}
sigString, err := m.signatureString()
if err != nil {
return fmt.Errorf("failed to generate signature string for verification: %w", err)
}
if err := gpgVerify([]byte(sigString), m.pbOuter.Signature, m.pbOuter.SigningPubKey); err != nil {
return fmt.Errorf("signature verification failed: %w", err)
}
log.Infof("signature verified successfully")
if len(m.pbOuter.GetSignature()) == 0 {
return nil
}
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
if len(m.pbOuter.GetSigningPubKey()) == 0 {
return errSignatureNoPubKey
}
sigString, err := m.signatureString()
if err != nil {
return fmt.Errorf(
"failed to generate signature string for verification: %w", err,
)
}
err = gpgVerify(
[]byte(sigString),
m.pbOuter.GetSignature(),
m.pbOuter.GetSigningPubKey(),
)
if err != nil {
return fmt.Errorf("signature verification failed: %w", err)
}
log.Infof("signature verified successfully")
return nil
}
// decompressInner decompresses the inner payload, enforcing size limits
// to prevent decompression bombs.
func (m *manifest) decompressInner() ([]byte, error) {
bb := bytes.NewBuffer(m.pbOuter.GetInnerMessage())
zr, err := zstd.NewReader(bb)
if err != nil {
return fmt.Errorf("deserialize: zstd reader: %w", err)
return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
}
defer zr.Close()
// Limit decompressed size to prevent decompression bombs.
// Use declared size + 1 byte to detect overflow, capped at MaxDecompressedSize.
maxSize := MaxDecompressedSize
if m.pbOuter.Size > 0 && m.pbOuter.Size < int64(maxSize) {
maxSize = int64(m.pbOuter.Size) + 1
if m.pbOuter.GetSize() > 0 && m.pbOuter.GetSize() < maxSize {
maxSize = m.pbOuter.GetSize() + 1
}
limitedReader := io.LimitReader(zr, maxSize)
dat, err := io.ReadAll(limitedReader)
if err != nil {
return fmt.Errorf("deserialize: decompress: %w", err)
return nil, fmt.Errorf("deserialize: decompress: %w", err)
}
if int64(len(dat)) >= MaxDecompressedSize {
return fmt.Errorf("decompressed data exceeds maximum allowed size of %d bytes", MaxDecompressedSize)
return nil, fmt.Errorf(
"%w of %d bytes", errDecompressedTooLarge, MaxDecompressedSize,
)
}
return dat, nil
}
func (m *manifest) deserializeInner() error {
err := m.validateOuterHeader()
if err != nil {
return err
}
err = m.verifyOuterIntegrity()
if err != nil {
return err
}
dat, err := m.decompressInner()
if err != nil {
return err
}
isize := len(dat)
if int64(isize) != m.pbOuter.Size {
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
if int64(isize) != m.pbOuter.GetSize() {
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.GetSize())
return bork.ErrFileTruncated
}
// Deserialize inner message
m.pbInner = new(MFFile)
if err := proto.Unmarshal(dat, m.pbInner); err != nil {
err = proto.Unmarshal(dat, m.pbInner)
if err != nil {
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
}
// Validate inner UUID
if err := validateUUID(m.pbInner.Uuid); err != nil {
return errors.New("inner UUID invalid: " + err.Error())
err = validateUUID(m.pbInner.GetUuid())
if err != nil {
return fmt.Errorf("inner UUID invalid: %w", err)
}
// Verify UUIDs match
if !bytes.Equal(m.pbOuter.Uuid, m.pbInner.Uuid) {
return errors.New("outer and inner UUID mismatch")
if !bytes.Equal(m.pbOuter.GetUuid(), m.pbInner.GetUuid()) {
return errUUIDMismatch
}
log.Infof("loaded manifest with %d files", len(m.pbInner.Files))
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
return nil
}
@@ -122,20 +191,26 @@ func validateMagic(dat []byte) bool {
if len(dat) < ml {
return false
}
got := dat[0:ml]
expected := []byte(MAGIC)
return bytes.Equal(got, expected)
}
// NewManifestFromReader reads a manifest from an io.Reader.
//
//nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromReader(input io.Reader) (*manifest, error) {
m := &manifest{}
dat, err := io.ReadAll(input)
if err != nil {
return nil, err
}
if !validateMagic(dat) {
return nil, errors.New("invalid file format")
return nil, errInvalidFileFormat
}
// remove magic bytes prefix:
@@ -145,12 +220,15 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
// deserialize outer:
m.pbOuter = new(MFFileOuter)
if err := proto.Unmarshal(dat, m.pbOuter); err != nil {
err = proto.Unmarshal(dat, m.pbOuter)
if err != nil {
return nil, err
}
// deserialize inner:
if err := m.deserializeInner(); err != nil {
err = m.deserializeInner()
if err != nil {
return nil, err
}
@@ -159,14 +237,19 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
// If fs is nil, the real filesystem (OsFs) is used.
//
//nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
if fs == nil {
fs = afero.NewOsFs()
}
f, err := fs.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
return NewManifestFromReader(f)
}
+85
View File
@@ -0,0 +1,85 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestValidatePathMessagesVerbatim pins the exact rendered text of every
// ValidatePath rejection.
//
// These strings are user-visible and are assembled by wrapping static
// sentinels mid-sentence, which makes them easy to reword by accident
// while refactoring for errors.Is matchability. Changing one is a
// deliberate change, not a refactoring side effect.
func TestValidatePathMessagesVerbatim(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
path string
want string
is error
}{
{
name: "empty",
path: "",
want: "path cannot be empty",
is: errPathEmpty,
},
{
name: "not utf8",
path: "a\xffb",
want: `path "a\xffb" is not valid UTF-8`,
is: errPathNotUTF8,
},
{
name: "backslash",
path: `a\b`,
want: `path "a\\b" contains backslash; ` +
"use forward slashes only",
is: errPathBackslash,
},
{
name: "absolute",
path: "/a/b",
want: `path "/a/b" is absolute; must be relative`,
is: errPathAbsolute,
},
{
name: "empty segment",
path: "a//b",
want: `path "a//b" contains empty segment`,
is: errPathEmptySegment,
},
{
name: "dotdot segment",
path: "a/../b",
want: `path "a/../b" contains '..' segment`,
is: errPathDotDot,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
err := ValidatePath(tc.path)
require.Error(t, err)
assert.Equal(t, tc.want, err.Error())
require.ErrorIs(t, err, tc.is)
})
}
}
// TestSerializeInternalErrorMessagesVerbatim pins the two distinct
// "internal error" messages, which differ between generate and
// generateOuter and have always done so.
func TestSerializeInternalErrorMessagesVerbatim(t *testing.T) {
t.Parallel()
m := &manifest{}
require.EqualError(t, m.generate(), "internal error: pbInner not set")
require.EqualError(t, m.generateOuter(), "internal error")
}
+149 -85
View File
@@ -2,13 +2,45 @@ package mfer
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
)
const (
// privateDirPerms is the permission mode for temporary GPG home
// directories.
privateDirPerms os.FileMode = 0o700
// privateFilePerms is the permission mode for temporary key,
// signature, and data files.
privateFilePerms os.FileMode = 0o600
// gpgFingerprintField is the record type tag for fingerprint lines
// in gpg --with-colons output.
gpgFingerprintField = "fpr"
// gpgFingerprintMinFields is the minimum number of colon-separated
// fields in a gpg fingerprint record (the fingerprint is field 10).
gpgFingerprintMinFields = 10
// gpg option names used from more than one call site.
gpgOptArmor = "--armor"
gpgOptHomedir = "--homedir"
gpgOptVerify = "--verify"
)
var (
errGPGKeyNotFound = errors.New("gpg key not found")
errFingerprintNotFound = errors.New("fingerprint not found for key")
errImportedFPRNotFound = errors.New("fingerprint not found in imported key")
)
// GPGKeyID represents a GPG key identifier (fingerprint or key ID).
type GPGKeyID string
@@ -17,22 +49,69 @@ type SigningOptions struct {
KeyID GPGKeyID
}
// gpgSign creates a detached signature of the data using the specified key.
// Returns the armored detached signature.
func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
cmd := exec.Command("gpg", "--batch", "--no-tty",
"--detach-sign",
"--armor",
"--local-user", string(keyID),
)
// gpgArgs builds a gpg argument list from opts followed by positional
// arguments, separated by an explicit "--" end-of-options marker.
//
// This matters because key IDs reach gpg as bare positional arguments
// (from --sign-key / MFER_SIGN_KEY) and gpg would otherwise parse a value
// beginning with "-" as one of its own options. Callers must route every
// non-option argument through here.
func gpgArgs(opts []string, positional ...string) []string {
args := make([]string, 0, len(opts)+1+len(positional))
args = append(args, opts...)
args = append(args, "--")
args = append(args, positional...)
cmd.Stdin = bytes.NewReader(data)
return args
}
// runGPG runs the gpg binary in batch mode with the given arguments and
// optional stdin, returning captured stdout and stderr.
func runGPG(stdin io.Reader, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
fullArgs := append([]string{"--batch", "--no-tty"}, args...)
// G204: the executable name is a compile-time constant. The arguments
// are not, so the guarantee that matters is placement: every
// caller-supplied value is passed either as the value of a named
// option or after the "--" end-of-options marker inserted by gpgArgs,
// and therefore cannot be reinterpreted by gpg as an option.
cmd := exec.CommandContext( //nolint:gosec // G204: see comment above
context.Background(), "gpg", fullArgs...)
cmd.Stdin = stdin
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
err := cmd.Run()
return &stdout, &stderr, err
}
// parseFingerprint extracts the first fingerprint from gpg --with-colons
// output, or returns ok=false if none is present.
func parseFingerprint(colonOutput string) (string, bool) {
for _, line := range strings.Split(colonOutput, "\n") {
fields := strings.Split(line, ":")
if len(fields) >= gpgFingerprintMinFields &&
fields[0] == gpgFingerprintField {
return fields[9], true
}
}
return "", false
}
// gpgSign creates a detached signature of the data using the specified key.
// Returns the armored detached signature.
func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
stdout, stderr, err := runGPG(bytes.NewReader(data),
"--detach-sign",
gpgOptArmor,
"--local-user", string(keyID),
)
if err != nil {
return nil, fmt.Errorf("gpg sign failed: %w: %s", err, stderr.String())
}
@@ -42,22 +121,15 @@ func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
// gpgExportPublicKey exports the public key for the specified key ID.
// Returns the armored public key.
func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) {
cmd := exec.Command("gpg", "--batch", "--no-tty",
"--export",
"--armor",
string(keyID),
stdout, stderr, err := runGPG(nil,
gpgArgs([]string{"--export", gpgOptArmor}, string(keyID))...,
)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
if err != nil {
return nil, fmt.Errorf("gpg export failed: %w: %s", err, stderr.String())
}
if stdout.Len() == 0 {
return nil, fmt.Errorf("gpg key not found: %s", keyID)
return nil, fmt.Errorf("%w: %s", errGPGKeyNotFound, keyID)
}
return stdout.Bytes(), nil
@@ -65,30 +137,21 @@ func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) {
// gpgGetKeyFingerprint gets the full fingerprint for a key ID.
func gpgGetKeyFingerprint(keyID GPGKeyID) ([]byte, error) {
cmd := exec.Command("gpg", "--batch", "--no-tty",
"--with-colons",
"--fingerprint",
string(keyID),
stdout, stderr, err := runGPG(nil,
gpgArgs([]string{"--with-colons", "--fingerprint"}, string(keyID))...,
)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("gpg fingerprint lookup failed: %w: %s", err, stderr.String())
if err != nil {
return nil, fmt.Errorf(
"gpg fingerprint lookup failed: %w: %s", err, stderr.String(),
)
}
// Parse the colon-delimited output to find the fingerprint
lines := strings.Split(stdout.String(), "\n")
for _, line := range lines {
fields := strings.Split(line, ":")
if len(fields) >= 10 && fields[0] == "fpr" {
return []byte(fields[9]), nil
}
fpr, ok := parseFingerprint(stdout.String())
if !ok {
return nil, fmt.Errorf("%w: %s", errFingerprintNotFound, keyID)
}
return nil, fmt.Errorf("fingerprint not found for key: %s", keyID)
return []byte(fpr), nil
}
// gpgExtractPubKeyFingerprint imports a public key into a temporary keyring
@@ -100,54 +163,51 @@ func gpgExtractPubKeyFingerprint(pubKey []byte) (string, error) {
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %w", err)
}
defer func() { _ = os.RemoveAll(tmpDir) }()
// Set restrictive permissions
if err := os.Chmod(tmpDir, 0o700); err != nil {
err = os.Chmod(tmpDir, privateDirPerms)
if err != nil {
return "", fmt.Errorf("failed to set temp dir permissions: %w", err)
}
// Write public key to temp file
pubKeyFile := filepath.Join(tmpDir, "pubkey.asc")
if err := os.WriteFile(pubKeyFile, pubKey, 0o600); err != nil {
err = os.WriteFile(pubKeyFile, pubKey, privateFilePerms)
if err != nil {
return "", fmt.Errorf("failed to write public key: %w", err)
}
// Import the public key into the temporary keyring
importCmd := exec.Command("gpg", "--batch", "--no-tty",
"--homedir", tmpDir,
"--import",
pubKeyFile,
_, importStderr, err := runGPG(nil,
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
)
var importStderr bytes.Buffer
importCmd.Stderr = &importStderr
if err := importCmd.Run(); err != nil {
return "", fmt.Errorf("failed to import public key: %w: %s", err, importStderr.String())
if err != nil {
return "", fmt.Errorf(
"failed to import public key: %w: %s", err, importStderr.String(),
)
}
// List keys to get fingerprint
listCmd := exec.Command("gpg", "--batch", "--no-tty",
listStdout, listStderr, err := runGPG(nil,
"--homedir", tmpDir,
"--with-colons",
"--fingerprint",
)
var listStdout, listStderr bytes.Buffer
listCmd.Stdout = &listStdout
listCmd.Stderr = &listStderr
if err := listCmd.Run(); err != nil {
return "", fmt.Errorf("failed to list keys: %w: %s", err, listStderr.String())
if err != nil {
return "", fmt.Errorf(
"failed to list keys: %w: %s", err, listStderr.String(),
)
}
// Parse the colon-delimited output to find the fingerprint
lines := strings.Split(listStdout.String(), "\n")
for _, line := range lines {
fields := strings.Split(line, ":")
if len(fields) >= 10 && fields[0] == "fpr" {
return fields[9], nil
}
fpr, ok := parseFingerprint(listStdout.String())
if !ok {
return "", errImportedFPRNotFound
}
return "", fmt.Errorf("fingerprint not found in imported key")
return fpr, nil
}
// gpgVerify verifies a detached signature against data using the provided public key.
@@ -158,54 +218,58 @@ func gpgVerify(data, signature, pubKey []byte) error {
if err != nil {
return fmt.Errorf("failed to create temp dir: %w", err)
}
defer func() { _ = os.RemoveAll(tmpDir) }()
// Set restrictive permissions
if err := os.Chmod(tmpDir, 0o700); err != nil {
err = os.Chmod(tmpDir, privateDirPerms)
if err != nil {
return fmt.Errorf("failed to set temp dir permissions: %w", err)
}
// Write public key to temp file
pubKeyFile := filepath.Join(tmpDir, "pubkey.asc")
if err := os.WriteFile(pubKeyFile, pubKey, 0o600); err != nil {
err = os.WriteFile(pubKeyFile, pubKey, privateFilePerms)
if err != nil {
return fmt.Errorf("failed to write public key: %w", err)
}
// Write signature to temp file
sigFile := filepath.Join(tmpDir, "signature.asc")
if err := os.WriteFile(sigFile, signature, 0o600); err != nil {
err = os.WriteFile(sigFile, signature, privateFilePerms)
if err != nil {
return fmt.Errorf("failed to write signature: %w", err)
}
// Write data to temp file
dataFile := filepath.Join(tmpDir, "data")
if err := os.WriteFile(dataFile, data, 0o600); err != nil {
err = os.WriteFile(dataFile, data, privateFilePerms)
if err != nil {
return fmt.Errorf("failed to write data: %w", err)
}
// Import the public key into the temporary keyring
importCmd := exec.Command("gpg", "--batch", "--no-tty",
"--homedir", tmpDir,
"--import",
pubKeyFile,
_, importStderr, err := runGPG(nil,
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
)
var importStderr bytes.Buffer
importCmd.Stderr = &importStderr
if err := importCmd.Run(); err != nil {
return fmt.Errorf("failed to import public key: %w: %s", err, importStderr.String())
if err != nil {
return fmt.Errorf(
"failed to import public key: %w: %s", err, importStderr.String(),
)
}
// Verify the signature
verifyCmd := exec.Command("gpg", "--batch", "--no-tty",
"--homedir", tmpDir,
"--verify",
sigFile,
dataFile,
_, verifyStderr, err := runGPG(nil,
gpgArgs([]string{gpgOptHomedir, tmpDir, gpgOptVerify},
sigFile, dataFile)...,
)
var verifyStderr bytes.Buffer
verifyCmd.Stderr = &verifyStderr
if err := verifyCmd.Run(); err != nil {
return fmt.Errorf("signature verification failed: %w: %s", err, verifyStderr.String())
if err != nil {
return fmt.Errorf(
"signature verification failed: %w: %s", err, verifyStderr.String(),
)
}
return nil
+112 -67
View File
@@ -1,3 +1,4 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
@@ -15,35 +16,20 @@ import (
)
// testGPGEnv sets up a temporary GPG home directory with a test key.
// Returns the key ID and a cleanup function.
func testGPGEnv(t *testing.T) (GPGKeyID, func()) {
// Returns the key ID and the GPG home directory; callers must point
// GNUPGHOME at the returned directory (via t.Setenv) before using the
// gpg helpers under test.
func testGPGEnv(t *testing.T) (GPGKeyID, string) {
t.Helper()
// Check if gpg is installed
if _, err := exec.LookPath("gpg"); err != nil {
_, err := exec.LookPath("gpg")
if err != nil {
t.Skip("gpg not installed, skipping signing test")
return "", func() {}
}
// Create temporary GPG home directory
gpgHome, err := os.MkdirTemp("", "mfer-gpg-test-*")
require.NoError(t, err)
// Set restrictive permissions on GPG home
require.NoError(t, os.Chmod(gpgHome, 0o700))
// Save original GNUPGHOME and set new one
origGPGHome := os.Getenv("GNUPGHOME")
require.NoError(t, os.Setenv("GNUPGHOME", gpgHome))
cleanup := func() {
if origGPGHome == "" {
_ = os.Unsetenv("GNUPGHOME")
} else {
_ = os.Setenv("GNUPGHOME", origGPGHome)
}
_ = os.RemoveAll(gpgHome)
}
// Create temporary GPG home directory (0700 by default)
gpgHome := t.TempDir()
// Generate a test key with no passphrase
keyParams := `%no-protection
@@ -57,45 +43,51 @@ Expire-Date: 0
paramsFile := filepath.Join(gpgHome, "key-params")
require.NoError(t, os.WriteFile(paramsFile, []byte(keyParams), 0o600))
cmd := exec.Command("gpg", "--batch", "--gen-key", paramsFile)
//nolint:gosec // paramsFile is a test-controlled path inside t.TempDir()
cmd := exec.CommandContext(context.Background(), "gpg",
"--batch", "--gen-key", paramsFile)
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
output, err := cmd.CombinedOutput()
if err != nil {
cleanup()
t.Skipf("failed to generate test GPG key: %v: %s", err, output)
return "", func() {}
}
// Get the key fingerprint
cmd = exec.Command("gpg", "--list-keys", "--with-colons", "test@mfer.test")
cmd = exec.CommandContext(context.Background(), "gpg",
"--list-keys", "--with-colons", "test@mfer.test")
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
output, err = cmd.Output()
if err != nil {
cleanup()
t.Fatalf("failed to list test key: %v", err)
}
// Parse fingerprint from output
var keyID string
for _, line := range strings.Split(string(output), "\n") {
fields := strings.Split(line, ":")
if len(fields) >= 10 && fields[0] == "fpr" {
if len(fields) >= gpgFingerprintMinFields &&
fields[0] == gpgFingerprintField {
keyID = fields[9]
break
}
}
if keyID == "" {
cleanup()
t.Fatal("failed to find test key fingerprint")
}
return GPGKeyID(keyID), cleanup
return GPGKeyID(keyID), gpgHome
}
func TestGPGSign(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
data := []byte("test data to sign")
sig, err := gpgSign(data, keyID)
@@ -106,8 +98,8 @@ func TestGPGSign(t *testing.T) {
}
func TestGPGExportPublicKey(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
pubKey, err := gpgExportPublicKey(keyID)
require.NoError(t, err)
@@ -117,8 +109,8 @@ func TestGPGExportPublicKey(t *testing.T) {
}
func TestGPGGetKeyFingerprint(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
fingerprint, err := gpgGetKeyFingerprint(keyID)
require.NoError(t, err)
@@ -127,10 +119,47 @@ func TestGPGGetKeyFingerprint(t *testing.T) {
assert.Len(t, fingerprint, 40, "fingerprint should be 40 hex chars")
}
// TestGPGArgsSeparatesPositionals pins that caller-supplied values are
// placed after an end-of-options marker. Key IDs arrive from --sign-key
// and MFER_SIGN_KEY as bare positional arguments, so without the marker
// a value beginning with "-" would be parsed by gpg as one of its own
// options.
func TestGPGArgsSeparatesPositionals(t *testing.T) {
t.Parallel()
assert.Equal(t,
[]string{"--opt-a", "--opt-b", "--", "--version"},
gpgArgs([]string{"--opt-a", "--opt-b"}, "--version"))
assert.Equal(t,
[]string{"--opt-c", "--", "sig", "data"},
gpgArgs([]string{"--opt-c"}, "sig", "data"))
assert.Equal(t, []string{"--opt-d", "--"},
gpgArgs([]string{"--opt-d"}))
}
// TestGPGOptionLikeKeyIDIsNotAnOption drives real gpg with a key ID that
// looks like an option and asserts it is treated as a (nonexistent) key
// rather than executed as gpg's own --version.
func TestGPGOptionLikeKeyIDIsNotAnOption(t *testing.T) {
_, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
pubKey, err := gpgExportPublicKey(GPGKeyID("--version"))
require.Error(t, err)
require.ErrorIs(t, err, errGPGKeyNotFound)
assert.NotContains(t, string(pubKey), "gpg (GnuPG)")
fpr, err := gpgGetKeyFingerprint(GPGKeyID("--version"))
require.Error(t, err)
assert.NotContains(t, string(fpr), "gpg (GnuPG)")
}
func TestGPGSignInvalidKey(t *testing.T) {
// Set up test environment (we need GNUPGHOME set)
_, cleanup := testGPGEnv(t)
defer cleanup()
_, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
data := []byte("test data")
_, err := gpgSign(data, GPGKeyID("NONEXISTENT_KEY_ID_12345"))
@@ -138,8 +167,8 @@ func TestGPGSignInvalidKey(t *testing.T) {
}
func TestBuilderWithSigning(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
// Create a builder with signing options
b := NewBuilder()
@@ -155,6 +184,7 @@ func TestBuilderWithSigning(t *testing.T) {
// Build the manifest
var buf bytes.Buffer
err = b.Build(&buf)
require.NoError(t, err)
@@ -163,26 +193,32 @@ func TestBuilderWithSigning(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, manifest.pbOuter)
assert.NotEmpty(t, manifest.pbOuter.Signature, "signature should be populated")
assert.NotEmpty(t, manifest.pbOuter.Signer, "signer should be populated")
assert.NotEmpty(t, manifest.pbOuter.SigningPubKey, "signing public key should be populated")
assert.NotEmpty(t, manifest.pbOuter.GetSignature(),
"signature should be populated")
assert.NotEmpty(t, manifest.pbOuter.GetSigner(), "signer should be populated")
assert.NotEmpty(t, manifest.pbOuter.GetSigningPubKey(),
"signing public key should be populated")
// Verify signature is a valid PGP signature
assert.Contains(t, string(manifest.pbOuter.Signature), "-----BEGIN PGP SIGNATURE-----")
assert.Contains(t, string(manifest.pbOuter.GetSignature()),
"-----BEGIN PGP SIGNATURE-----")
// Verify public key is a valid PGP public key block
assert.Contains(t, string(manifest.pbOuter.SigningPubKey), "-----BEGIN PGP PUBLIC KEY BLOCK-----")
assert.Contains(t, string(manifest.pbOuter.GetSigningPubKey()),
"-----BEGIN PGP PUBLIC KEY BLOCK-----")
}
func TestScannerWithSigning(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
// Create in-memory filesystem with test files
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("content1"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("content2"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/file1.txt", []byte("content1"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/file2.txt", []byte("content2"), 0o644))
// Create scanner with signing options
opts := &ScannerOptions{
@@ -205,14 +241,14 @@ func TestScannerWithSigning(t *testing.T) {
manifest, err := NewManifestFromReader(&buf)
require.NoError(t, err)
assert.NotEmpty(t, manifest.pbOuter.Signature)
assert.NotEmpty(t, manifest.pbOuter.Signer)
assert.NotEmpty(t, manifest.pbOuter.SigningPubKey)
assert.NotEmpty(t, manifest.pbOuter.GetSignature())
assert.NotEmpty(t, manifest.pbOuter.GetSigner())
assert.NotEmpty(t, manifest.pbOuter.GetSigningPubKey())
}
func TestGPGVerify(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
data := []byte("test data to sign and verify")
sig, err := gpgSign(data, keyID)
@@ -227,8 +263,8 @@ func TestGPGVerify(t *testing.T) {
}
func TestGPGVerifyInvalidSignature(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
data := []byte("test data to sign")
sig, err := gpgSign(data, keyID)
@@ -244,8 +280,8 @@ func TestGPGVerifyInvalidSignature(t *testing.T) {
}
func TestGPGVerifyBadPublicKey(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
data := []byte("test data")
sig, err := gpgSign(data, keyID)
@@ -258,8 +294,8 @@ func TestGPGVerifyBadPublicKey(t *testing.T) {
}
func TestManifestSignatureVerification(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
// Create a builder with signing options
b := NewBuilder()
@@ -275,6 +311,7 @@ func TestManifestSignatureVerification(t *testing.T) {
// Build the manifest
var buf bytes.Buffer
err = b.Build(&buf)
require.NoError(t, err)
@@ -284,12 +321,12 @@ func TestManifestSignatureVerification(t *testing.T) {
require.NotNil(t, manifest)
// Signature should be present and valid
assert.NotEmpty(t, manifest.pbOuter.Signature)
assert.NotEmpty(t, manifest.pbOuter.GetSignature())
}
func TestManifestTamperedSignatureFails(t *testing.T) {
keyID, cleanup := testGPGEnv(t)
defer cleanup()
keyID, gpgHome := testGPGEnv(t)
t.Setenv("GNUPGHOME", gpgHome)
// Create a signed manifest
b := NewBuilder()
@@ -303,6 +340,7 @@ func TestManifestTamperedSignatureFails(t *testing.T) {
require.NoError(t, err)
var buf bytes.Buffer
err = b.Build(&buf)
require.NoError(t, err)
@@ -312,6 +350,7 @@ func TestManifestTamperedSignatureFails(t *testing.T) {
for i := range data {
if i > 100 && data[i] == 'A' {
data[i] = 'B'
break
}
}
@@ -322,6 +361,8 @@ func TestManifestTamperedSignatureFails(t *testing.T) {
}
func TestBuilderWithoutSigning(t *testing.T) {
t.Parallel()
// Create a builder without signing options
b := NewBuilder()
@@ -333,6 +374,7 @@ func TestBuilderWithoutSigning(t *testing.T) {
// Build the manifest
var buf bytes.Buffer
err = b.Build(&buf)
require.NoError(t, err)
@@ -341,7 +383,10 @@ func TestBuilderWithoutSigning(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, manifest.pbOuter)
assert.Empty(t, manifest.pbOuter.Signature, "signature should be empty when not signing")
assert.Empty(t, manifest.pbOuter.Signer, "signer should be empty when not signing")
assert.Empty(t, manifest.pbOuter.SigningPubKey, "signing public key should be empty when not signing")
assert.Empty(t, manifest.pbOuter.GetSignature(),
"signature should be empty when not signing")
assert.Empty(t, manifest.pbOuter.GetSigner(),
"signer should be empty when not signing")
assert.Empty(t, manifest.pbOuter.GetSigningPubKey(),
"signing public key should be empty when not signing")
}
+28 -13
View File
@@ -9,9 +9,18 @@ import (
"github.com/multiformats/go-multihash"
)
var (
errOuterNotSet = errors.New("pbOuter not set")
errUUIDNotSet = errors.New("UUID not set")
errSHA256NotSet = errors.New("SHA256 hash not set")
)
// manifest holds the internal representation of a manifest file.
// Use NewManifestFromFile or NewManifestFromReader to load an existing manifest,
// or use Builder to create a new one.
// Use NewManifestFromFile or NewManifestFromReader to load an existing
// manifest, or use Builder to create a new one.
//
// Whether this type should be exported is an open design question owned by
// the repository owner; see README design question 13.
type manifest struct {
pbInner *MFFile
pbOuter *MFFileOuter
@@ -23,8 +32,9 @@ type manifest struct {
func (m *manifest) String() string {
count := 0
if m.pbInner != nil {
count = len(m.pbInner.Files)
count = len(m.pbInner.GetFiles())
}
return fmt.Sprintf("<Manifest count=%d>", count)
}
@@ -33,7 +43,8 @@ func (m *manifest) Files() []*MFFilePath {
if m.pbInner == nil {
return nil
}
return m.pbInner.Files
return m.pbInner.GetFiles()
}
// signatureString generates the canonical string used for signing/verification.
@@ -41,20 +52,24 @@ func (m *manifest) Files() []*MFFilePath {
// Requires pbOuter to be set with Uuid and Sha256 fields.
func (m *manifest) signatureString() (string, error) {
if m.pbOuter == nil {
return "", errors.New("pbOuter not set")
}
if len(m.pbOuter.Uuid) == 0 {
return "", errors.New("UUID not set")
}
if len(m.pbOuter.Sha256) == 0 {
return "", errors.New("SHA256 hash not set")
return "", errOuterNotSet
}
mh, err := multihash.Encode(m.pbOuter.Sha256, multihash.SHA2_256)
if len(m.pbOuter.GetUuid()) == 0 {
return "", errUUIDNotSet
}
if len(m.pbOuter.GetSha256()) == 0 {
return "", errSHA256NotSet
}
mh, err := multihash.Encode(m.pbOuter.GetSha256(), multihash.SHA2_256)
if err != nil {
return "", fmt.Errorf("failed to encode multihash: %w", err)
}
uuidStr := hex.EncodeToString(m.pbOuter.Uuid)
uuidStr := hex.EncodeToString(m.pbOuter.GetUuid())
mhStr := hex.EncodeToString(mh)
return fmt.Sprintf("%s-%s-%s", MAGIC, uuidStr, mhStr), nil
}
+311 -159
View File
@@ -43,12 +43,20 @@ type ScanStatus struct {
// ScannerOptions configures scanner behavior.
type ScannerOptions struct {
IncludeDotfiles bool // Include files and directories starting with a dot (default: exclude)
FollowSymLinks bool // Resolve symlinks instead of skipping them
IncludeTimestamps bool // Include createdAt timestamp in manifest (default: omit for determinism)
Fs afero.Fs // Filesystem to use, defaults to OsFs if nil
SigningOptions *SigningOptions // GPG signing options (nil = no signing)
Seed string // If set, derive a deterministic UUID from this seed
// IncludeDotfiles includes files and directories starting with a dot
// (default: exclude).
IncludeDotfiles bool
// FollowSymLinks resolves symlinks instead of skipping them.
FollowSymLinks bool
// IncludeTimestamps includes a createdAt timestamp in the manifest
// (default: omit for determinism).
IncludeTimestamps bool
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
// SigningOptions holds GPG signing options (nil = no signing).
SigningOptions *SigningOptions
// Seed, if set, derives a deterministic UUID from this seed.
Seed string
}
// FileEntry represents a file that has been enumerated.
@@ -79,10 +87,12 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
if opts == nil {
opts = &ScannerOptions{}
}
fs := opts.Fs
if fs == nil {
fs = afero.NewOsFs()
}
return &Scanner{
files: make([]*FileEntry, 0),
options: opts,
@@ -96,47 +106,63 @@ func (s *Scanner) EnumerateFile(filePath string) error {
if err != nil {
return err
}
info, err := s.fs.Stat(abs)
if err != nil {
return err
}
// For single files, use the filename as the relative path
basePath := filepath.Dir(abs)
return s.enumerateFileWithInfo(filepath.Base(abs), basePath, info, nil)
}
// EnumeratePath walks a directory path and adds all files to the scanner.
// If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns.
func (s *Scanner) EnumeratePath(inputPath string, progress chan<- EnumerateStatus) error {
func (s *Scanner) EnumeratePath(
inputPath string,
progress chan<- EnumerateStatus,
) error {
if progress != nil {
defer close(progress)
}
abs, err := filepath.Abs(inputPath)
if err != nil {
return err
}
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
return s.enumerateFS(afs, abs, progress)
}
// EnumeratePaths walks multiple directory paths and adds all files to the scanner.
// If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns.
func (s *Scanner) EnumeratePaths(progress chan<- EnumerateStatus, inputPaths ...string) error {
func (s *Scanner) EnumeratePaths(
progress chan<- EnumerateStatus,
inputPaths ...string,
) error {
if progress != nil {
defer close(progress)
}
for _, p := range inputPaths {
abs, err := filepath.Abs(p)
if err != nil {
return err
}
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
if err := s.enumerateFS(afs, abs, progress); err != nil {
err = s.enumerateFS(afs, abs, progress)
if err != nil {
return err
}
}
return nil
}
@@ -144,31 +170,231 @@ func (s *Scanner) EnumeratePaths(progress chan<- EnumerateStatus, inputPaths ...
// If progress is non-nil, status updates are sent as files are discovered.
// The progress channel is closed when the method returns.
// basePath is used to compute absolute paths for file reading.
func (s *Scanner) EnumerateFS(afs afero.Fs, basePath string, progress chan<- EnumerateStatus) error {
func (s *Scanner) EnumerateFS(
afs afero.Fs,
basePath string,
progress chan<- EnumerateStatus,
) error {
if progress != nil {
defer close(progress)
}
return s.enumerateFS(afs, basePath, progress)
}
// enumerateFS is the internal implementation that doesn't close the progress channel.
func (s *Scanner) enumerateFS(afs afero.Fs, basePath string, progress chan<- EnumerateStatus) error {
// Files returns a copy of all files added to the scanner.
func (s *Scanner) Files() []*FileEntry {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]*FileEntry, len(s.files))
copy(out, s.files)
return out
}
// FileCount returns the number of files in the scanner.
func (s *Scanner) FileCount() FileCount {
s.mu.RLock()
defer s.mu.RUnlock()
return FileCount(len(s.files))
}
// TotalBytes returns the total size of all files in the scanner.
func (s *Scanner) TotalBytes() FileSize {
s.mu.RLock()
defer s.mu.RUnlock()
return s.totalBytes
}
// ToManifest reads all file contents, computes hashes, and generates a manifest.
// If progress is non-nil, status updates are sent approximately once per second.
// The progress channel is closed when the method returns.
// The manifest is written to the provided io.Writer.
func (s *Scanner) ToManifest(
ctx context.Context, w io.Writer, progress chan<- ScanStatus,
) error {
if progress != nil {
defer close(progress)
}
s.mu.RLock()
files := make([]*FileEntry, len(s.files))
copy(files, s.files)
totalFiles := FileCount(len(files))
var totalBytes FileSize
for _, f := range files {
totalBytes += f.Size
}
s.mu.RUnlock()
builder := s.configureBuilder()
var (
scannedFiles FileCount
scannedBytes FileSize
)
lastProgressTime := time.Now()
startTime := time.Now()
pt := &scanProgressTracker{
progress: progress,
totalFiles: totalFiles,
totalBytes: totalBytes,
startTime: startTime,
lastProgress: &lastProgressTime,
}
for _, entry := range files {
// Check for cancellation
select {
case <-ctx.Done():
return ctx.Err()
default:
}
bytesRead, err := s.scanFile(builder, pt, entry, scannedFiles, scannedBytes)
if err != nil {
return err
}
scannedFiles++
scannedBytes += bytesRead
}
// Send final progress (ETA is 0 at completion; remaining bytes are 0,
// so computeRateETA yields eta 0 and the same average rate as before)
if progress != nil {
rate, _ := computeRateETA(time.Since(startTime), scannedBytes, totalBytes)
sendScanStatus(progress, ScanStatus{
TotalFiles: totalFiles,
ScannedFiles: scannedFiles,
TotalBytes: totalBytes,
ScannedBytes: scannedBytes,
BytesPerSec: rate,
ETA: 0,
})
}
// Build and write manifest
//nolint:contextcheck // Build's GPG signing exec is not cancellable by design
return builder.Build(w)
}
// configureBuilder constructs a manifest builder configured from the
// scanner options.
func (s *Scanner) configureBuilder() *Builder {
builder := NewBuilder()
if s.options.IncludeTimestamps {
builder.SetIncludeTimestamps(true)
}
if s.options.SigningOptions != nil {
builder.SetSigningOptions(s.options.SigningOptions)
}
if s.options.Seed != "" {
builder.SetSeed(s.options.Seed)
}
return builder
}
// scanFile hashes a single file into the builder, forwarding per-file
// progress updates, and returns the number of bytes read.
func (s *Scanner) scanFile(
builder *Builder,
pt *scanProgressTracker,
entry *FileEntry,
scannedFiles FileCount,
scannedBytes FileSize,
) (FileSize, error) {
// Open file
f, err := s.fs.Open(string(entry.AbsPath))
if err != nil {
return 0, err
}
// Create progress channel for this file
var (
fileProgress chan FileHashProgress
wg sync.WaitGroup
)
if pt.progress != nil {
fileProgress = make(chan FileHashProgress, 1)
wg.Add(1)
go func(base FileSize, done FileCount) {
defer wg.Done()
pt.forward(fileProgress, done, base)
}(scannedBytes, scannedFiles)
}
// Add to manifest with progress channel
bytesRead, err := builder.AddFile(
entry.Path,
entry.Size,
entry.Mtime,
f,
fileProgress,
)
_ = f.Close()
// Close channel and wait for goroutine to finish
if fileProgress != nil {
close(fileProgress)
wg.Wait()
}
if err != nil {
return 0, err
}
log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(sizeToUint64(bytesRead)))
return bytesRead, nil
}
// enumerateFS is the internal implementation that doesn't close the
// progress channel.
func (s *Scanner) enumerateFS(
afs afero.Fs,
basePath string,
progress chan<- EnumerateStatus,
) error {
return afero.Walk(afs, "/", func(p string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if !s.options.IncludeDotfiles && IsHiddenPath(p) {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
return s.enumerateFileWithInfo(p, basePath, info, progress)
})
}
// enumerateFileWithInfo adds a file with pre-existing fs.FileInfo.
func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info fs.FileInfo, progress chan<- EnumerateStatus) error {
func (s *Scanner) enumerateFileWithInfo(
filePath string,
basePath string,
info fs.FileInfo,
progress chan<- EnumerateStatus,
) error {
if info.IsDir() {
// Manifests contain only files, directories are implied
return nil
@@ -193,11 +419,13 @@ func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info f
realPath, err := filepath.EvalSymlinks(absPath)
if err != nil {
// Skip broken symlinks
return nil
return nil //nolint:nilerr // broken symlinks are skipped by design
}
realInfo, err := s.fs.Stat(realPath)
if err != nil {
return nil
// Skip symlinks whose target cannot be stat'd
return nil //nolint:nilerr // unreadable targets are skipped by design
}
// Skip if symlink points to a directory
if realInfo.IsDir() {
@@ -232,160 +460,78 @@ func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info f
return nil
}
// Files returns a copy of all files added to the scanner.
func (s *Scanner) Files() []*FileEntry {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]*FileEntry, len(s.files))
copy(out, s.files)
return out
// scanProgressTracker carries the shared state needed to report rate-limited
// scan progress updates.
type scanProgressTracker struct {
progress chan<- ScanStatus
totalFiles FileCount
totalBytes FileSize
startTime time.Time
lastProgress *time.Time
}
// FileCount returns the number of files in the scanner.
func (s *Scanner) FileCount() FileCount {
s.mu.RLock()
defer s.mu.RUnlock()
return FileCount(len(s.files))
}
// TotalBytes returns the total size of all files in the scanner.
func (s *Scanner) TotalBytes() FileSize {
s.mu.RLock()
defer s.mu.RUnlock()
return s.totalBytes
}
// ToManifest reads all file contents, computes hashes, and generates a manifest.
// If progress is non-nil, status updates are sent approximately once per second.
// The progress channel is closed when the method returns.
// The manifest is written to the provided io.Writer.
func (s *Scanner) ToManifest(ctx context.Context, w io.Writer, progress chan<- ScanStatus) error {
if progress != nil {
defer close(progress)
}
s.mu.RLock()
files := make([]*FileEntry, len(s.files))
copy(files, s.files)
totalFiles := FileCount(len(files))
var totalBytes FileSize
for _, f := range files {
totalBytes += f.Size
}
s.mu.RUnlock()
builder := NewBuilder()
if s.options.IncludeTimestamps {
builder.SetIncludeTimestamps(true)
}
if s.options.SigningOptions != nil {
builder.SetSigningOptions(s.options.SigningOptions)
}
if s.options.Seed != "" {
builder.SetSeed(s.options.Seed)
}
var scannedFiles FileCount
var scannedBytes FileSize
lastProgressTime := time.Now()
startTime := time.Now()
for _, entry := range files {
// Check for cancellation
select {
case <-ctx.Done():
return ctx.Err()
default:
// forward relays per-file hash progress to the scan progress channel,
// rate-limited to one update per second.
func (pt *scanProgressTracker) forward(
fileProgress <-chan FileHashProgress,
scannedFiles FileCount,
baseBytes FileSize,
) {
for p := range fileProgress {
// Send progress at most once per second
now := time.Now()
if now.Sub(*pt.lastProgress) < time.Second {
continue
}
// Open file
f, err := s.fs.Open(string(entry.AbsPath))
if err != nil {
return err
}
currentBytes := baseBytes + p.BytesRead
rate, eta := computeRateETA(now.Sub(pt.startTime), currentBytes, pt.totalBytes)
// Create progress channel for this file
var fileProgress chan FileHashProgress
var wg sync.WaitGroup
if progress != nil {
fileProgress = make(chan FileHashProgress, 1)
wg.Add(1)
go func(baseScannedBytes FileSize) {
defer wg.Done()
for p := range fileProgress {
// Send progress at most once per second
now := time.Now()
if now.Sub(lastProgressTime) >= time.Second {
elapsed := now.Sub(startTime).Seconds()
currentBytes := baseScannedBytes + p.BytesRead
var rate float64
var eta time.Duration
if elapsed > 0 && currentBytes > 0 {
rate = float64(currentBytes) / elapsed
remainingBytes := totalBytes - currentBytes
if rate > 0 {
eta = time.Duration(float64(remainingBytes)/rate) * time.Second
}
}
sendScanStatus(progress, ScanStatus{
TotalFiles: totalFiles,
ScannedFiles: scannedFiles,
TotalBytes: totalBytes,
ScannedBytes: currentBytes,
BytesPerSec: rate,
ETA: eta,
})
lastProgressTime = now
}
}
}(scannedBytes)
}
// Add to manifest with progress channel
bytesRead, err := builder.AddFile(
entry.Path,
entry.Size,
entry.Mtime,
f,
fileProgress,
)
_ = f.Close()
// Close channel and wait for goroutine to finish
if fileProgress != nil {
close(fileProgress)
wg.Wait()
}
if err != nil {
return err
}
log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(uint64(bytesRead)))
scannedFiles++
scannedBytes += bytesRead
}
// Send final progress (ETA is 0 at completion)
if progress != nil {
elapsed := time.Since(startTime).Seconds()
var rate float64
if elapsed > 0 {
rate = float64(scannedBytes) / elapsed
}
sendScanStatus(progress, ScanStatus{
TotalFiles: totalFiles,
sendScanStatus(pt.progress, ScanStatus{
TotalFiles: pt.totalFiles,
ScannedFiles: scannedFiles,
TotalBytes: totalBytes,
ScannedBytes: scannedBytes,
TotalBytes: pt.totalBytes,
ScannedBytes: currentBytes,
BytesPerSec: rate,
ETA: 0,
ETA: eta,
})
*pt.lastProgress = now
}
}
// computeRateETA returns the average throughput over elapsed time and the
// estimated time to process the remaining bytes at that rate.
func computeRateETA(
elapsed time.Duration,
done FileSize,
total FileSize,
) (float64, time.Duration) {
var (
rate float64
eta time.Duration
)
if elapsed > 0 && done > 0 {
rate = float64(done) / elapsed.Seconds()
remaining := total - done
if rate > 0 {
eta = time.Duration(float64(remaining)/rate) * time.Second
}
}
// Build and write manifest
return builder.Build(w)
return rate, eta
}
// sizeToUint64 converts a FileSize to uint64 for display, clamping
// negative values to zero so the conversion cannot overflow.
func sizeToUint64(v FileSize) uint64 {
if v < 0 {
return 0
}
return uint64(v)
}
// IsHiddenPath returns true if the path or any of its parent directories
@@ -396,17 +542,21 @@ func IsHiddenPath(p string) bool {
if tp == "." || tp == "/" {
return false
}
if strings.HasPrefix(tp, ".") {
return true
}
for {
d, f := path.Split(tp)
if strings.HasPrefix(f, ".") {
return true
}
if d == "" {
return false
}
tp = d[0 : len(d)-1] // trim trailing slash from dir
}
}
@@ -417,6 +567,7 @@ func sendEnumerateStatus(ch chan<- EnumerateStatus, status EnumerateStatus) {
if ch == nil {
return
}
select {
case ch <- status:
default:
@@ -430,6 +581,7 @@ func sendScanStatus(ch chan<- ScanStatus, status ScanStatus) {
if ch == nil {
return
}
select {
case ch <- status:
default:
+84 -17
View File
@@ -1,3 +1,4 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
@@ -12,6 +13,8 @@ import (
)
func TestNewScanner(t *testing.T) {
t.Parallel()
s := NewScanner()
assert.NotNil(t, s)
assert.Equal(t, FileCount(0), s.FileCount())
@@ -19,12 +22,18 @@ func TestNewScanner(t *testing.T) {
}
func TestNewScannerWithOptions(t *testing.T) {
t.Parallel()
t.Run("nil options", func(t *testing.T) {
t.Parallel()
s := NewScannerWithOptions(nil)
assert.NotNil(t, s)
})
t.Run("with options", func(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
opts := &ScannerOptions{
IncludeDotfiles: true,
@@ -37,6 +46,8 @@ func TestNewScannerWithOptions(t *testing.T) {
}
func TestScannerEnumerateFile(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("hello world"), 0o644))
@@ -54,6 +65,8 @@ func TestScannerEnumerateFile(t *testing.T) {
}
func TestScannerEnumerateFileMissing(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
err := s.EnumerateFile("/nonexistent.txt")
@@ -61,11 +74,14 @@ func TestScannerEnumerateFileMissing(t *testing.T) {
}
func TestScannerEnumeratePath(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir/subdir", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("one"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("two"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/subdir/file3.txt", []byte("three"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/subdir/file3.txt", []byte("three"), 0o644))
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
err := s.EnumeratePath("/testdir", nil)
@@ -76,6 +92,8 @@ func TestScannerEnumeratePath(t *testing.T) {
}
func TestScannerEnumeratePathWithProgress(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("one"), 0o644))
@@ -100,6 +118,8 @@ func TestScannerEnumeratePathWithProgress(t *testing.T) {
}
func TestScannerEnumeratePaths(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/dir1", 0o755))
require.NoError(t, fs.MkdirAll("/dir2", 0o755))
@@ -114,13 +134,20 @@ func TestScannerEnumeratePaths(t *testing.T) {
}
func TestScannerExcludeDotfiles(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir/.hidden", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/visible.txt", []byte("visible"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/.hidden.txt", []byte("hidden"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/.hidden/inside.txt", []byte("inside"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/visible.txt", []byte("visible"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/.hidden.txt", []byte("hidden"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/.hidden/inside.txt", []byte("inside"), 0o644))
t.Run("exclude by default", func(t *testing.T) {
t.Parallel()
s := NewScannerWithOptions(&ScannerOptions{Fs: fs, IncludeDotfiles: false})
err := s.EnumeratePath("/testdir", nil)
require.NoError(t, err)
@@ -131,6 +158,8 @@ func TestScannerExcludeDotfiles(t *testing.T) {
})
t.Run("include when enabled", func(t *testing.T) {
t.Parallel()
s := NewScannerWithOptions(&ScannerOptions{Fs: fs, IncludeDotfiles: true})
err := s.EnumeratePath("/testdir", nil)
require.NoError(t, err)
@@ -140,34 +169,43 @@ func TestScannerExcludeDotfiles(t *testing.T) {
}
func TestScannerToManifest(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("content one"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("content two"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/file1.txt", []byte("content one"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/file2.txt", []byte("content two"), 0o644))
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
err := s.EnumeratePath("/testdir", nil)
require.NoError(t, err)
var buf bytes.Buffer
err = s.ToManifest(context.Background(), &buf, nil)
require.NoError(t, err)
// Manifest should have magic bytes
assert.True(t, buf.Len() > 0)
assert.Positive(t, buf.Len())
assert.Equal(t, MAGIC, string(buf.Bytes()[:8]))
}
func TestScannerToManifestWithProgress(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/file.txt", bytes.Repeat([]byte("x"), 1000), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/file.txt", bytes.Repeat([]byte("x"), 1000), 0o644))
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
err := s.EnumeratePath("/testdir", nil)
require.NoError(t, err)
var buf bytes.Buffer
progress := make(chan ScanStatus, 10)
err = s.ToManifest(context.Background(), &buf, progress)
@@ -188,12 +226,15 @@ func TestScannerToManifestWithProgress(t *testing.T) {
}
func TestScannerToManifestContextCancellation(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
// Create many files to ensure we have time to cancel
for i := 0; i < 100; i++ {
for i := range 100 {
name := string(rune('a'+i%26)) + string(rune('0'+i/26)) + ".txt"
require.NoError(t, afero.WriteFile(fs, "/testdir/"+name, bytes.Repeat([]byte("x"), 100), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/"+name, bytes.Repeat([]byte("x"), 100), 0o644))
}
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
@@ -204,24 +245,30 @@ func TestScannerToManifestContextCancellation(t *testing.T) {
cancel() // Cancel immediately
var buf bytes.Buffer
err = s.ToManifest(ctx, &buf, nil)
assert.ErrorIs(t, err, context.Canceled)
}
func TestScannerToManifestEmptyScanner(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
var buf bytes.Buffer
err := s.ToManifest(context.Background(), &buf, nil)
require.NoError(t, err)
// Should still produce a valid manifest
assert.True(t, buf.Len() > 0)
assert.Positive(t, buf.Len())
assert.Equal(t, MAGIC, string(buf.Bytes()[:8]))
}
func TestScannerFilesCopiesSlice(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("hello"), 0o644))
@@ -236,10 +283,13 @@ func TestScannerFilesCopiesSlice(t *testing.T) {
}
func TestScannerEnumerateFS(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir/sub", 0o755))
require.NoError(t, afero.WriteFile(fs, "/testdir/file.txt", []byte("hello"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/testdir/sub/nested.txt", []byte("world"), 0o644))
require.NoError(t,
afero.WriteFile(fs, "/testdir/sub/nested.txt", []byte("world"), 0o644))
// Create a basepath filesystem
baseFs := afero.NewBasePathFs(fs, "/testdir")
@@ -252,13 +302,17 @@ func TestScannerEnumerateFS(t *testing.T) {
}
func TestSendEnumerateStatusNonBlocking(t *testing.T) {
t.Parallel()
// Channel with no buffer - send should not block
ch := make(chan EnumerateStatus)
// This should not block
done := make(chan bool)
go func() {
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
done <- true
}()
@@ -271,12 +325,16 @@ func TestSendEnumerateStatusNonBlocking(t *testing.T) {
}
func TestSendScanStatusNonBlocking(t *testing.T) {
t.Parallel()
// Channel with no buffer - send should not block
ch := make(chan ScanStatus)
done := make(chan bool)
go func() {
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
done <- true
}()
@@ -289,14 +347,19 @@ func TestSendScanStatusNonBlocking(t *testing.T) {
}
func TestSendStatusNilChannel(t *testing.T) {
t.Parallel()
// Should not panic with nil channel
sendEnumerateStatus(nil, EnumerateStatus{})
sendScanStatus(nil, ScanStatus{})
}
func TestScannerFileEntryFields(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
now := time.Now().Truncate(time.Second)
require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("content"), 0o644))
require.NoError(t, fs.Chtimes("/test.txt", now, now))
@@ -315,11 +378,13 @@ func TestScannerFileEntryFields(t *testing.T) {
}
func TestScannerLargeFileEnumeration(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
// Create 100 files
for i := 0; i < 100; i++ {
for i := range 100 {
name := "/testdir/" + string(rune('a'+i%26)) + string(rune('0'+i/26%10)) + ".txt"
require.NoError(t, afero.WriteFile(fs, name, []byte("data"), 0o644))
}
@@ -330,20 +395,20 @@ func TestScannerLargeFileEnumeration(t *testing.T) {
err := s.EnumeratePath("/testdir", progress)
require.NoError(t, err)
// Drain channel
for range progress {
}
// progress is fully buffered and closed; no draining needed
assert.Equal(t, FileCount(100), s.FileCount())
assert.Equal(t, FileSize(400), s.TotalBytes()) // 100 * 4 bytes
}
func TestIsHiddenPath(t *testing.T) {
t.Parallel()
tests := []struct {
path string
hidden bool
}{
{"file.txt", false},
{testFileName, false},
{".hidden", true},
{"dir/file.txt", false},
{"dir/.hidden", true},
@@ -360,6 +425,8 @@ func TestIsHiddenPath(t *testing.T) {
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
t.Parallel()
assert.Equal(t, tt.hidden, IsHiddenPath(tt.path), "IsHiddenPath(%q)", tt.path)
})
}
+79 -28
View File
@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"errors"
"fmt"
"math"
"time"
"github.com/google/uuid"
@@ -15,47 +16,80 @@ import (
// MAGIC is the file format magic bytes prefix (rot13 of "MANIFEST").
const MAGIC string = "ZNAVSRFG"
var (
// errInnerNotSet is returned by generate when the inner manifest is
// missing.
errInnerNotSet = errors.New("internal error: pbInner not set")
// errInternal is returned by generateOuter for the same condition.
// The two messages differ, and both are load-bearing for callers that
// match on text, so they are kept distinct.
errInternal = errors.New("internal error")
)
// nanosecondsInt32 converts t's nanosecond component to int32.
// time.Time.Nanosecond is documented to return a value in [0, 999999999],
// so the conversion cannot overflow. This sits directly in the manifest
// content path: silently substituting a default would zero every entry's
// mtime nanos and change the serialized bytes and their hash, so an
// out-of-contract value is a programming error and panics rather than
// being papered over.
func nanosecondsInt32(t time.Time) int32 {
n := t.Nanosecond()
if n < 0 || n > math.MaxInt32 {
panic(fmt.Sprintf(
"mfer: time.Time.Nanosecond out of contract: %d", n))
}
return int32(n)
}
func newTimestampFromTime(t time.Time) *Timestamp {
return &Timestamp{
Seconds: t.Unix(),
Nanos: int32(t.Nanosecond()),
Nanos: nanosecondsInt32(t),
}
}
func (m *manifest) generate() error {
if m.pbInner == nil {
return errors.New("internal error: pbInner not set")
return errInnerNotSet
}
if m.pbOuter == nil {
e := m.generateOuter()
if e != nil {
return e
}
}
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
if err != nil {
return fmt.Errorf("serialize: marshal outer: %w", err)
}
m.output = bytes.NewBuffer([]byte(MAGIC))
m.output = bytes.NewBufferString(MAGIC)
_, err = m.output.Write(dat)
if err != nil {
return fmt.Errorf("serialize: write output: %w", err)
}
return nil
}
func (m *manifest) generateOuter() error {
if m.pbInner == nil {
return errors.New("internal error")
return errInternal
}
// Use fixed UUID if provided, otherwise generate a new one
var manifestUUID uuid.UUID
if len(m.fixedUUID) == 16 {
if len(m.fixedUUID) == uuidLength {
copy(manifestUUID[:], m.fixedUUID)
} else {
manifestUUID = uuid.New()
}
m.pbInner.Uuid = manifestUUID[:]
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
@@ -65,23 +99,29 @@ func (m *manifest) generateOuter() error {
// Compress the inner data
idc := new(bytes.Buffer)
zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
if err != nil {
return fmt.Errorf("serialize: create compressor: %w", err)
}
_, err = zw.Write(innerData)
if err != nil {
return fmt.Errorf("serialize: compress: %w", err)
}
_ = zw.Close()
compressedData := idc.Bytes()
// Hash the compressed data for integrity verification before decompression
h := sha256.New()
if _, err := h.Write(compressedData); err != nil {
_, err = h.Write(compressedData)
if err != nil {
return fmt.Errorf("serialize: hash write: %w", err)
}
sha256Hash := h.Sum(nil)
m.pbOuter = &MFFileOuter{
@@ -95,29 +135,40 @@ func (m *manifest) generateOuter() error {
// Sign the manifest if signing options are provided
if m.signingOptions != nil && m.signingOptions.KeyID != "" {
sigString, err := m.signatureString()
if err != nil {
return fmt.Errorf("failed to generate signature string: %w", err)
}
sig, err := gpgSign([]byte(sigString), m.signingOptions.KeyID)
if err != nil {
return fmt.Errorf("failed to sign manifest: %w", err)
}
m.pbOuter.Signature = sig
fingerprint, err := gpgGetKeyFingerprint(m.signingOptions.KeyID)
if err != nil {
return fmt.Errorf("failed to get key fingerprint: %w", err)
}
m.pbOuter.Signer = fingerprint
pubKey, err := gpgExportPublicKey(m.signingOptions.KeyID)
if err != nil {
return fmt.Errorf("failed to export public key: %w", err)
}
m.pbOuter.SigningPubKey = pubKey
return m.signOuter()
}
return nil
}
// signOuter signs the outer message with the configured GPG key and
// embeds the signature, signer fingerprint, and public key.
func (m *manifest) signOuter() error {
sigString, err := m.signatureString()
if err != nil {
return fmt.Errorf("failed to generate signature string: %w", err)
}
sig, err := gpgSign([]byte(sigString), m.signingOptions.KeyID)
if err != nil {
return fmt.Errorf("failed to sign manifest: %w", err)
}
m.pbOuter.Signature = sig
fingerprint, err := gpgGetKeyFingerprint(m.signingOptions.KeyID)
if err != nil {
return fmt.Errorf("failed to get key fingerprint: %w", err)
}
m.pbOuter.Signer = fingerprint
pubKey, err := gpgExportPublicKey(m.signingOptions.KeyID)
if err != nil {
return fmt.Errorf("failed to export public key: %w", err)
}
m.pbOuter.SigningPubKey = pubKey
return nil
}
+2
View File
@@ -32,12 +32,14 @@ func (b BaseURL) JoinPath(path RelFilePath) (FileURL, error) {
for i, seg := range segments {
segments[i] = url.PathEscape(seg)
}
ref, err := url.Parse(strings.Join(segments, "/"))
if err != nil {
return "", err
}
resolved := base.ResolveReference(ref)
return FileURL(resolved.String()), nil
}
+18 -3
View File
@@ -1,3 +1,4 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
@@ -8,19 +9,27 @@ import (
)
func TestBaseURLJoinPath(t *testing.T) {
t.Parallel()
tests := []struct {
base BaseURL
path RelFilePath
expected string
}{
{"https://example.com/dir/", "file.txt", "https://example.com/dir/file.txt"},
{"https://example.com/dir", "file.txt", "https://example.com/dir/file.txt"},
{"https://example.com/dir/", testFileName, "https://example.com/dir/file.txt"},
{"https://example.com/dir", testFileName, "https://example.com/dir/file.txt"},
{"https://example.com/", "sub/file.txt", "https://example.com/sub/file.txt"},
{"https://example.com/dir/", "file with spaces.txt", "https://example.com/dir/file%20with%20spaces.txt"},
{
"https://example.com/dir/",
"file with spaces.txt",
"https://example.com/dir/file%20with%20spaces.txt",
},
}
for _, tt := range tests {
t.Run(string(tt.base)+"+"+string(tt.path), func(t *testing.T) {
t.Parallel()
result, err := tt.base.JoinPath(tt.path)
require.NoError(t, err)
assert.Equal(t, tt.expected, string(result))
@@ -29,16 +38,22 @@ func TestBaseURLJoinPath(t *testing.T) {
}
func TestBaseURLString(t *testing.T) {
t.Parallel()
b := BaseURL("https://example.com/")
assert.Equal(t, "https://example.com/", b.String())
}
func TestFileURLString(t *testing.T) {
t.Parallel()
f := FileURL("https://example.com/file.txt")
assert.Equal(t, "https://example.com/file.txt", f.String())
}
func TestManifestURLString(t *testing.T) {
t.Parallel()
m := ManifestURL("https://example.com/index.mf")
assert.Equal(t, "https://example.com/index.mf", m.String())
}