Merge branch 'next' into fix/issue-24

Add `ValidatePath()` enforcing UTF-8, forward-slash, relative, no `..`, no empty segments. Applied in `AddFile` and `AddFileWithHash`. Proto comments document the rules.

Co-authored-by: clawbot <clawbot@openclaw>
Co-authored-by: Jeffrey Paul <sneak@noreply.example.org>
Reviewed-on: #31
Co-authored-by: clawbot <clawbot@noreply.example.org>
Co-committed-by: clawbot <clawbot@noreply.example.org>

internal/checker/checker.go

@@ -227,14 +227,12 @@ func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *int64) Result
 
 // FindExtraFiles walks the filesystem and reports files not in the manifest.
 // Results are sent to the results channel. The channel is closed when done.
-// Hidden files/directories (starting with .) are skipped, as they are excluded
-// from manifests by default. The manifest file itself is also skipped.
 func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
 	if results != nil {
 		defer close(results)
 	}
 
-	return afero.Walk(c.fs, c.basePath, func(walkPath string, info os.FileInfo, err error) error {
+	return afero.Walk(c.fs, c.basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -245,29 +243,15 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
 		default:
 		}
 
-		// Get relative path
-		relPath, err := filepath.Rel(c.basePath, walkPath)
-		if err != nil {
-			return err
-		}
-
-		// Skip hidden files and directories (dotfiles)
-		if mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
 		// Skip directories
 		if info.IsDir() {
 			return nil
 		}
 
-		// Skip manifest files
+		// Get relative path
-		base := filepath.Base(relPath)
+		relPath, err := filepath.Rel(c.basePath, path)
-		if base == "index.mf" || base == ".index.mf" {
+		if err != nil {
-			return nil
+			return err
 		}
 
 		// Check if path is in manifest

mfer/builder.go

@@ -3,13 +3,47 @@ package mfer
 import (
 	"crypto/sha256"
 	"errors"
+	"fmt"
 	"io"
+	"strings"
 	"sync"
 	"time"
+	"unicode/utf8"
 
 	"github.com/multiformats/go-multihash"
 )
 
+// ValidatePath checks that a file path conforms to manifest path invariants:
+//   - Must be valid UTF-8
+//   - Must use forward slashes only (no backslashes)
+//   - Must be relative (no leading /)
+//   - Must not contain ".." segments
+//   - Must not contain empty segments (no "//")
+//   - Must not be empty
+func ValidatePath(p string) error {
+	if p == "" {
+		return errors.New("path cannot be empty")
+	}
+	if !utf8.ValidString(p) {
+		return fmt.Errorf("path %q is not valid UTF-8", p)
+	}
+	if strings.ContainsRune(p, '\\') {
+		return fmt.Errorf("path %q contains backslash; use forward slashes only", p)
+	}
+	if strings.HasPrefix(p, "/") {
+		return fmt.Errorf("path %q is absolute; must be relative", p)
+	}
+	for _, seg := range strings.Split(p, "/") {
+		if seg == "" {
+			return fmt.Errorf("path %q contains empty segment", p)
+		}
+		if seg == ".." {
+			return fmt.Errorf("path %q contains '..' segment", p)
+		}
+	}
+	return nil
+}
+
 // RelFilePath represents a relative file path within a manifest.
 type RelFilePath string
 
@@ -74,6 +108,10 @@ func (b *Builder) AddFile(
 	reader io.Reader,
 	progress chan<- FileHashProgress,
 ) (FileSize, error) {
+	if err := ValidatePath(string(path)); err != nil {
+		return 0, err
+	}
+
 	// Create hash writer
 	h := sha256.New()
 
@@ -96,6 +134,11 @@ func (b *Builder) AddFile(
 		}
 	}
 
+	// Verify actual bytes read matches declared size
+	if totalRead != size {
+		return totalRead, fmt.Errorf("size mismatch for %q: declared %d bytes but read %d bytes", path, size, totalRead)
+	}
+
 	// Encode hash as multihash (SHA2-256)
 	mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
 	if err != nil {
@@ -141,8 +184,8 @@ func (b *Builder) FileCount() int {
 // This is useful when the hash is already known (e.g., from an existing manifest).
 // Returns an error if path is empty, size is negative, or hash is nil/empty.
 func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime, hash Multihash) error {
-	if path == "" {
+	if err := ValidatePath(string(path)); err != nil {
-		return errors.New("path cannot be empty")
+		return err
 	}
 	if size < 0 {
 		return errors.New("size cannot be negative")

mfer/checker.go

@@ -277,14 +277,12 @@ func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
 
 // FindExtraFiles walks the filesystem and reports files not in the manifest.
 // Results are sent to the results channel. The channel is closed when done.
-// Hidden files/directories (starting with .) are skipped, as they are excluded
-// from manifests by default. The manifest file itself is also skipped.
 func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
 	if results != nil {
 		defer close(results)
 	}
 
-	return afero.Walk(c.fs, string(c.basePath), func(walkPath string, info os.FileInfo, err error) error {
+	return afero.Walk(c.fs, string(c.basePath), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -295,31 +293,16 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
 		default:
 		}
 
-		// Get relative path
-		rel, err := filepath.Rel(string(c.basePath), walkPath)
-		if err != nil {
-			return err
-		}
-
-		// Skip hidden files and directories (dotfiles)
-		if IsHiddenPath(filepath.ToSlash(rel)) {
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
 		// Skip directories
 		if info.IsDir() {
 			return nil
 		}
 
-		// Skip manifest files
+		// Get relative path
-		base := filepath.Base(rel)
+		rel, err := filepath.Rel(string(c.basePath), path)
-		if base == "index.mf" || base == ".index.mf" {
+		if err != nil {
-			return nil
+			return err
 		}
-
 		relPath := RelFilePath(rel)
 
 		// Check if path is in manifest

mfer/checker_test.go

@@ -305,44 +305,6 @@ func TestFindExtraFiles(t *testing.T) {
 	assert.Equal(t, "not in manifest", extras[0].Message)
 }
 
-func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
-	fs := afero.NewMemMapFs()
-	manifestFiles := map[string][]byte{
-		"file1.txt": []byte("in manifest"),
-	}
-	createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
-	createFilesOnDisk(t, fs, "/data", map[string][]byte{
-		"file1.txt": []byte("in manifest"),
-	})
-	// Create dotfile and manifest that should be skipped
-	require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
-	require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
-	// Create a real extra file
-	require.NoError(t, fs.MkdirAll("/data", 0o755))
-	require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
-
-	chk, err := NewChecker("/data/.index.mf", "/data", fs)
-	require.NoError(t, err)
-
-	results := make(chan Result, 10)
-	err = chk.FindExtraFiles(context.Background(), results)
-	require.NoError(t, err)
-
-	var extras []Result
-	for r := range results {
-		extras = append(extras, r)
-	}
-
-	// Should only report extra.txt, not .hidden, .config/settings, or .index.mf
-	for _, e := range extras {
-		t.Logf("extra: %s", e.Path)
-	}
-	assert.Len(t, extras, 1)
-	if len(extras) > 0 {
-		assert.Equal(t, RelFilePath("extra.txt"), extras[0].Path)
-	}
-}
-
 func TestFindExtraFilesContextCancellation(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	files := map[string][]byte{"file.txt": []byte("data")}

mfer/constants.go

@@ -3,4 +3,9 @@ package mfer
 const (
 	Version     = "0.1.0"
 	ReleaseDate = "2025-12-17"
+
+	// MaxDecompressedSize is the maximum allowed size of decompressed manifest
+	// data (256 MB). This prevents decompression bombs from consuming excessive
+	// memory.
+	MaxDecompressedSize int64 = 256 * 1024 * 1024
 )

mfer/deserialize.go

@@ -76,10 +76,20 @@ func (m *manifest) deserializeInner() error {
 	}
 	defer zr.Close()
 
-	dat, err := io.ReadAll(zr)
+	// Limit decompressed size to prevent decompression bombs.
+	// Use declared size + 1 byte to detect overflow, capped at MaxDecompressedSize.
+	maxSize := MaxDecompressedSize
+	if m.pbOuter.Size > 0 && m.pbOuter.Size < int64(maxSize) {
+		maxSize = int64(m.pbOuter.Size) + 1
+	}
+	limitedReader := io.LimitReader(zr, maxSize)
+	dat, err := io.ReadAll(limitedReader)
 	if err != nil {
 		return err
 	}
+	if int64(len(dat)) >= MaxDecompressedSize {
+		return fmt.Errorf("decompressed data exceeds maximum allowed size of %d bytes", MaxDecompressedSize)
+	}
 
 	isize := len(dat)
 	if int64(isize) != m.pbOuter.Size {

mfer/mf.proto

@@ -46,6 +46,9 @@ message MFFileOuter {
 
 message MFFilePath {
     // required attributes:
+    // Path invariants: must be valid UTF-8, use forward slashes only,
+    // be relative (no leading /), contain no ".." segments, and no
+    // empty segments (no "//").
     string path = 1;
     int64 size = 2;
 

mfer/scanner.go

@@ -385,9 +385,6 @@ func (s *Scanner) ToManifest(ctx context.Context, w io.Writer, progress chan<- S
 // The path should use forward slashes.
 func IsHiddenPath(p string) bool {
 	tp := path.Clean(p)
-	if tp == "." || tp == "/" {
-		return false
-	}
 	if strings.HasPrefix(tp, ".") {
 		return true
 	}