Add custom types for type safety throughout codebase

- Add FileCount, FileSize, RelFilePath, AbsFilePath, ModTime, Multihash types - Add UnixSeconds and UnixNanos types for timestamp handling - Add URL types (ManifestURL, FileURL, BaseURL) with safe path joining - Consolidate scanner package into mfer package - Update checker to use custom types in Result and CheckStatus - Add ModTime.Timestamp() method for protobuf conversion - Update all tests to use proper custom types
2025-12-18 01:01:18 -08:00
parent a9f0d2abe4
commit dc115c5ba2
9 changed files with 428 additions and 247 deletions
@@ -2,6 +2,7 @@ package mfer

 import (
 	"crypto/sha256"
+	"errors"
 	"io"
 	"sync"
 	"time"
@@ -9,9 +10,42 @@ import (
 	"github.com/multiformats/go-multihash"
 )

+// RelFilePath represents a relative file path within a manifest.
+type RelFilePath string
+
+// AbsFilePath represents an absolute file path on the filesystem.
+type AbsFilePath string
+
+// FileSize represents the size of a file in bytes.
+type FileSize int64
+
+// FileCount represents a count of files.
+type FileCount int64
+
+// ModTime represents a file's modification time.
+type ModTime time.Time
+
+// UnixSeconds represents seconds since Unix epoch.
+type UnixSeconds int64
+
+// UnixNanos represents the nanosecond component of a timestamp (0-999999999).
+type UnixNanos int32
+
+// Timestamp converts ModTime to a protobuf Timestamp.
+func (m ModTime) Timestamp() *Timestamp {
+	t := time.Time(m)
+	return &Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+}
+
+// Multihash represents a multihash-encoded file hash (typically SHA2-256).
+type Multihash []byte
+
 // FileHashProgress reports progress during file hashing.
 type FileHashProgress struct {
-	BytesRead int64 // Total bytes read so far for the current file
+	BytesRead FileSize // Total bytes read so far for the current file
 }

 // Builder constructs a manifest by adding files one at a time.
@@ -33,24 +67,24 @@ func NewBuilder() *Builder {
 // Progress updates are sent to the progress channel (if non-nil) without blocking.
 // Returns the number of bytes read.
 func (b *Builder) AddFile(
-	path string,
-	size int64,
-	mtime time.Time,
+	path RelFilePath,
+	size FileSize,
+	mtime ModTime,
 	reader io.Reader,
 	progress chan<- FileHashProgress,
-) (int64, error) {
+) (FileSize, error) {
 	// Create hash writer
 	h := sha256.New()

 	// Read file in chunks, updating hash and progress
-	var totalRead int64
+	var totalRead FileSize
 	buf := make([]byte, 64*1024) // 64KB chunks

 	for {
 		n, err := reader.Read(buf)
 		if n > 0 {
 			h.Write(buf[:n])
-			totalRead += int64(n)
+			totalRead += FileSize(n)
 			sendFileHashProgress(progress, FileHashProgress{BytesRead: totalRead})
 		}
 		if err == io.EOF {
@@ -69,12 +103,12 @@ func (b *Builder) AddFile(

 	// Create file entry
 	entry := &MFFilePath{
-		Path: path,
-		Size: size,
+		Path: string(path),
+		Size: int64(size),
 		Hashes: []*MFFileChecksum{
 			{MultiHash: mh},
 		},
-		Mtime: newTimestampFromTime(mtime),
+		Mtime: mtime.Timestamp(),
 	}

 	b.mu.Lock()
@@ -104,19 +138,31 @@ func (b *Builder) FileCount() int {

 // AddFileWithHash adds a file entry with a pre-computed hash.
 // This is useful when the hash is already known (e.g., from an existing manifest).
-func (b *Builder) AddFileWithHash(path string, size int64, mtime time.Time, hash []byte) {
+// Returns an error if path is empty, size is negative, or hash is nil/empty.
+func (b *Builder) AddFileWithHash(path RelFilePath, size FileSize, mtime ModTime, hash Multihash) error {
+	if path == "" {
+		return errors.New("path cannot be empty")
+	}
+	if size < 0 {
+		return errors.New("size cannot be negative")
+	}
+	if len(hash) == 0 {
+		return errors.New("hash cannot be nil or empty")
+	}
+
 	entry := &MFFilePath{
-		Path: path,
-		Size: size,
+		Path: string(path),
+		Size: int64(size),
 		Hashes: []*MFFileChecksum{
 			{MultiHash: hash},
 		},
-		Mtime: newTimestampFromTime(mtime),
+		Mtime: mtime.Timestamp(),
 	}

 	b.mu.Lock()
 	b.files = append(b.files, entry)
 	b.mu.Unlock()
+	return nil
 }

 // Build finalizes the manifest and writes it to the writer.
@@ -0,0 +1,104 @@
+package mfer
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewBuilder(t *testing.T) {
+	b := NewBuilder()
+	assert.NotNil(t, b)
+	assert.Equal(t, 0, b.FileCount())
+}
+
+func TestBuilderAddFile(t *testing.T) {
+	b := NewBuilder()
+	content := []byte("test content")
+	reader := bytes.NewReader(content)
+
+	bytesRead, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil)
+	require.NoError(t, err)
+	assert.Equal(t, FileSize(len(content)), bytesRead)
+	assert.Equal(t, 1, b.FileCount())
+}
+
+func TestBuilderAddFileWithHash(t *testing.T) {
+	b := NewBuilder()
+	hash := make([]byte, 34) // SHA256 multihash is 34 bytes
+
+	err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
+	require.NoError(t, err)
+	assert.Equal(t, 1, b.FileCount())
+}
+
+func TestBuilderAddFileWithHashValidation(t *testing.T) {
+	t.Run("empty path", func(t *testing.T) {
+		b := NewBuilder()
+		hash := make([]byte, 34)
+		err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "path")
+	})
+
+	t.Run("negative size", func(t *testing.T) {
+		b := NewBuilder()
+		hash := make([]byte, 34)
+		err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "size")
+	})
+
+	t.Run("nil hash", func(t *testing.T) {
+		b := NewBuilder()
+		err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "hash")
+	})
+
+	t.Run("empty hash", func(t *testing.T) {
+		b := NewBuilder()
+		err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "hash")
+	})
+
+	t.Run("valid inputs", func(t *testing.T) {
+		b := NewBuilder()
+		hash := make([]byte, 34)
+		err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, b.FileCount())
+	})
+}
+
+func TestBuilderBuild(t *testing.T) {
+	b := NewBuilder()
+	content := []byte("test content")
+	reader := bytes.NewReader(content)
+
+	_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	err = b.Build(&buf)
+	require.NoError(t, err)
+
+	// Should have magic bytes
+	assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
+}
+
+func TestBuilderBuildEmpty(t *testing.T) {
+	b := NewBuilder()
+
+	var buf bytes.Buffer
+	err := b.Build(&buf)
+	require.NoError(t, err)
+
+	// Should still produce valid manifest with 0 files
+	assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
+}
@@ -0,0 +1,423 @@
+package mfer
+
+import (
+	"context"
+	"io"
+	"io/fs"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/spf13/afero"
+	"sneak.berlin/go/mfer/internal/log"
+)
+
+// Phase 1: Enumeration
+// ---------------------
+// Walking directories and calling stat() on files to collect metadata.
+// Builds the list of files to be scanned. Relatively fast (metadata only).
+
+// EnumerateStatus contains progress information for the enumeration phase.
+type EnumerateStatus struct {
+	FilesFound FileCount // Number of files discovered so far
+	BytesFound FileSize  // Total size of discovered files (from stat)
+}
+
+// Phase 2: Scan (ToManifest)
+// --------------------------
+// Reading file contents and computing hashes for manifest generation.
+// This is the expensive phase that reads all file data.
+
+// ScanStatus contains progress information for the scan phase.
+type ScanStatus struct {
+	TotalFiles   FileCount     // Total number of files to scan
+	ScannedFiles FileCount     // Number of files scanned so far
+	TotalBytes   FileSize      // Total bytes to read (sum of all file sizes)
+	ScannedBytes FileSize      // Bytes read so far
+	BytesPerSec  float64       // Current throughput rate
+	ETA          time.Duration // Estimated time to completion
+}
+
+// ScannerOptions configures scanner behavior.
+type ScannerOptions struct {
+	IncludeDotfiles bool     // Include files and directories starting with a dot (default: exclude)
+	FollowSymLinks  bool     // Resolve symlinks instead of skipping them
+	Fs              afero.Fs // Filesystem to use, defaults to OsFs if nil
+}
+
+// FileEntry represents a file that has been enumerated.
+type FileEntry struct {
+	Path    RelFilePath // Relative path (used in manifest)
+	AbsPath AbsFilePath // Absolute path (used for reading file content)
+	Size    FileSize    // File size in bytes
+	Mtime   ModTime     // Last modification time
+	Ctime   time.Time   // Creation time (platform-dependent)
+}
+
+// Scanner accumulates files and generates manifests from them.
+type Scanner struct {
+	mu         sync.RWMutex
+	files      []*FileEntry
+	totalBytes FileSize // cached sum of all file sizes
+	options    *ScannerOptions
+	fs         afero.Fs
+}
+
+// NewScanner creates a new Scanner with default options.
+func NewScanner() *Scanner {
+	return NewScannerWithOptions(nil)
+}
+
+// NewScannerWithOptions creates a new Scanner with the given options.
+func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
+	if opts == nil {
+		opts = &ScannerOptions{}
+	}
+	fs := opts.Fs
+	if fs == nil {
+		fs = afero.NewOsFs()
+	}
+	return &Scanner{
+		files:   make([]*FileEntry, 0),
+		options: opts,
+		fs:      fs,
+	}
+}
+
+// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
+func (s *Scanner) EnumerateFile(filePath string) error {
+	abs, err := filepath.Abs(filePath)
+	if err != nil {
+		return err
+	}
+	info, err := s.fs.Stat(abs)
+	if err != nil {
+		return err
+	}
+	// For single files, use the filename as the relative path
+	basePath := filepath.Dir(abs)
+	return s.enumerateFileWithInfo(filepath.Base(abs), basePath, info, nil)
+}
+
+// EnumeratePath walks a directory path and adds all files to the scanner.
+// If progress is non-nil, status updates are sent as files are discovered.
+// The progress channel is closed when the method returns.
+func (s *Scanner) EnumeratePath(inputPath string, progress chan<- EnumerateStatus) error {
+	if progress != nil {
+		defer close(progress)
+	}
+	abs, err := filepath.Abs(inputPath)
+	if err != nil {
+		return err
+	}
+	afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
+	return s.enumerateFS(afs, abs, progress)
+}
+
+// EnumeratePaths walks multiple directory paths and adds all files to the scanner.
+// If progress is non-nil, status updates are sent as files are discovered.
+// The progress channel is closed when the method returns.
+func (s *Scanner) EnumeratePaths(progress chan<- EnumerateStatus, inputPaths ...string) error {
+	if progress != nil {
+		defer close(progress)
+	}
+	for _, p := range inputPaths {
+		abs, err := filepath.Abs(p)
+		if err != nil {
+			return err
+		}
+		afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
+		if err := s.enumerateFS(afs, abs, progress); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// EnumerateFS walks an afero filesystem and adds all files to the scanner.
+// If progress is non-nil, status updates are sent as files are discovered.
+// The progress channel is closed when the method returns.
+// basePath is used to compute absolute paths for file reading.
+func (s *Scanner) EnumerateFS(afs afero.Fs, basePath string, progress chan<- EnumerateStatus) error {
+	if progress != nil {
+		defer close(progress)
+	}
+	return s.enumerateFS(afs, basePath, progress)
+}
+
+// enumerateFS is the internal implementation that doesn't close the progress channel.
+func (s *Scanner) enumerateFS(afs afero.Fs, basePath string, progress chan<- EnumerateStatus) error {
+	return afero.Walk(afs, "/", func(p string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !s.options.IncludeDotfiles && IsHiddenPath(p) {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		return s.enumerateFileWithInfo(p, basePath, info, progress)
+	})
+}
+
+// enumerateFileWithInfo adds a file with pre-existing fs.FileInfo.
+func (s *Scanner) enumerateFileWithInfo(filePath string, basePath string, info fs.FileInfo, progress chan<- EnumerateStatus) error {
+	if info.IsDir() {
+		// Manifests contain only files, directories are implied
+		return nil
+	}
+
+	// Clean the path - remove leading slash if present
+	cleanPath := filePath
+	if len(cleanPath) > 0 && cleanPath[0] == '/' {
+		cleanPath = cleanPath[1:]
+	}
+
+	// Compute absolute path for file reading
+	absPath := filepath.Join(basePath, cleanPath)
+
+	// Handle symlinks
+	if info.Mode()&fs.ModeSymlink != 0 {
+		if !s.options.FollowSymLinks {
+			// Skip symlinks when not following them
+			return nil
+		}
+		// Resolve symlink to get real file info
+		realPath, err := filepath.EvalSymlinks(absPath)
+		if err != nil {
+			// Skip broken symlinks
+			return nil
+		}
+		realInfo, err := s.fs.Stat(realPath)
+		if err != nil {
+			return nil
+		}
+		// Skip if symlink points to a directory
+		if realInfo.IsDir() {
+			return nil
+		}
+		// Use resolved path for reading, but keep original path in manifest
+		absPath = realPath
+		info = realInfo
+	}
+
+	entry := &FileEntry{
+		Path:    RelFilePath(cleanPath),
+		AbsPath: AbsFilePath(absPath),
+		Size:    FileSize(info.Size()),
+		Mtime:   ModTime(info.ModTime()),
+		// Note: Ctime not available from fs.FileInfo on all platforms
+		// Will need platform-specific code to extract it
+	}
+
+	s.mu.Lock()
+	s.files = append(s.files, entry)
+	s.totalBytes += entry.Size
+	filesFound := FileCount(len(s.files))
+	bytesFound := s.totalBytes
+	s.mu.Unlock()
+
+	sendEnumerateStatus(progress, EnumerateStatus{
+		FilesFound: filesFound,
+		BytesFound: bytesFound,
+	})
+
+	return nil
+}
+
+// Files returns a copy of all files added to the scanner.
+func (s *Scanner) Files() []*FileEntry {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := make([]*FileEntry, len(s.files))
+	copy(out, s.files)
+	return out
+}
+
+// FileCount returns the number of files in the scanner.
+func (s *Scanner) FileCount() FileCount {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return FileCount(len(s.files))
+}
+
+// TotalBytes returns the total size of all files in the scanner.
+func (s *Scanner) TotalBytes() FileSize {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.totalBytes
+}
+
+// ToManifest reads all file contents, computes hashes, and generates a manifest.
+// If progress is non-nil, status updates are sent approximately once per second.
+// The progress channel is closed when the method returns.
+// The manifest is written to the provided io.Writer.
+func (s *Scanner) ToManifest(ctx context.Context, w io.Writer, progress chan<- ScanStatus) error {
+	if progress != nil {
+		defer close(progress)
+	}
+
+	s.mu.RLock()
+	files := make([]*FileEntry, len(s.files))
+	copy(files, s.files)
+	totalFiles := FileCount(len(files))
+	var totalBytes FileSize
+	for _, f := range files {
+		totalBytes += f.Size
+	}
+	s.mu.RUnlock()
+
+	builder := NewBuilder()
+
+	var scannedFiles FileCount
+	var scannedBytes FileSize
+	lastProgressTime := time.Now()
+	startTime := time.Now()
+
+	for _, entry := range files {
+		// Check for cancellation
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		// Open file
+		f, err := s.fs.Open(string(entry.AbsPath))
+		if err != nil {
+			return err
+		}
+
+		// Create progress channel for this file
+		var fileProgress chan FileHashProgress
+		var wg sync.WaitGroup
+		if progress != nil {
+			fileProgress = make(chan FileHashProgress, 1)
+			wg.Add(1)
+			go func(baseScannedBytes FileSize) {
+				defer wg.Done()
+				for p := range fileProgress {
+					// Send progress at most once per second
+					now := time.Now()
+					if now.Sub(lastProgressTime) >= time.Second {
+						elapsed := now.Sub(startTime).Seconds()
+						currentBytes := baseScannedBytes + p.BytesRead
+						var rate float64
+						var eta time.Duration
+						if elapsed > 0 && currentBytes > 0 {
+							rate = float64(currentBytes) / elapsed
+							remainingBytes := totalBytes - currentBytes
+							if rate > 0 {
+								eta = time.Duration(float64(remainingBytes)/rate) * time.Second
+							}
+						}
+						sendScanStatus(progress, ScanStatus{
+							TotalFiles:   totalFiles,
+							ScannedFiles: scannedFiles,
+							TotalBytes:   totalBytes,
+							ScannedBytes: currentBytes,
+							BytesPerSec:  rate,
+							ETA:          eta,
+						})
+						lastProgressTime = now
+					}
+				}
+			}(scannedBytes)
+		}
+
+		// Add to manifest with progress channel
+		bytesRead, err := builder.AddFile(
+			entry.Path,
+			entry.Size,
+			entry.Mtime,
+			f,
+			fileProgress,
+		)
+		_ = f.Close()
+
+		// Close channel and wait for goroutine to finish
+		if fileProgress != nil {
+			close(fileProgress)
+			wg.Wait()
+		}
+
+		if err != nil {
+			return err
+		}
+
+		log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(uint64(bytesRead)))
+
+		scannedFiles++
+		scannedBytes += bytesRead
+	}
+
+	// Send final progress (ETA is 0 at completion)
+	if progress != nil {
+		elapsed := time.Since(startTime).Seconds()
+		var rate float64
+		if elapsed > 0 {
+			rate = float64(scannedBytes) / elapsed
+		}
+		sendScanStatus(progress, ScanStatus{
+			TotalFiles:   totalFiles,
+			ScannedFiles: scannedFiles,
+			TotalBytes:   totalBytes,
+			ScannedBytes: scannedBytes,
+			BytesPerSec:  rate,
+			ETA:          0,
+		})
+	}
+
+	// Build and write manifest
+	return builder.Build(w)
+}
+
+// IsHiddenPath returns true if the path or any of its parent directories
+// start with a dot (hidden files/directories).
+// The path should use forward slashes.
+func IsHiddenPath(p string) bool {
+	tp := path.Clean(p)
+	if strings.HasPrefix(tp, ".") {
+		return true
+	}
+	for {
+		d, f := path.Split(tp)
+		if strings.HasPrefix(f, ".") {
+			return true
+		}
+		if d == "" {
+			return false
+		}
+		tp = d[0 : len(d)-1] // trim trailing slash from dir
+	}
+}
+
+// sendEnumerateStatus sends a status update without blocking.
+// If the channel is full, the update is dropped.
+func sendEnumerateStatus(ch chan<- EnumerateStatus, status EnumerateStatus) {
+	if ch == nil {
+		return
+	}
+	select {
+	case ch <- status:
+	default:
+		// Channel full, drop this update
+	}
+}
+
+// sendScanStatus sends a status update without blocking.
+// If the channel is full, the update is dropped.
+func sendScanStatus(ch chan<- ScanStatus, status ScanStatus) {
+	if ch == nil {
+		return
+	}
+	select {
+	case ch <- status:
+	default:
+		// Channel full, drop this update
+	}
+}
@@ -0,0 +1,362 @@
+package mfer
+
+import (
+	"bytes"
+	"context"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewScanner(t *testing.T) {
+	s := NewScanner()
+	assert.NotNil(t, s)
+	assert.Equal(t, FileCount(0), s.FileCount())
+	assert.Equal(t, FileSize(0), s.TotalBytes())
+}
+
+func TestNewScannerWithOptions(t *testing.T) {
+	t.Run("nil options", func(t *testing.T) {
+		s := NewScannerWithOptions(nil)
+		assert.NotNil(t, s)
+	})
+
+	t.Run("with options", func(t *testing.T) {
+		fs := afero.NewMemMapFs()
+		opts := &ScannerOptions{
+			IncludeDotfiles: true,
+			FollowSymLinks:  true,
+			Fs:              fs,
+		}
+		s := NewScannerWithOptions(opts)
+		assert.NotNil(t, s)
+	})
+}
+
+func TestScannerEnumerateFile(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("hello world"), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumerateFile("/test.txt")
+	require.NoError(t, err)
+
+	assert.Equal(t, FileCount(1), s.FileCount())
+	assert.Equal(t, FileSize(11), s.TotalBytes())
+
+	files := s.Files()
+	require.Len(t, files, 1)
+	assert.Equal(t, RelFilePath("test.txt"), files[0].Path)
+	assert.Equal(t, FileSize(11), files[0].Size)
+}
+
+func TestScannerEnumerateFileMissing(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumerateFile("/nonexistent.txt")
+	assert.Error(t, err)
+}
+
+func TestScannerEnumeratePath(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir/subdir", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("one"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("two"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/subdir/file3.txt", []byte("three"), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumeratePath("/testdir", nil)
+	require.NoError(t, err)
+
+	assert.Equal(t, FileCount(3), s.FileCount())
+	assert.Equal(t, FileSize(3+3+5), s.TotalBytes())
+}
+
+func TestScannerEnumeratePathWithProgress(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("one"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("two"), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	progress := make(chan EnumerateStatus, 10)
+
+	err := s.EnumeratePath("/testdir", progress)
+	require.NoError(t, err)
+
+	var updates []EnumerateStatus
+	for p := range progress {
+		updates = append(updates, p)
+	}
+
+	assert.NotEmpty(t, updates)
+	// Final update should show all files
+	final := updates[len(updates)-1]
+	assert.Equal(t, FileCount(2), final.FilesFound)
+	assert.Equal(t, FileSize(6), final.BytesFound)
+}
+
+func TestScannerEnumeratePaths(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/dir1", 0755))
+	require.NoError(t, fs.MkdirAll("/dir2", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/dir1/a.txt", []byte("aaa"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/dir2/b.txt", []byte("bbb"), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumeratePaths(nil, "/dir1", "/dir2")
+	require.NoError(t, err)
+
+	assert.Equal(t, FileCount(2), s.FileCount())
+}
+
+func TestScannerExcludeDotfiles(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir/.hidden", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/visible.txt", []byte("visible"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/.hidden.txt", []byte("hidden"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/.hidden/inside.txt", []byte("inside"), 0644))
+
+	t.Run("exclude by default", func(t *testing.T) {
+		s := NewScannerWithOptions(&ScannerOptions{Fs: fs, IncludeDotfiles: false})
+		err := s.EnumeratePath("/testdir", nil)
+		require.NoError(t, err)
+
+		assert.Equal(t, FileCount(1), s.FileCount())
+		files := s.Files()
+		assert.Equal(t, RelFilePath("visible.txt"), files[0].Path)
+	})
+
+	t.Run("include when enabled", func(t *testing.T) {
+		s := NewScannerWithOptions(&ScannerOptions{Fs: fs, IncludeDotfiles: true})
+		err := s.EnumeratePath("/testdir", nil)
+		require.NoError(t, err)
+
+		assert.Equal(t, FileCount(3), s.FileCount())
+	})
+}
+
+func TestScannerToManifest(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("content one"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("content two"), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumeratePath("/testdir", nil)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	err = s.ToManifest(context.Background(), &buf, nil)
+	require.NoError(t, err)
+
+	// Manifest should have magic bytes
+	assert.True(t, buf.Len() > 0)
+	assert.Equal(t, MAGIC, string(buf.Bytes()[:8]))
+}
+
+func TestScannerToManifestWithProgress(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file.txt", bytes.Repeat([]byte("x"), 1000), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumeratePath("/testdir", nil)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	progress := make(chan ScanStatus, 10)
+
+	err = s.ToManifest(context.Background(), &buf, progress)
+	require.NoError(t, err)
+
+	var updates []ScanStatus
+	for p := range progress {
+		updates = append(updates, p)
+	}
+
+	assert.NotEmpty(t, updates)
+	// Final update should show completion
+	final := updates[len(updates)-1]
+	assert.Equal(t, FileCount(1), final.TotalFiles)
+	assert.Equal(t, FileCount(1), final.ScannedFiles)
+	assert.Equal(t, FileSize(1000), final.TotalBytes)
+	assert.Equal(t, FileSize(1000), final.ScannedBytes)
+}
+
+func TestScannerToManifestContextCancellation(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir", 0755))
+	// Create many files to ensure we have time to cancel
+	for i := 0; i < 100; i++ {
+		name := string(rune('a'+i%26)) + string(rune('0'+i/26)) + ".txt"
+		require.NoError(t, afero.WriteFile(fs, "/testdir/"+name, bytes.Repeat([]byte("x"), 100), 0644))
+	}
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumeratePath("/testdir", nil)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // Cancel immediately
+
+	var buf bytes.Buffer
+	err = s.ToManifest(ctx, &buf, nil)
+	assert.ErrorIs(t, err, context.Canceled)
+}
+
+func TestScannerToManifestEmptyScanner(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+
+	var buf bytes.Buffer
+	err := s.ToManifest(context.Background(), &buf, nil)
+	require.NoError(t, err)
+
+	// Should still produce a valid manifest
+	assert.True(t, buf.Len() > 0)
+	assert.Equal(t, MAGIC, string(buf.Bytes()[:8]))
+}
+
+func TestScannerFilesCopiesSlice(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("hello"), 0644))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	require.NoError(t, s.EnumerateFile("/test.txt"))
+
+	files1 := s.Files()
+	files2 := s.Files()
+
+	// Should be different slices
+	assert.NotSame(t, &files1[0], &files2[0])
+}
+
+func TestScannerEnumerateFS(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir/sub", 0755))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/file.txt", []byte("hello"), 0644))
+	require.NoError(t, afero.WriteFile(fs, "/testdir/sub/nested.txt", []byte("world"), 0644))
+
+	// Create a basepath filesystem
+	baseFs := afero.NewBasePathFs(fs, "/testdir")
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	err := s.EnumerateFS(baseFs, "/testdir", nil)
+	require.NoError(t, err)
+
+	assert.Equal(t, FileCount(2), s.FileCount())
+}
+
+func TestSendEnumerateStatusNonBlocking(t *testing.T) {
+	// Channel with no buffer - send should not block
+	ch := make(chan EnumerateStatus)
+
+	// This should not block
+	done := make(chan bool)
+	go func() {
+		sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success - did not block
+	case <-time.After(100 * time.Millisecond):
+		t.Fatal("sendEnumerateStatus blocked on full channel")
+	}
+}
+
+func TestSendScanStatusNonBlocking(t *testing.T) {
+	// Channel with no buffer - send should not block
+	ch := make(chan ScanStatus)
+
+	done := make(chan bool)
+	go func() {
+		sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success - did not block
+	case <-time.After(100 * time.Millisecond):
+		t.Fatal("sendScanStatus blocked on full channel")
+	}
+}
+
+func TestSendStatusNilChannel(t *testing.T) {
+	// Should not panic with nil channel
+	sendEnumerateStatus(nil, EnumerateStatus{})
+	sendScanStatus(nil, ScanStatus{})
+}
+
+func TestScannerFileEntryFields(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	now := time.Now().Truncate(time.Second)
+	require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("content"), 0644))
+	require.NoError(t, fs.Chtimes("/test.txt", now, now))
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	require.NoError(t, s.EnumerateFile("/test.txt"))
+
+	files := s.Files()
+	require.Len(t, files, 1)
+
+	entry := files[0]
+	assert.Equal(t, RelFilePath("test.txt"), entry.Path)
+	assert.Contains(t, string(entry.AbsPath), "test.txt")
+	assert.Equal(t, FileSize(7), entry.Size)
+	// Mtime should be set (within a second of now)
+	assert.WithinDuration(t, now, time.Time(entry.Mtime), 2*time.Second)
+}
+
+func TestScannerLargeFileEnumeration(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	require.NoError(t, fs.MkdirAll("/testdir", 0755))
+
+	// Create 100 files
+	for i := 0; i < 100; i++ {
+		name := "/testdir/" + string(rune('a'+i%26)) + string(rune('0'+i/26%10)) + ".txt"
+		require.NoError(t, afero.WriteFile(fs, name, []byte("data"), 0644))
+	}
+
+	s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
+	progress := make(chan EnumerateStatus, 200)
+
+	err := s.EnumeratePath("/testdir", progress)
+	require.NoError(t, err)
+
+	// Drain channel
+	for range progress {
+	}
+
+	assert.Equal(t, FileCount(100), s.FileCount())
+	assert.Equal(t, FileSize(400), s.TotalBytes()) // 100 * 4 bytes
+}
+
+func TestIsHiddenPath(t *testing.T) {
+	tests := []struct {
+		path   string
+		hidden bool
+	}{
+		{"file.txt", false},
+		{".hidden", true},
+		{"dir/file.txt", false},
+		{"dir/.hidden", true},
+		{".dir/file.txt", true},
+		{"/absolute/path", false},
+		{"/absolute/.hidden", true},
+		{"./relative", false}, // path.Clean removes leading ./
+		{"a/b/c/.d/e", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.path, func(t *testing.T) {
+			assert.Equal(t, tt.hidden, IsHiddenPath(tt.path), "IsHiddenPath(%q)", tt.path)
+		})
+	}
+}
@@ -0,0 +1,53 @@
+package mfer
+
+import (
+	"net/url"
+	"strings"
+)
+
+// ManifestURL represents a URL pointing to a manifest file.
+type ManifestURL string
+
+// FileURL represents a URL pointing to a file to be fetched.
+type FileURL string
+
+// BaseURL represents a base URL for constructing file URLs.
+type BaseURL string
+
+// JoinPath safely joins a relative file path to a base URL.
+// The path is properly URL-encoded to prevent path traversal.
+func (b BaseURL) JoinPath(path RelFilePath) (FileURL, error) {
+	base, err := url.Parse(string(b))
+	if err != nil {
+		return "", err
+	}
+
+	// Ensure base path ends with /
+	if !strings.HasSuffix(base.Path, "/") {
+		base.Path += "/"
+	}
+
+	// Parse and encode the relative path
+	ref, err := url.Parse(url.PathEscape(string(path)))
+	if err != nil {
+		return "", err
+	}
+
+	resolved := base.ResolveReference(ref)
+	return FileURL(resolved.String()), nil
+}
+
+// String returns the URL as a string.
+func (b BaseURL) String() string {
+	return string(b)
+}
+
+// String returns the URL as a string.
+func (f FileURL) String() string {
+	return string(f)
+}
+
+// String returns the URL as a string.
+func (m ManifestURL) String() string {
+	return string(m)
+}