Add testable CLI with dependency injection and new scanner/checker packages
Major changes: - Refactor CLI to accept injected I/O streams and filesystem (afero.Fs) for testing without touching the real filesystem - Add RunOptions struct and RunWithOptions() for configurable CLI execution - Add internal/scanner package with two-phase manifest generation: - Phase 1 (Enumeration): walk directories, collect metadata - Phase 2 (Scan): read contents, compute hashes, write manifest - Add internal/checker package for manifest verification with progress reporting and channel-based result streaming - Add mfer/builder.go for incremental manifest construction - Add --no-extra-files flag to check command to detect files not in manifest - Add timing summaries showing file count, size, elapsed time, and throughput - Add comprehensive tests using afero.MemMapFs (no real filesystem access) - Add contrib/usage.sh integration test script - Fix banner ASCII art alignment (consistent spacing) - Fix verbosity levels so summaries display at default log level - Update internal/log to support configurable output writers
This commit is contained in:
+124
@@ -0,0 +1,124 @@
|
||||
package mfer
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
// FileProgress is called during file processing to report bytes read.
|
||||
type FileProgress func(bytesRead int64)
|
||||
|
||||
// ManifestBuilder constructs a manifest by adding files one at a time.
|
||||
type ManifestBuilder struct {
|
||||
mu sync.Mutex
|
||||
files []*MFFilePath
|
||||
createdAt time.Time
|
||||
}
|
||||
|
||||
// NewBuilder creates a new ManifestBuilder.
|
||||
func NewBuilder() *ManifestBuilder {
|
||||
return &ManifestBuilder{
|
||||
files: make([]*MFFilePath, 0),
|
||||
createdAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
||||
// The progress callback is called periodically with total bytes read so far.
|
||||
// Returns the number of bytes read.
|
||||
func (b *ManifestBuilder) AddFile(
|
||||
path string,
|
||||
size int64,
|
||||
mtime time.Time,
|
||||
reader io.Reader,
|
||||
progress FileProgress,
|
||||
) (int64, error) {
|
||||
// Create hash writer
|
||||
h := sha256.New()
|
||||
|
||||
// Read file in chunks, updating hash and progress
|
||||
var totalRead int64
|
||||
buf := make([]byte, 64*1024) // 64KB chunks
|
||||
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
h.Write(buf[:n])
|
||||
totalRead += int64(n)
|
||||
if progress != nil {
|
||||
progress(totalRead)
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return totalRead, err
|
||||
}
|
||||
}
|
||||
|
||||
// Encode hash as multihash (SHA2-256)
|
||||
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
||||
if err != nil {
|
||||
return totalRead, err
|
||||
}
|
||||
|
||||
// Create file entry
|
||||
entry := &MFFilePath{
|
||||
Path: path,
|
||||
Size: size,
|
||||
Hashes: []*MFFileChecksum{
|
||||
{MultiHash: mh},
|
||||
},
|
||||
Mtime: newTimestampFromTime(mtime),
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
b.files = append(b.files, entry)
|
||||
b.mu.Unlock()
|
||||
|
||||
return totalRead, nil
|
||||
}
|
||||
|
||||
// FileCount returns the number of files added to the builder.
|
||||
func (b *ManifestBuilder) FileCount() int {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return len(b.files)
|
||||
}
|
||||
|
||||
// Build finalizes the manifest and writes it to the writer.
|
||||
func (b *ManifestBuilder) Build(w io.Writer) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
// Create inner manifest
|
||||
inner := &MFFile{
|
||||
Version: MFFile_VERSION_ONE,
|
||||
CreatedAt: newTimestampFromTime(b.createdAt),
|
||||
Files: b.files,
|
||||
}
|
||||
|
||||
// Create a temporary manifest to use existing serialization
|
||||
m := &manifest{
|
||||
pbInner: inner,
|
||||
}
|
||||
|
||||
// Generate outer wrapper
|
||||
if err := m.generateOuter(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Generate final output
|
||||
if err := m.generate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write to output
|
||||
_, err := w.Write(m.output.Bytes())
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user