forked from sneak/mfer
sneak
dc2ea47f6a
Major changes: - Refactor CLI to accept injected I/O streams and filesystem (afero.Fs) for testing without touching the real filesystem - Add RunOptions struct and RunWithOptions() for configurable CLI execution - Add internal/scanner package with two-phase manifest generation: - Phase 1 (Enumeration): walk directories, collect metadata - Phase 2 (Scan): read contents, compute hashes, write manifest - Add internal/checker package for manifest verification with progress reporting and channel-based result streaming - Add mfer/builder.go for incremental manifest construction - Add --no-extra-files flag to check command to detect files not in manifest - Add timing summaries showing file count, size, elapsed time, and throughput - Add comprehensive tests using afero.MemMapFs (no real filesystem access) - Add contrib/usage.sh integration test script - Fix banner ASCII art alignment (consistent spacing) - Fix verbosity levels so summaries display at default log level - Update internal/log to support configurable output writers
125 lines
2.5 KiB
Go
125 lines
2.5 KiB
Go
package mfer
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"io"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/multiformats/go-multihash"
|
|
)
|
|
|
|
// FileProgress is called during file processing to report bytes read.
|
|
type FileProgress func(bytesRead int64)
|
|
|
|
// ManifestBuilder constructs a manifest by adding files one at a time.
|
|
type ManifestBuilder struct {
|
|
mu sync.Mutex
|
|
files []*MFFilePath
|
|
createdAt time.Time
|
|
}
|
|
|
|
// NewBuilder creates a new ManifestBuilder.
|
|
func NewBuilder() *ManifestBuilder {
|
|
return &ManifestBuilder{
|
|
files: make([]*MFFilePath, 0),
|
|
createdAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
|
// The progress callback is called periodically with total bytes read so far.
|
|
// Returns the number of bytes read.
|
|
func (b *ManifestBuilder) AddFile(
|
|
path string,
|
|
size int64,
|
|
mtime time.Time,
|
|
reader io.Reader,
|
|
progress FileProgress,
|
|
) (int64, error) {
|
|
// Create hash writer
|
|
h := sha256.New()
|
|
|
|
// Read file in chunks, updating hash and progress
|
|
var totalRead int64
|
|
buf := make([]byte, 64*1024) // 64KB chunks
|
|
|
|
for {
|
|
n, err := reader.Read(buf)
|
|
if n > 0 {
|
|
h.Write(buf[:n])
|
|
totalRead += int64(n)
|
|
if progress != nil {
|
|
progress(totalRead)
|
|
}
|
|
}
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return totalRead, err
|
|
}
|
|
}
|
|
|
|
// Encode hash as multihash (SHA2-256)
|
|
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
|
if err != nil {
|
|
return totalRead, err
|
|
}
|
|
|
|
// Create file entry
|
|
entry := &MFFilePath{
|
|
Path: path,
|
|
Size: size,
|
|
Hashes: []*MFFileChecksum{
|
|
{MultiHash: mh},
|
|
},
|
|
Mtime: newTimestampFromTime(mtime),
|
|
}
|
|
|
|
b.mu.Lock()
|
|
b.files = append(b.files, entry)
|
|
b.mu.Unlock()
|
|
|
|
return totalRead, nil
|
|
}
|
|
|
|
// FileCount returns the number of files added to the builder.
|
|
func (b *ManifestBuilder) FileCount() int {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return len(b.files)
|
|
}
|
|
|
|
// Build finalizes the manifest and writes it to the writer.
|
|
func (b *ManifestBuilder) Build(w io.Writer) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
// Create inner manifest
|
|
inner := &MFFile{
|
|
Version: MFFile_VERSION_ONE,
|
|
CreatedAt: newTimestampFromTime(b.createdAt),
|
|
Files: b.files,
|
|
}
|
|
|
|
// Create a temporary manifest to use existing serialization
|
|
m := &manifest{
|
|
pbInner: inner,
|
|
}
|
|
|
|
// Generate outer wrapper
|
|
if err := m.generateOuter(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Generate final output
|
|
if err := m.generate(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write to output
|
|
_, err := w.Write(m.output.Bytes())
|
|
return err
|
|
}
|