Add testable CLI with dependency injection and new scanner/checker packages

Major changes:
- Refactor CLI to accept injected I/O streams and filesystem (afero.Fs)
  for testing without touching the real filesystem
- Add RunOptions struct and RunWithOptions() for configurable CLI execution
- Add internal/scanner package with two-phase manifest generation:
  - Phase 1 (Enumeration): walk directories, collect metadata
  - Phase 2 (Scan): read contents, compute hashes, write manifest
- Add internal/checker package for manifest verification with progress
  reporting and channel-based result streaming
- Add mfer/builder.go for incremental manifest construction
- Add --no-extra-files flag to check command to detect files not in manifest
- Add timing summaries showing file count, size, elapsed time, and throughput
- Add comprehensive tests using afero.MemMapFs (no real filesystem access)
- Add contrib/usage.sh integration test script
- Fix banner ASCII art alignment (consistent spacing)
- Fix verbosity levels so summaries display at default log level
- Update internal/log to support configurable output writers
This commit is contained in:
2025-12-17 11:00:55 -08:00
parent 13f39d598f
commit dc2ea47f6a
15 changed files with 1744 additions and 81 deletions

View File

@@ -6,12 +6,13 @@ import (
"errors"
"io"
"github.com/spf13/afero"
"google.golang.org/protobuf/proto"
"sneak.berlin/go/mfer/internal/bork"
"sneak.berlin/go/mfer/internal/log"
)
func (m *manifest) validateProtoOuter() error {
func (m *manifest) deserializeInner() error {
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
return errors.New("unknown version")
}
@@ -25,10 +26,9 @@ func (m *manifest) validateProtoOuter() error {
if err != nil {
return err
}
dat, err := io.ReadAll(gzr)
defer gzr.Close()
dat, err := io.ReadAll(gzr)
if err != nil {
return err
}
@@ -38,9 +38,14 @@ func (m *manifest) validateProtoOuter() error {
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
return bork.ErrFileTruncated
}
log.Debugf("inner data size is %d", isize)
log.Dump(dat)
log.Dump(m.pbOuter.Sha256)
// Deserialize inner message
m.pbInner = new(MFFile)
if err := proto.Unmarshal(dat, m.pbInner); err != nil {
return err
}
log.Debugf("loaded manifest with %d files", len(m.pbInner.Files))
return nil
}
@@ -54,7 +59,8 @@ func validateMagic(dat []byte) bool {
return bytes.Equal(got, expected)
}
func NewFromProto(input io.Reader) (*manifest, error) {
// NewManifestFromReader reads a manifest from an io.Reader.
func NewManifestFromReader(input io.Reader) (*manifest, error) {
m := New()
dat, err := io.ReadAll(input)
if err != nil {
@@ -69,21 +75,35 @@ func NewFromProto(input io.Reader) (*manifest, error) {
bb := bytes.NewBuffer(dat[ml:])
dat = bb.Bytes()
log.Dump(dat)
// deserialize:
// deserialize outer:
m.pbOuter = new(MFFileOuter)
err = proto.Unmarshal(dat, m.pbOuter)
if err != nil {
if err := proto.Unmarshal(dat, m.pbOuter); err != nil {
return nil, err
}
ve := m.validateProtoOuter()
if ve != nil {
return nil, ve
// deserialize inner:
if err := m.deserializeInner(); err != nil {
return nil, err
}
// FIXME TODO deserialize inner
return m, nil
}
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
// If fs is nil, the real filesystem (OsFs) is used.
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
if fs == nil {
fs = afero.NewOsFs()
}
f, err := fs.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
return NewManifestFromReader(f)
}
// NewFromProto is deprecated, use NewManifestFromReader instead.
func NewFromProto(input io.Reader) (*manifest, error) {
return NewManifestFromReader(input)
}