Add testable CLI with dependency injection and new scanner/checker packages
Major changes: - Refactor CLI to accept injected I/O streams and filesystem (afero.Fs) for testing without touching the real filesystem - Add RunOptions struct and RunWithOptions() for configurable CLI execution - Add internal/scanner package with two-phase manifest generation: - Phase 1 (Enumeration): walk directories, collect metadata - Phase 2 (Scan): read contents, compute hashes, write manifest - Add internal/checker package for manifest verification with progress reporting and channel-based result streaming - Add mfer/builder.go for incremental manifest construction - Add --no-extra-files flag to check command to detect files not in manifest - Add timing summaries showing file count, size, elapsed time, and throughput - Add comprehensive tests using afero.MemMapFs (no real filesystem access) - Add contrib/usage.sh integration test script - Fix banner ASCII art alignment (consistent spacing) - Fix verbosity levels so summaries display at default log level - Update internal/log to support configurable output writers
This commit is contained in:
@@ -6,12 +6,13 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"sneak.berlin/go/mfer/internal/bork"
|
||||
"sneak.berlin/go/mfer/internal/log"
|
||||
)
|
||||
|
||||
func (m *manifest) validateProtoOuter() error {
|
||||
func (m *manifest) deserializeInner() error {
|
||||
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
|
||||
return errors.New("unknown version")
|
||||
}
|
||||
@@ -25,10 +26,9 @@ func (m *manifest) validateProtoOuter() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dat, err := io.ReadAll(gzr)
|
||||
defer gzr.Close()
|
||||
|
||||
dat, err := io.ReadAll(gzr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -38,9 +38,14 @@ func (m *manifest) validateProtoOuter() error {
|
||||
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
|
||||
return bork.ErrFileTruncated
|
||||
}
|
||||
log.Debugf("inner data size is %d", isize)
|
||||
log.Dump(dat)
|
||||
log.Dump(m.pbOuter.Sha256)
|
||||
|
||||
// Deserialize inner message
|
||||
m.pbInner = new(MFFile)
|
||||
if err := proto.Unmarshal(dat, m.pbInner); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("loaded manifest with %d files", len(m.pbInner.Files))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -54,7 +59,8 @@ func validateMagic(dat []byte) bool {
|
||||
return bytes.Equal(got, expected)
|
||||
}
|
||||
|
||||
func NewFromProto(input io.Reader) (*manifest, error) {
|
||||
// NewManifestFromReader reads a manifest from an io.Reader.
|
||||
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
||||
m := New()
|
||||
dat, err := io.ReadAll(input)
|
||||
if err != nil {
|
||||
@@ -69,21 +75,35 @@ func NewFromProto(input io.Reader) (*manifest, error) {
|
||||
bb := bytes.NewBuffer(dat[ml:])
|
||||
dat = bb.Bytes()
|
||||
|
||||
log.Dump(dat)
|
||||
|
||||
// deserialize:
|
||||
// deserialize outer:
|
||||
m.pbOuter = new(MFFileOuter)
|
||||
err = proto.Unmarshal(dat, m.pbOuter)
|
||||
|
||||
if err != nil {
|
||||
if err := proto.Unmarshal(dat, m.pbOuter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ve := m.validateProtoOuter()
|
||||
if ve != nil {
|
||||
return nil, ve
|
||||
// deserialize inner:
|
||||
if err := m.deserializeInner(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// FIXME TODO deserialize inner
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
||||
// If fs is nil, the real filesystem (OsFs) is used.
|
||||
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
||||
if fs == nil {
|
||||
fs = afero.NewOsFs()
|
||||
}
|
||||
f, err := fs.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return NewManifestFromReader(f)
|
||||
}
|
||||
|
||||
// NewFromProto is deprecated, use NewManifestFromReader instead.
|
||||
func NewFromProto(input io.Reader) (*manifest, error) {
|
||||
return NewManifestFromReader(input)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user