forked from sneak/mfer
sneak
dc2ea47f6a
Major changes: - Refactor CLI to accept injected I/O streams and filesystem (afero.Fs) for testing without touching the real filesystem - Add RunOptions struct and RunWithOptions() for configurable CLI execution - Add internal/scanner package with two-phase manifest generation: - Phase 1 (Enumeration): walk directories, collect metadata - Phase 2 (Scan): read contents, compute hashes, write manifest - Add internal/checker package for manifest verification with progress reporting and channel-based result streaming - Add mfer/builder.go for incremental manifest construction - Add --no-extra-files flag to check command to detect files not in manifest - Add timing summaries showing file count, size, elapsed time, and throughput - Add comprehensive tests using afero.MemMapFs (no real filesystem access) - Add contrib/usage.sh integration test script - Fix banner ASCII art alignment (consistent spacing) - Fix verbosity levels so summaries display at default log level - Update internal/log to support configurable output writers
110 lines
2.3 KiB
Go
110 lines
2.3 KiB
Go
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"errors"
|
|
"io"
|
|
|
|
"github.com/spf13/afero"
|
|
"google.golang.org/protobuf/proto"
|
|
"sneak.berlin/go/mfer/internal/bork"
|
|
"sneak.berlin/go/mfer/internal/log"
|
|
)
|
|
|
|
func (m *manifest) deserializeInner() error {
|
|
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
|
|
return errors.New("unknown version")
|
|
}
|
|
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
|
|
return errors.New("unknown compression type")
|
|
}
|
|
|
|
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
|
|
|
|
gzr, err := gzip.NewReader(bb)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gzr.Close()
|
|
|
|
dat, err := io.ReadAll(gzr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
isize := len(dat)
|
|
if int64(isize) != m.pbOuter.Size {
|
|
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
|
|
return bork.ErrFileTruncated
|
|
}
|
|
|
|
// Deserialize inner message
|
|
m.pbInner = new(MFFile)
|
|
if err := proto.Unmarshal(dat, m.pbInner); err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Debugf("loaded manifest with %d files", len(m.pbInner.Files))
|
|
return nil
|
|
}
|
|
|
|
func validateMagic(dat []byte) bool {
|
|
ml := len([]byte(MAGIC))
|
|
if len(dat) < ml {
|
|
return false
|
|
}
|
|
got := dat[0:ml]
|
|
expected := []byte(MAGIC)
|
|
return bytes.Equal(got, expected)
|
|
}
|
|
|
|
// NewManifestFromReader reads a manifest from an io.Reader.
|
|
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|
m := New()
|
|
dat, err := io.ReadAll(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !validateMagic(dat) {
|
|
return nil, errors.New("invalid file format")
|
|
}
|
|
|
|
// remove magic bytes prefix:
|
|
ml := len([]byte(MAGIC))
|
|
bb := bytes.NewBuffer(dat[ml:])
|
|
dat = bb.Bytes()
|
|
|
|
// deserialize outer:
|
|
m.pbOuter = new(MFFileOuter)
|
|
if err := proto.Unmarshal(dat, m.pbOuter); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// deserialize inner:
|
|
if err := m.deserializeInner(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return m, nil
|
|
}
|
|
|
|
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
|
// If fs is nil, the real filesystem (OsFs) is used.
|
|
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
|
if fs == nil {
|
|
fs = afero.NewOsFs()
|
|
}
|
|
f, err := fs.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
return NewManifestFromReader(f)
|
|
}
|
|
|
|
// NewFromProto is deprecated, use NewManifestFromReader instead.
|
|
func NewFromProto(input io.Reader) (*manifest, error) {
|
|
return NewManifestFromReader(input)
|
|
}
|