forked from sneak/mfer
sneak
1588e1bb9f
Removed: - New(), NewFromPaths(), NewFromFS() - unused constructors - Scan(), addFile(), addInputPath(), addInputFS() - unused scanning code - WriteToFile(), Write() - unused output methods (Builder.Build() is used) - GetFileCount(), GetTotalFileSize() - unused accessors - pathIsHidden() - duplicated in internal/scanner - ManifestScanOptions - unused options struct - HasError(), AddError(), WithContext() - unused error/context handling - NewFromProto() - deprecated alias - manifestFile struct - unused internal type Kept: - manifest struct (simplified to just pbInner, pbOuter, output) - NewManifestFromReader(), NewManifestFromFile() - for loading manifests - Files() - returns files from loaded manifest - Builder and its methods - for creating manifests
105 lines
2.2 KiB
Go
105 lines
2.2 KiB
Go
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/spf13/afero"
|
|
"google.golang.org/protobuf/proto"
|
|
"sneak.berlin/go/mfer/internal/bork"
|
|
"sneak.berlin/go/mfer/internal/log"
|
|
)
|
|
|
|
func (m *manifest) deserializeInner() error {
|
|
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
|
|
return errors.New("unknown version")
|
|
}
|
|
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_ZSTD {
|
|
return errors.New("unknown compression type")
|
|
}
|
|
|
|
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
|
|
|
|
zr, err := zstd.NewReader(bb)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zr.Close()
|
|
|
|
dat, err := io.ReadAll(zr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
isize := len(dat)
|
|
if int64(isize) != m.pbOuter.Size {
|
|
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
|
|
return bork.ErrFileTruncated
|
|
}
|
|
|
|
// Deserialize inner message
|
|
m.pbInner = new(MFFile)
|
|
if err := proto.Unmarshal(dat, m.pbInner); err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Infof("loaded manifest with %d files", len(m.pbInner.Files))
|
|
return nil
|
|
}
|
|
|
|
func validateMagic(dat []byte) bool {
|
|
ml := len([]byte(MAGIC))
|
|
if len(dat) < ml {
|
|
return false
|
|
}
|
|
got := dat[0:ml]
|
|
expected := []byte(MAGIC)
|
|
return bytes.Equal(got, expected)
|
|
}
|
|
|
|
// NewManifestFromReader reads a manifest from an io.Reader.
|
|
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|
m := &manifest{}
|
|
dat, err := io.ReadAll(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !validateMagic(dat) {
|
|
return nil, errors.New("invalid file format")
|
|
}
|
|
|
|
// remove magic bytes prefix:
|
|
ml := len([]byte(MAGIC))
|
|
bb := bytes.NewBuffer(dat[ml:])
|
|
dat = bb.Bytes()
|
|
|
|
// deserialize outer:
|
|
m.pbOuter = new(MFFileOuter)
|
|
if err := proto.Unmarshal(dat, m.pbOuter); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// deserialize inner:
|
|
if err := m.deserializeInner(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return m, nil
|
|
}
|
|
|
|
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
|
// If fs is nil, the real filesystem (OsFs) is used.
|
|
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
|
if fs == nil {
|
|
fs = afero.NewOsFs()
|
|
}
|
|
f, err := fs.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
return NewManifestFromReader(f)
|
|
}
|