sneak
21028af9aa
Use zstd with SpeedBestCompression level for better compression ratios. Remove gzip support entirely. Include generated protobuf file to allow building without protoc.
110 lines
2.4 KiB
Go
110 lines
2.4 KiB
Go
package mfer
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/spf13/afero"
|
|
"google.golang.org/protobuf/proto"
|
|
"sneak.berlin/go/mfer/internal/bork"
|
|
"sneak.berlin/go/mfer/internal/log"
|
|
)
|
|
|
|
func (m *manifest) deserializeInner() error {
|
|
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
|
|
return errors.New("unknown version")
|
|
}
|
|
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_ZSTD {
|
|
return errors.New("unknown compression type")
|
|
}
|
|
|
|
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
|
|
|
|
zr, err := zstd.NewReader(bb)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zr.Close()
|
|
|
|
dat, err := io.ReadAll(zr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
isize := len(dat)
|
|
if int64(isize) != m.pbOuter.Size {
|
|
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
|
|
return bork.ErrFileTruncated
|
|
}
|
|
|
|
// Deserialize inner message
|
|
m.pbInner = new(MFFile)
|
|
if err := proto.Unmarshal(dat, m.pbInner); err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Debugf("loaded manifest with %d files", len(m.pbInner.Files))
|
|
return nil
|
|
}
|
|
|
|
func validateMagic(dat []byte) bool {
|
|
ml := len([]byte(MAGIC))
|
|
if len(dat) < ml {
|
|
return false
|
|
}
|
|
got := dat[0:ml]
|
|
expected := []byte(MAGIC)
|
|
return bytes.Equal(got, expected)
|
|
}
|
|
|
|
// NewManifestFromReader reads a manifest from an io.Reader.
|
|
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|
m := New()
|
|
dat, err := io.ReadAll(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !validateMagic(dat) {
|
|
return nil, errors.New("invalid file format")
|
|
}
|
|
|
|
// remove magic bytes prefix:
|
|
ml := len([]byte(MAGIC))
|
|
bb := bytes.NewBuffer(dat[ml:])
|
|
dat = bb.Bytes()
|
|
|
|
// deserialize outer:
|
|
m.pbOuter = new(MFFileOuter)
|
|
if err := proto.Unmarshal(dat, m.pbOuter); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// deserialize inner:
|
|
if err := m.deserializeInner(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return m, nil
|
|
}
|
|
|
|
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
|
// If fs is nil, the real filesystem (OsFs) is used.
|
|
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
|
if fs == nil {
|
|
fs = afero.NewOsFs()
|
|
}
|
|
f, err := fs.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
return NewManifestFromReader(f)
|
|
}
|
|
|
|
// NewFromProto is deprecated, use NewManifestFromReader instead.
|
|
func NewFromProto(input io.Reader) (*manifest, error) {
|
|
return NewManifestFromReader(input)
|
|
}
|