sneak/mfer
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Releases Wiki Activity

Replace gzip with zstd compression

Browse Source 
Use zstd with SpeedBestCompression level for better compression
ratios. Remove gzip support entirely. Include generated protobuf
file to allow building without protoc.
This commit is contained in:
Jeffrey Paul
2025-12-17 14:49:30 -08:00
parent 150bac82cf
commit 21028af9aa
8 changed files with 662 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
mfer/deserialize.go
View File

@@ -2,10 +2,10 @@ package mfer
import (
"bytes"
"compress/gzip"
"errors"
"io"
"github.com/klauspost/compress/zstd"
"github.com/spf13/afero"
"google.golang.org/protobuf/proto"
"sneak.berlin/go/mfer/internal/bork"
@@ -16,19 +16,19 @@ func (m *manifest) deserializeInner() error {
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
return errors.New("unknown version")
}
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_ZSTD {
return errors.New("unknown compression type")
}
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
gzr, err := gzip.NewReader(bb)
zr, err := zstd.NewReader(bb)
if err != nil {
return err
}
defer func() { _ = gzr.Close() }()
defer zr.Close()
dat, err := io.ReadAll(gzr)
dat, err := io.ReadAll(zr)
if err != nil {
return err
}

646
mfer/mf.pb.go Normal file
View File

@@ -0,0 +1,646 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc v6.33.0
// source: mf.proto
package mfer
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type MFFileOuter_Version int32
const (
MFFileOuter_VERSION_NONE MFFileOuter_Version = 0
MFFileOuter_VERSION_ONE MFFileOuter_Version = 1 // only one for now
)
// Enum value maps for MFFileOuter_Version.
var (
MFFileOuter_Version_name = map[int32]string{
0: "VERSION_NONE",
1: "VERSION_ONE",
}
MFFileOuter_Version_value = map[string]int32{
"VERSION_NONE": 0,
"VERSION_ONE": 1,
}
)
func (x MFFileOuter_Version) Enum() *MFFileOuter_Version {
p := new(MFFileOuter_Version)
*p = x
return p
}
func (x MFFileOuter_Version) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (MFFileOuter_Version) Descriptor() protoreflect.EnumDescriptor {
return file_mf_proto_enumTypes[0].Descriptor()
}
func (MFFileOuter_Version) Type() protoreflect.EnumType {
return &file_mf_proto_enumTypes[0]
}
func (x MFFileOuter_Version) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use MFFileOuter_Version.Descriptor instead.
func (MFFileOuter_Version) EnumDescriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{1, 0}
}
type MFFileOuter_CompressionType int32
const (
MFFileOuter_COMPRESSION_NONE MFFileOuter_CompressionType = 0
MFFileOuter_COMPRESSION_ZSTD MFFileOuter_CompressionType = 1
)
// Enum value maps for MFFileOuter_CompressionType.
var (
MFFileOuter_CompressionType_name = map[int32]string{
0: "COMPRESSION_NONE",
1: "COMPRESSION_ZSTD",
}
MFFileOuter_CompressionType_value = map[string]int32{
"COMPRESSION_NONE": 0,
"COMPRESSION_ZSTD": 1,
}
)
func (x MFFileOuter_CompressionType) Enum() *MFFileOuter_CompressionType {
p := new(MFFileOuter_CompressionType)
*p = x
return p
}
func (x MFFileOuter_CompressionType) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (MFFileOuter_CompressionType) Descriptor() protoreflect.EnumDescriptor {
return file_mf_proto_enumTypes[1].Descriptor()
}
func (MFFileOuter_CompressionType) Type() protoreflect.EnumType {
return &file_mf_proto_enumTypes[1]
}
func (x MFFileOuter_CompressionType) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use MFFileOuter_CompressionType.Descriptor instead.
func (MFFileOuter_CompressionType) EnumDescriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{1, 1}
}
type MFFile_Version int32
const (
MFFile_VERSION_NONE MFFile_Version = 0
MFFile_VERSION_ONE MFFile_Version = 1 // only one for now
)
// Enum value maps for MFFile_Version.
var (
MFFile_Version_name = map[int32]string{
0: "VERSION_NONE",
1: "VERSION_ONE",
}
MFFile_Version_value = map[string]int32{
"VERSION_NONE": 0,
"VERSION_ONE": 1,
}
)
func (x MFFile_Version) Enum() *MFFile_Version {
p := new(MFFile_Version)
*p = x
return p
}
func (x MFFile_Version) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (MFFile_Version) Descriptor() protoreflect.EnumDescriptor {
return file_mf_proto_enumTypes[2].Descriptor()
}
func (MFFile_Version) Type() protoreflect.EnumType {
return &file_mf_proto_enumTypes[2]
}
func (x MFFile_Version) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use MFFile_Version.Descriptor instead.
func (MFFile_Version) EnumDescriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{4, 0}
}
type Timestamp struct {
state protoimpl.MessageState `protogen:"open.v1"`
Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"`
Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Timestamp) Reset() {
*x = Timestamp{}
mi := &file_mf_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *Timestamp) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Timestamp) ProtoMessage() {}
func (x *Timestamp) ProtoReflect() protoreflect.Message {
mi := &file_mf_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Timestamp.ProtoReflect.Descriptor instead.
func (*Timestamp) Descriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{0}
}
func (x *Timestamp) GetSeconds() int64 {
if x != nil {
return x.Seconds
}
return 0
}
func (x *Timestamp) GetNanos() int32 {
if x != nil {
return x.Nanos
}
return 0
}
type MFFileOuter struct {
state protoimpl.MessageState `protogen:"open.v1"`
// required mffile root attributes 1xx
Version MFFileOuter_Version `protobuf:"varint,101,opt,name=version,proto3,enum=MFFileOuter_Version" json:"version,omitempty"`
CompressionType MFFileOuter_CompressionType `protobuf:"varint,102,opt,name=compressionType,proto3,enum=MFFileOuter_CompressionType" json:"compressionType,omitempty"`
// these are used solely to detect corruption/truncation
// and not for cryptographic integrity.
Size int64 `protobuf:"varint,103,opt,name=size,proto3" json:"size,omitempty"`
Sha256 []byte `protobuf:"bytes,104,opt,name=sha256,proto3" json:"sha256,omitempty"`
InnerMessage []byte `protobuf:"bytes,199,opt,name=innerMessage,proto3" json:"innerMessage,omitempty"`
// detached signature, ascii or binary
Signature []byte `protobuf:"bytes,201,opt,name=signature,proto3,oneof" json:"signature,omitempty"`
// full GPG key id
Signer []byte `protobuf:"bytes,202,opt,name=signer,proto3,oneof" json:"signer,omitempty"`
// full GPG signing public key, ascii or binary
SigningPubKey []byte `protobuf:"bytes,203,opt,name=signingPubKey,proto3,oneof" json:"signingPubKey,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *MFFileOuter) Reset() {
*x = MFFileOuter{}
mi := &file_mf_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *MFFileOuter) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MFFileOuter) ProtoMessage() {}
func (x *MFFileOuter) ProtoReflect() protoreflect.Message {
mi := &file_mf_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MFFileOuter.ProtoReflect.Descriptor instead.
func (*MFFileOuter) Descriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{1}
}
func (x *MFFileOuter) GetVersion() MFFileOuter_Version {
if x != nil {
return x.Version
}
return MFFileOuter_VERSION_NONE
}
func (x *MFFileOuter) GetCompressionType() MFFileOuter_CompressionType {
if x != nil {
return x.CompressionType
}
return MFFileOuter_COMPRESSION_NONE
}
func (x *MFFileOuter) GetSize() int64 {
if x != nil {
return x.Size
}
return 0
}
func (x *MFFileOuter) GetSha256() []byte {
if x != nil {
return x.Sha256
}
return nil
}
func (x *MFFileOuter) GetInnerMessage() []byte {
if x != nil {
return x.InnerMessage
}
return nil
}
func (x *MFFileOuter) GetSignature() []byte {
if x != nil {
return x.Signature
}
return nil
}
func (x *MFFileOuter) GetSigner() []byte {
if x != nil {
return x.Signer
}
return nil
}
func (x *MFFileOuter) GetSigningPubKey() []byte {
if x != nil {
return x.SigningPubKey
}
return nil
}
type MFFilePath struct {
state protoimpl.MessageState `protogen:"open.v1"`
// required attributes:
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
// gotta have at least one:
Hashes []*MFFileChecksum `protobuf:"bytes,3,rep,name=hashes,proto3" json:"hashes,omitempty"`
// optional per-file metadata
MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"`
Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"`
Atime *Timestamp `protobuf:"bytes,304,opt,name=atime,proto3,oneof" json:"atime,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *MFFilePath) Reset() {
*x = MFFilePath{}
mi := &file_mf_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *MFFilePath) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MFFilePath) ProtoMessage() {}
func (x *MFFilePath) ProtoReflect() protoreflect.Message {
mi := &file_mf_proto_msgTypes[2]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MFFilePath.ProtoReflect.Descriptor instead.
func (*MFFilePath) Descriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{2}
}
func (x *MFFilePath) GetPath() string {
if x != nil {
return x.Path
}
return ""
}
func (x *MFFilePath) GetSize() int64 {
if x != nil {
return x.Size
}
return 0
}
func (x *MFFilePath) GetHashes() []*MFFileChecksum {
if x != nil {
return x.Hashes
}
return nil
}
func (x *MFFilePath) GetMimeType() string {
if x != nil && x.MimeType != nil {
return *x.MimeType
}
return ""
}
func (x *MFFilePath) GetMtime() *Timestamp {
if x != nil {
return x.Mtime
}
return nil
}
func (x *MFFilePath) GetCtime() *Timestamp {
if x != nil {
return x.Ctime
}
return nil
}
func (x *MFFilePath) GetAtime() *Timestamp {
if x != nil {
return x.Atime
}
return nil
}
type MFFileChecksum struct {
state protoimpl.MessageState `protogen:"open.v1"`
// 1.0 golang implementation must write a multihash here
// it's ok to only ever use/verify sha256 multihash
MultiHash []byte `protobuf:"bytes,1,opt,name=multiHash,proto3" json:"multiHash,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *MFFileChecksum) Reset() {
*x = MFFileChecksum{}
mi := &file_mf_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *MFFileChecksum) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MFFileChecksum) ProtoMessage() {}
func (x *MFFileChecksum) ProtoReflect() protoreflect.Message {
mi := &file_mf_proto_msgTypes[3]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MFFileChecksum.ProtoReflect.Descriptor instead.
func (*MFFileChecksum) Descriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{3}
}
func (x *MFFileChecksum) GetMultiHash() []byte {
if x != nil {
return x.MultiHash
}
return nil
}
type MFFile struct {
state protoimpl.MessageState `protogen:"open.v1"`
Version MFFile_Version `protobuf:"varint,100,opt,name=version,proto3,enum=MFFile_Version" json:"version,omitempty"`
// required manifest attributes:
Files []*MFFilePath `protobuf:"bytes,101,rep,name=files,proto3" json:"files,omitempty"`
// optional manifest attributes 2xx:
CreatedAt *Timestamp `protobuf:"bytes,201,opt,name=createdAt,proto3,oneof" json:"createdAt,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *MFFile) Reset() {
*x = MFFile{}
mi := &file_mf_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *MFFile) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MFFile) ProtoMessage() {}
func (x *MFFile) ProtoReflect() protoreflect.Message {
mi := &file_mf_proto_msgTypes[4]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MFFile.ProtoReflect.Descriptor instead.
func (*MFFile) Descriptor() ([]byte, []int) {
return file_mf_proto_rawDescGZIP(), []int{4}
}
func (x *MFFile) GetVersion() MFFile_Version {
if x != nil {
return x.Version
}
return MFFile_VERSION_NONE
}
func (x *MFFile) GetFiles() []*MFFilePath {
if x != nil {
return x.Files
}
return nil
}
func (x *MFFile) GetCreatedAt() *Timestamp {
if x != nil {
return x.CreatedAt
}
return nil
}
var File_mf_proto protoreflect.FileDescriptor
const file_mf_proto_rawDesc = "" +
"\n" +
"\bmf.proto\";\n" +
"\tTimestamp\x12\x18\n" +
"\aseconds\x18\x01 \x01(\x03R\aseconds\x12\x14\n" +
"\x05nanos\x18\x02 \x01(\x05R\x05nanos\"\xdc\x03\n" +
"\vMFFileOuter\x12.\n" +
"\aversion\x18e \x01(\x0e2\x14.MFFileOuter.VersionR\aversion\x12F\n" +
"\x0fcompressionType\x18f \x01(\x0e2\x1c.MFFileOuter.CompressionTypeR\x0fcompressionType\x12\x12\n" +
"\x04size\x18g \x01(\x03R\x04size\x12\x16\n" +
"\x06sha256\x18h \x01(\fR\x06sha256\x12#\n" +
"\finnerMessage\x18\xc7\x01 \x01(\fR\finnerMessage\x12\"\n" +
"\tsignature\x18\xc9\x01 \x01(\fH\x00R\tsignature\x88\x01\x01\x12\x1c\n" +
"\x06signer\x18\xca\x01 \x01(\fH\x01R\x06signer\x88\x01\x01\x12*\n" +
"\rsigningPubKey\x18\xcb\x01 \x01(\fH\x02R\rsigningPubKey\x88\x01\x01\",\n" +
"\aVersion\x12\x10\n" +
"\fVERSION_NONE\x10\x00\x12\x0f\n" +
"\vVERSION_ONE\x10\x01\"=\n" +
"\x0fCompressionType\x12\x14\n" +
"\x10COMPRESSION_NONE\x10\x00\x12\x14\n" +
"\x10COMPRESSION_ZSTD\x10\x01B\f\n" +
"\n" +
"_signatureB\t\n" +
"\a_signerB\x10\n" +
"\x0e_signingPubKey\"\xa2\x02\n" +
"\n" +
"MFFilePath\x12\x12\n" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
"\x04size\x18\x02 \x01(\x03R\x04size\x12'\n" +
"\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12 \n" +
"\bmimeType\x18\xad\x02 \x01(\tH\x00R\bmimeType\x88\x01\x01\x12&\n" +
"\x05mtime\x18\xae\x02 \x01(\v2\n" +
".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" +
"\x05ctime\x18\xaf\x02 \x01(\v2\n" +
".TimestampH\x02R\x05ctime\x88\x01\x01\x12&\n" +
"\x05atime\x18\xb0\x02 \x01(\v2\n" +
".TimestampH\x03R\x05atime\x88\x01\x01B\v\n" +
"\t_mimeTypeB\b\n" +
"\x06_mtimeB\b\n" +
"\x06_ctimeB\b\n" +
"\x06_atime\".\n" +
"\x0eMFFileChecksum\x12\x1c\n" +
"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xc2\x01\n" +
"\x06MFFile\x12)\n" +
"\aversion\x18d \x01(\x0e2\x0f.MFFile.VersionR\aversion\x12!\n" +
"\x05files\x18e \x03(\v2\v.MFFilePathR\x05files\x12.\n" +
"\tcreatedAt\x18\xc9\x01 \x01(\v2\n" +
".TimestampH\x00R\tcreatedAt\x88\x01\x01\",\n" +
"\aVersion\x12\x10\n" +
"\fVERSION_NONE\x10\x00\x12\x0f\n" +
"\vVERSION_ONE\x10\x01B\f\n" +
"\n" +
"_createdAtB\x1dZ\x1bgit.eeqj.de/sneak/mfer/mferb\x06proto3"
var (
file_mf_proto_rawDescOnce sync.Once
file_mf_proto_rawDescData []byte
)
func file_mf_proto_rawDescGZIP() []byte {
file_mf_proto_rawDescOnce.Do(func() {
file_mf_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)))
})
return file_mf_proto_rawDescData
}
var file_mf_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
var file_mf_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_mf_proto_goTypes = []any{
(MFFileOuter_Version)(0), // 0: MFFileOuter.Version
(MFFileOuter_CompressionType)(0), // 1: MFFileOuter.CompressionType
(MFFile_Version)(0), // 2: MFFile.Version
(*Timestamp)(nil), // 3: Timestamp
(*MFFileOuter)(nil), // 4: MFFileOuter
(*MFFilePath)(nil), // 5: MFFilePath
(*MFFileChecksum)(nil), // 6: MFFileChecksum
(*MFFile)(nil), // 7: MFFile
}
var file_mf_proto_depIdxs = []int32{
0, // 0: MFFileOuter.version:type_name -> MFFileOuter.Version
1, // 1: MFFileOuter.compressionType:type_name -> MFFileOuter.CompressionType
6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum
3, // 3: MFFilePath.mtime:type_name -> Timestamp
3, // 4: MFFilePath.ctime:type_name -> Timestamp
3, // 5: MFFilePath.atime:type_name -> Timestamp
2, // 6: MFFile.version:type_name -> MFFile.Version
5, // 7: MFFile.files:type_name -> MFFilePath
3, // 8: MFFile.createdAt:type_name -> Timestamp
9, // [9:9] is the sub-list for method output_type
9, // [9:9] is the sub-list for method input_type
9, // [9:9] is the sub-list for extension type_name
9, // [9:9] is the sub-list for extension extendee
0, // [0:9] is the sub-list for field type_name
}
func init() { file_mf_proto_init() }
func file_mf_proto_init() {
if File_mf_proto != nil {
return
}
file_mf_proto_msgTypes[1].OneofWrappers = []any{}
file_mf_proto_msgTypes[2].OneofWrappers = []any{}
file_mf_proto_msgTypes[4].OneofWrappers = []any{}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)),
NumEnums: 3,
NumMessages: 5,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_mf_proto_goTypes,
DependencyIndexes: file_mf_proto_depIdxs,
EnumInfos: file_mf_proto_enumTypes,
MessageInfos: file_mf_proto_msgTypes,
}.Build()
File_mf_proto = out.File
file_mf_proto_goTypes = nil
file_mf_proto_depIdxs = nil
}

2
mfer/mf.proto
View File

@@ -18,7 +18,7 @@ message MFFileOuter {
enum CompressionType {
COMPRESSION_NONE = 0;
COMPRESSION_GZIP = 1;
COMPRESSION_ZSTD = 1;
}
CompressionType compressionType = 102;

10
mfer/serialize.go
View File

@@ -2,11 +2,11 @@ package mfer
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"errors"
"time"
"github.com/klauspost/compress/zstd"
"google.golang.org/protobuf/proto"
)
@@ -56,23 +56,23 @@ func (m *manifest) generateOuter() error {
h.Write(innerData)
idc := new(bytes.Buffer)
gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
if err != nil {
return err
}
_, err = gzw.Write(innerData)
_, err = zw.Write(innerData)
if err != nil {
return err
}
_ = gzw.Close()
_ = zw.Close()
o := &MFFileOuter{
InnerMessage: idc.Bytes(),
Size: int64(len(innerData)),
Sha256: h.Sum(nil),
Version: MFFileOuter_VERSION_ONE,
CompressionType: MFFileOuter_COMPRESSION_GZIP,
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
}
m.pbOuter = o
return nil