From 21028af9aa32a57fbc2d32d08e71bae9a10e1279 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 17 Dec 2025 14:49:30 -0800 Subject: [PATCH] Replace gzip with zstd compression Use zstd with SpeedBestCompression level for better compression ratios. Remove gzip support entirely. Include generated protobuf file to allow building without protoc. --- .gitignore | 1 - README.md | 2 +- go.mod | 3 +- go.sum | 5 +- mfer/deserialize.go | 10 +- mfer/mf.pb.go | 646 ++++++++++++++++++++++++++++++++++++++++++++ mfer/mf.proto | 2 +- mfer/serialize.go | 10 +- 8 files changed, 662 insertions(+), 17 deletions(-) create mode 100644 mfer/mf.pb.go diff --git a/.gitignore b/.gitignore index 105d015..ffc61dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -mfer/*.pb.go /mfer.cmd /tmp *.tmp diff --git a/README.md b/README.md index 04b2c4d..badd95d 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Reading file contents and computing cryptographic hashes for manifest generation - `MFFileOuter_VERSION_ONE` - `MFFileOuter_CompressionType` - Compression type for inner message - `MFFileOuter_COMPRESSION_NONE` - - `MFFileOuter_COMPRESSION_GZIP` + - `MFFileOuter_COMPRESSION_ZSTD` - `MFFile_Version` - Inner file format version - `MFFile_VERSION_NONE` - `MFFile_VERSION_ONE` diff --git a/go.mod b/go.mod index e440182..9271ad8 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module sneak.berlin/go/mfer -go 1.17 +go 1.23 require ( github.com/apex/log v1.9.0 github.com/davecgh/go-spew v1.1.1 + github.com/klauspost/compress v1.18.2 github.com/multiformats/go-multihash v0.2.3 github.com/pterm/pterm v0.12.35 github.com/spf13/afero v1.8.0 diff --git a/go.sum b/go.sum index 660d476..b4c7c10 100644 --- a/go.sum +++ b/go.sum @@ -37,7 +37,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= @@ -150,6 +149,8 @@ github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgb github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -318,7 +319,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -373,7 +373,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/mfer/deserialize.go b/mfer/deserialize.go index 56ff037..3555775 100644 --- a/mfer/deserialize.go +++ b/mfer/deserialize.go @@ -2,10 +2,10 @@ package mfer import ( "bytes" - "compress/gzip" "errors" "io" + "github.com/klauspost/compress/zstd" "github.com/spf13/afero" "google.golang.org/protobuf/proto" "sneak.berlin/go/mfer/internal/bork" @@ -16,19 +16,19 @@ func (m *manifest) deserializeInner() error { if m.pbOuter.Version != MFFileOuter_VERSION_ONE { return errors.New("unknown version") } - if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP { + if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_ZSTD { return errors.New("unknown compression type") } bb := bytes.NewBuffer(m.pbOuter.InnerMessage) - gzr, err := gzip.NewReader(bb) + zr, err := zstd.NewReader(bb) if err != nil { return err } - defer func() { _ = gzr.Close() }() + defer zr.Close() - dat, err := io.ReadAll(gzr) + dat, err := io.ReadAll(zr) if err != nil { return err } diff --git a/mfer/mf.pb.go b/mfer/mf.pb.go new file mode 100644 index 0000000..1cf2de6 --- /dev/null +++ b/mfer/mf.pb.go @@ -0,0 +1,646 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v6.33.0 +// source: mf.proto + +package mfer + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type MFFileOuter_Version int32 + +const ( + MFFileOuter_VERSION_NONE MFFileOuter_Version = 0 + MFFileOuter_VERSION_ONE MFFileOuter_Version = 1 // only one for now +) + +// Enum value maps for MFFileOuter_Version. +var ( + MFFileOuter_Version_name = map[int32]string{ + 0: "VERSION_NONE", + 1: "VERSION_ONE", + } + MFFileOuter_Version_value = map[string]int32{ + "VERSION_NONE": 0, + "VERSION_ONE": 1, + } +) + +func (x MFFileOuter_Version) Enum() *MFFileOuter_Version { + p := new(MFFileOuter_Version) + *p = x + return p +} + +func (x MFFileOuter_Version) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MFFileOuter_Version) Descriptor() protoreflect.EnumDescriptor { + return file_mf_proto_enumTypes[0].Descriptor() +} + +func (MFFileOuter_Version) Type() protoreflect.EnumType { + return &file_mf_proto_enumTypes[0] +} + +func (x MFFileOuter_Version) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MFFileOuter_Version.Descriptor instead. +func (MFFileOuter_Version) EnumDescriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{1, 0} +} + +type MFFileOuter_CompressionType int32 + +const ( + MFFileOuter_COMPRESSION_NONE MFFileOuter_CompressionType = 0 + MFFileOuter_COMPRESSION_ZSTD MFFileOuter_CompressionType = 1 +) + +// Enum value maps for MFFileOuter_CompressionType. +var ( + MFFileOuter_CompressionType_name = map[int32]string{ + 0: "COMPRESSION_NONE", + 1: "COMPRESSION_ZSTD", + } + MFFileOuter_CompressionType_value = map[string]int32{ + "COMPRESSION_NONE": 0, + "COMPRESSION_ZSTD": 1, + } +) + +func (x MFFileOuter_CompressionType) Enum() *MFFileOuter_CompressionType { + p := new(MFFileOuter_CompressionType) + *p = x + return p +} + +func (x MFFileOuter_CompressionType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MFFileOuter_CompressionType) Descriptor() protoreflect.EnumDescriptor { + return file_mf_proto_enumTypes[1].Descriptor() +} + +func (MFFileOuter_CompressionType) Type() protoreflect.EnumType { + return &file_mf_proto_enumTypes[1] +} + +func (x MFFileOuter_CompressionType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MFFileOuter_CompressionType.Descriptor instead. +func (MFFileOuter_CompressionType) EnumDescriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{1, 1} +} + +type MFFile_Version int32 + +const ( + MFFile_VERSION_NONE MFFile_Version = 0 + MFFile_VERSION_ONE MFFile_Version = 1 // only one for now +) + +// Enum value maps for MFFile_Version. +var ( + MFFile_Version_name = map[int32]string{ + 0: "VERSION_NONE", + 1: "VERSION_ONE", + } + MFFile_Version_value = map[string]int32{ + "VERSION_NONE": 0, + "VERSION_ONE": 1, + } +) + +func (x MFFile_Version) Enum() *MFFile_Version { + p := new(MFFile_Version) + *p = x + return p +} + +func (x MFFile_Version) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MFFile_Version) Descriptor() protoreflect.EnumDescriptor { + return file_mf_proto_enumTypes[2].Descriptor() +} + +func (MFFile_Version) Type() protoreflect.EnumType { + return &file_mf_proto_enumTypes[2] +} + +func (x MFFile_Version) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MFFile_Version.Descriptor instead. +func (MFFile_Version) EnumDescriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{4, 0} +} + +type Timestamp struct { + state protoimpl.MessageState `protogen:"open.v1"` + Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"` + Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Timestamp) Reset() { + *x = Timestamp{} + mi := &file_mf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Timestamp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Timestamp) ProtoMessage() {} + +func (x *Timestamp) ProtoReflect() protoreflect.Message { + mi := &file_mf_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Timestamp.ProtoReflect.Descriptor instead. +func (*Timestamp) Descriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{0} +} + +func (x *Timestamp) GetSeconds() int64 { + if x != nil { + return x.Seconds + } + return 0 +} + +func (x *Timestamp) GetNanos() int32 { + if x != nil { + return x.Nanos + } + return 0 +} + +type MFFileOuter struct { + state protoimpl.MessageState `protogen:"open.v1"` + // required mffile root attributes 1xx + Version MFFileOuter_Version `protobuf:"varint,101,opt,name=version,proto3,enum=MFFileOuter_Version" json:"version,omitempty"` + CompressionType MFFileOuter_CompressionType `protobuf:"varint,102,opt,name=compressionType,proto3,enum=MFFileOuter_CompressionType" json:"compressionType,omitempty"` + // these are used solely to detect corruption/truncation + // and not for cryptographic integrity. + Size int64 `protobuf:"varint,103,opt,name=size,proto3" json:"size,omitempty"` + Sha256 []byte `protobuf:"bytes,104,opt,name=sha256,proto3" json:"sha256,omitempty"` + InnerMessage []byte `protobuf:"bytes,199,opt,name=innerMessage,proto3" json:"innerMessage,omitempty"` + // detached signature, ascii or binary + Signature []byte `protobuf:"bytes,201,opt,name=signature,proto3,oneof" json:"signature,omitempty"` + // full GPG key id + Signer []byte `protobuf:"bytes,202,opt,name=signer,proto3,oneof" json:"signer,omitempty"` + // full GPG signing public key, ascii or binary + SigningPubKey []byte `protobuf:"bytes,203,opt,name=signingPubKey,proto3,oneof" json:"signingPubKey,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MFFileOuter) Reset() { + *x = MFFileOuter{} + mi := &file_mf_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MFFileOuter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MFFileOuter) ProtoMessage() {} + +func (x *MFFileOuter) ProtoReflect() protoreflect.Message { + mi := &file_mf_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MFFileOuter.ProtoReflect.Descriptor instead. +func (*MFFileOuter) Descriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{1} +} + +func (x *MFFileOuter) GetVersion() MFFileOuter_Version { + if x != nil { + return x.Version + } + return MFFileOuter_VERSION_NONE +} + +func (x *MFFileOuter) GetCompressionType() MFFileOuter_CompressionType { + if x != nil { + return x.CompressionType + } + return MFFileOuter_COMPRESSION_NONE +} + +func (x *MFFileOuter) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *MFFileOuter) GetSha256() []byte { + if x != nil { + return x.Sha256 + } + return nil +} + +func (x *MFFileOuter) GetInnerMessage() []byte { + if x != nil { + return x.InnerMessage + } + return nil +} + +func (x *MFFileOuter) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +func (x *MFFileOuter) GetSigner() []byte { + if x != nil { + return x.Signer + } + return nil +} + +func (x *MFFileOuter) GetSigningPubKey() []byte { + if x != nil { + return x.SigningPubKey + } + return nil +} + +type MFFilePath struct { + state protoimpl.MessageState `protogen:"open.v1"` + // required attributes: + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + // gotta have at least one: + Hashes []*MFFileChecksum `protobuf:"bytes,3,rep,name=hashes,proto3" json:"hashes,omitempty"` + // optional per-file metadata + MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"` + Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"` + Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"` + Atime *Timestamp `protobuf:"bytes,304,opt,name=atime,proto3,oneof" json:"atime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MFFilePath) Reset() { + *x = MFFilePath{} + mi := &file_mf_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MFFilePath) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MFFilePath) ProtoMessage() {} + +func (x *MFFilePath) ProtoReflect() protoreflect.Message { + mi := &file_mf_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MFFilePath.ProtoReflect.Descriptor instead. +func (*MFFilePath) Descriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{2} +} + +func (x *MFFilePath) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *MFFilePath) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *MFFilePath) GetHashes() []*MFFileChecksum { + if x != nil { + return x.Hashes + } + return nil +} + +func (x *MFFilePath) GetMimeType() string { + if x != nil && x.MimeType != nil { + return *x.MimeType + } + return "" +} + +func (x *MFFilePath) GetMtime() *Timestamp { + if x != nil { + return x.Mtime + } + return nil +} + +func (x *MFFilePath) GetCtime() *Timestamp { + if x != nil { + return x.Ctime + } + return nil +} + +func (x *MFFilePath) GetAtime() *Timestamp { + if x != nil { + return x.Atime + } + return nil +} + +type MFFileChecksum struct { + state protoimpl.MessageState `protogen:"open.v1"` + // 1.0 golang implementation must write a multihash here + // it's ok to only ever use/verify sha256 multihash + MultiHash []byte `protobuf:"bytes,1,opt,name=multiHash,proto3" json:"multiHash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MFFileChecksum) Reset() { + *x = MFFileChecksum{} + mi := &file_mf_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MFFileChecksum) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MFFileChecksum) ProtoMessage() {} + +func (x *MFFileChecksum) ProtoReflect() protoreflect.Message { + mi := &file_mf_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MFFileChecksum.ProtoReflect.Descriptor instead. +func (*MFFileChecksum) Descriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{3} +} + +func (x *MFFileChecksum) GetMultiHash() []byte { + if x != nil { + return x.MultiHash + } + return nil +} + +type MFFile struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version MFFile_Version `protobuf:"varint,100,opt,name=version,proto3,enum=MFFile_Version" json:"version,omitempty"` + // required manifest attributes: + Files []*MFFilePath `protobuf:"bytes,101,rep,name=files,proto3" json:"files,omitempty"` + // optional manifest attributes 2xx: + CreatedAt *Timestamp `protobuf:"bytes,201,opt,name=createdAt,proto3,oneof" json:"createdAt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MFFile) Reset() { + *x = MFFile{} + mi := &file_mf_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MFFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MFFile) ProtoMessage() {} + +func (x *MFFile) ProtoReflect() protoreflect.Message { + mi := &file_mf_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MFFile.ProtoReflect.Descriptor instead. +func (*MFFile) Descriptor() ([]byte, []int) { + return file_mf_proto_rawDescGZIP(), []int{4} +} + +func (x *MFFile) GetVersion() MFFile_Version { + if x != nil { + return x.Version + } + return MFFile_VERSION_NONE +} + +func (x *MFFile) GetFiles() []*MFFilePath { + if x != nil { + return x.Files + } + return nil +} + +func (x *MFFile) GetCreatedAt() *Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +var File_mf_proto protoreflect.FileDescriptor + +const file_mf_proto_rawDesc = "" + + "\n" + + "\bmf.proto\";\n" + + "\tTimestamp\x12\x18\n" + + "\aseconds\x18\x01 \x01(\x03R\aseconds\x12\x14\n" + + "\x05nanos\x18\x02 \x01(\x05R\x05nanos\"\xdc\x03\n" + + "\vMFFileOuter\x12.\n" + + "\aversion\x18e \x01(\x0e2\x14.MFFileOuter.VersionR\aversion\x12F\n" + + "\x0fcompressionType\x18f \x01(\x0e2\x1c.MFFileOuter.CompressionTypeR\x0fcompressionType\x12\x12\n" + + "\x04size\x18g \x01(\x03R\x04size\x12\x16\n" + + "\x06sha256\x18h \x01(\fR\x06sha256\x12#\n" + + "\finnerMessage\x18\xc7\x01 \x01(\fR\finnerMessage\x12\"\n" + + "\tsignature\x18\xc9\x01 \x01(\fH\x00R\tsignature\x88\x01\x01\x12\x1c\n" + + "\x06signer\x18\xca\x01 \x01(\fH\x01R\x06signer\x88\x01\x01\x12*\n" + + "\rsigningPubKey\x18\xcb\x01 \x01(\fH\x02R\rsigningPubKey\x88\x01\x01\",\n" + + "\aVersion\x12\x10\n" + + "\fVERSION_NONE\x10\x00\x12\x0f\n" + + "\vVERSION_ONE\x10\x01\"=\n" + + "\x0fCompressionType\x12\x14\n" + + "\x10COMPRESSION_NONE\x10\x00\x12\x14\n" + + "\x10COMPRESSION_ZSTD\x10\x01B\f\n" + + "\n" + + "_signatureB\t\n" + + "\a_signerB\x10\n" + + "\x0e_signingPubKey\"\xa2\x02\n" + + "\n" + + "MFFilePath\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" + + "\x04size\x18\x02 \x01(\x03R\x04size\x12'\n" + + "\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12 \n" + + "\bmimeType\x18\xad\x02 \x01(\tH\x00R\bmimeType\x88\x01\x01\x12&\n" + + "\x05mtime\x18\xae\x02 \x01(\v2\n" + + ".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" + + "\x05ctime\x18\xaf\x02 \x01(\v2\n" + + ".TimestampH\x02R\x05ctime\x88\x01\x01\x12&\n" + + "\x05atime\x18\xb0\x02 \x01(\v2\n" + + ".TimestampH\x03R\x05atime\x88\x01\x01B\v\n" + + "\t_mimeTypeB\b\n" + + "\x06_mtimeB\b\n" + + "\x06_ctimeB\b\n" + + "\x06_atime\".\n" + + "\x0eMFFileChecksum\x12\x1c\n" + + "\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xc2\x01\n" + + "\x06MFFile\x12)\n" + + "\aversion\x18d \x01(\x0e2\x0f.MFFile.VersionR\aversion\x12!\n" + + "\x05files\x18e \x03(\v2\v.MFFilePathR\x05files\x12.\n" + + "\tcreatedAt\x18\xc9\x01 \x01(\v2\n" + + ".TimestampH\x00R\tcreatedAt\x88\x01\x01\",\n" + + "\aVersion\x12\x10\n" + + "\fVERSION_NONE\x10\x00\x12\x0f\n" + + "\vVERSION_ONE\x10\x01B\f\n" + + "\n" + + "_createdAtB\x1dZ\x1bgit.eeqj.de/sneak/mfer/mferb\x06proto3" + +var ( + file_mf_proto_rawDescOnce sync.Once + file_mf_proto_rawDescData []byte +) + +func file_mf_proto_rawDescGZIP() []byte { + file_mf_proto_rawDescOnce.Do(func() { + file_mf_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc))) + }) + return file_mf_proto_rawDescData +} + +var file_mf_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_mf_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_mf_proto_goTypes = []any{ + (MFFileOuter_Version)(0), // 0: MFFileOuter.Version + (MFFileOuter_CompressionType)(0), // 1: MFFileOuter.CompressionType + (MFFile_Version)(0), // 2: MFFile.Version + (*Timestamp)(nil), // 3: Timestamp + (*MFFileOuter)(nil), // 4: MFFileOuter + (*MFFilePath)(nil), // 5: MFFilePath + (*MFFileChecksum)(nil), // 6: MFFileChecksum + (*MFFile)(nil), // 7: MFFile +} +var file_mf_proto_depIdxs = []int32{ + 0, // 0: MFFileOuter.version:type_name -> MFFileOuter.Version + 1, // 1: MFFileOuter.compressionType:type_name -> MFFileOuter.CompressionType + 6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum + 3, // 3: MFFilePath.mtime:type_name -> Timestamp + 3, // 4: MFFilePath.ctime:type_name -> Timestamp + 3, // 5: MFFilePath.atime:type_name -> Timestamp + 2, // 6: MFFile.version:type_name -> MFFile.Version + 5, // 7: MFFile.files:type_name -> MFFilePath + 3, // 8: MFFile.createdAt:type_name -> Timestamp + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_mf_proto_init() } +func file_mf_proto_init() { + if File_mf_proto != nil { + return + } + file_mf_proto_msgTypes[1].OneofWrappers = []any{} + file_mf_proto_msgTypes[2].OneofWrappers = []any{} + file_mf_proto_msgTypes[4].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)), + NumEnums: 3, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_mf_proto_goTypes, + DependencyIndexes: file_mf_proto_depIdxs, + EnumInfos: file_mf_proto_enumTypes, + MessageInfos: file_mf_proto_msgTypes, + }.Build() + File_mf_proto = out.File + file_mf_proto_goTypes = nil + file_mf_proto_depIdxs = nil +} diff --git a/mfer/mf.proto b/mfer/mf.proto index ebac757..4c256b1 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -18,7 +18,7 @@ message MFFileOuter { enum CompressionType { COMPRESSION_NONE = 0; - COMPRESSION_GZIP = 1; + COMPRESSION_ZSTD = 1; } CompressionType compressionType = 102; diff --git a/mfer/serialize.go b/mfer/serialize.go index 16ad2eb..fee0bfe 100644 --- a/mfer/serialize.go +++ b/mfer/serialize.go @@ -2,11 +2,11 @@ package mfer import ( "bytes" - "compress/gzip" "crypto/sha256" "errors" "time" + "github.com/klauspost/compress/zstd" "google.golang.org/protobuf/proto" ) @@ -56,23 +56,23 @@ func (m *manifest) generateOuter() error { h.Write(innerData) idc := new(bytes.Buffer) - gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression) + zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) if err != nil { return err } - _, err = gzw.Write(innerData) + _, err = zw.Write(innerData) if err != nil { return err } - _ = gzw.Close() + _ = zw.Close() o := &MFFileOuter{ InnerMessage: idc.Bytes(), Size: int64(len(innerData)), Sha256: h.Sum(nil), Version: MFFileOuter_VERSION_ONE, - CompressionType: MFFileOuter_COMPRESSION_GZIP, + CompressionType: MFFileOuter_COMPRESSION_ZSTD, } m.pbOuter = o return nil