Replace gzip with zstd compression

Use zstd with SpeedBestCompression level for better compression
ratios. Remove gzip support entirely. Include generated protobuf
file to allow building without protoc.

.gitignore

@@ -1,4 +1,3 @@
-mfer/*.pb.go
 /mfer.cmd
 /tmp
 *.tmp

README.md

@@ -219,7 +219,7 @@ Reading file contents and computing cryptographic hashes for manifest generation
     - `MFFileOuter_VERSION_ONE`
   - `MFFileOuter_CompressionType` - Compression type for inner message
     - `MFFileOuter_COMPRESSION_NONE`
-    - `MFFileOuter_COMPRESSION_GZIP`
+    - `MFFileOuter_COMPRESSION_ZSTD`
   - `MFFile_Version` - Inner file format version
     - `MFFile_VERSION_NONE`
     - `MFFile_VERSION_ONE`

go.mod

@@ -1,10 +1,11 @@
 module sneak.berlin/go/mfer
 
-go 1.17
+go 1.23
 
 require (
 	github.com/apex/log v1.9.0
 	github.com/davecgh/go-spew v1.1.1
+	github.com/klauspost/compress v1.18.2
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/pterm/pterm v0.12.35
 	github.com/spf13/afero v1.8.0

go.sum

@@ -37,7 +37,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
 github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
@@ -150,6 +149,8 @@ github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgb
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
+github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -318,7 +319,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -373,7 +373,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

mfer/deserialize.go

@@ -2,10 +2,10 @@ package mfer
 
 import (
 	"bytes"
-	"compress/gzip"
 	"errors"
 	"io"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/afero"
 	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/internal/bork"
@@ -16,19 +16,19 @@ func (m *manifest) deserializeInner() error {
 	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
 		return errors.New("unknown version")
 	}
-	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
+	if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_ZSTD {
 		return errors.New("unknown compression type")
 	}
 
 	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
 
-	gzr, err := gzip.NewReader(bb)
+	zr, err := zstd.NewReader(bb)
 	if err != nil {
 		return err
 	}
-	defer func() { _ = gzr.Close() }()
+	defer zr.Close()
 
-	dat, err := io.ReadAll(gzr)
+	dat, err := io.ReadAll(zr)
 	if err != nil {
 		return err
 	}

mfer/mf.pb.go

@@ -0,0 +1,646 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.11
+// 	protoc        v6.33.0
+// source: mf.proto
+
+package mfer
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type MFFileOuter_Version int32
+
+const (
+	MFFileOuter_VERSION_NONE MFFileOuter_Version = 0
+	MFFileOuter_VERSION_ONE  MFFileOuter_Version = 1 // only one for now
+)
+
+// Enum value maps for MFFileOuter_Version.
+var (
+	MFFileOuter_Version_name = map[int32]string{
+		0: "VERSION_NONE",
+		1: "VERSION_ONE",
+	}
+	MFFileOuter_Version_value = map[string]int32{
+		"VERSION_NONE": 0,
+		"VERSION_ONE":  1,
+	}
+)
+
+func (x MFFileOuter_Version) Enum() *MFFileOuter_Version {
+	p := new(MFFileOuter_Version)
+	*p = x
+	return p
+}
+
+func (x MFFileOuter_Version) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (MFFileOuter_Version) Descriptor() protoreflect.EnumDescriptor {
+	return file_mf_proto_enumTypes[0].Descriptor()
+}
+
+func (MFFileOuter_Version) Type() protoreflect.EnumType {
+	return &file_mf_proto_enumTypes[0]
+}
+
+func (x MFFileOuter_Version) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use MFFileOuter_Version.Descriptor instead.
+func (MFFileOuter_Version) EnumDescriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{1, 0}
+}
+
+type MFFileOuter_CompressionType int32
+
+const (
+	MFFileOuter_COMPRESSION_NONE MFFileOuter_CompressionType = 0
+	MFFileOuter_COMPRESSION_ZSTD MFFileOuter_CompressionType = 1
+)
+
+// Enum value maps for MFFileOuter_CompressionType.
+var (
+	MFFileOuter_CompressionType_name = map[int32]string{
+		0: "COMPRESSION_NONE",
+		1: "COMPRESSION_ZSTD",
+	}
+	MFFileOuter_CompressionType_value = map[string]int32{
+		"COMPRESSION_NONE": 0,
+		"COMPRESSION_ZSTD": 1,
+	}
+)
+
+func (x MFFileOuter_CompressionType) Enum() *MFFileOuter_CompressionType {
+	p := new(MFFileOuter_CompressionType)
+	*p = x
+	return p
+}
+
+func (x MFFileOuter_CompressionType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (MFFileOuter_CompressionType) Descriptor() protoreflect.EnumDescriptor {
+	return file_mf_proto_enumTypes[1].Descriptor()
+}
+
+func (MFFileOuter_CompressionType) Type() protoreflect.EnumType {
+	return &file_mf_proto_enumTypes[1]
+}
+
+func (x MFFileOuter_CompressionType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use MFFileOuter_CompressionType.Descriptor instead.
+func (MFFileOuter_CompressionType) EnumDescriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{1, 1}
+}
+
+type MFFile_Version int32
+
+const (
+	MFFile_VERSION_NONE MFFile_Version = 0
+	MFFile_VERSION_ONE  MFFile_Version = 1 // only one for now
+)
+
+// Enum value maps for MFFile_Version.
+var (
+	MFFile_Version_name = map[int32]string{
+		0: "VERSION_NONE",
+		1: "VERSION_ONE",
+	}
+	MFFile_Version_value = map[string]int32{
+		"VERSION_NONE": 0,
+		"VERSION_ONE":  1,
+	}
+)
+
+func (x MFFile_Version) Enum() *MFFile_Version {
+	p := new(MFFile_Version)
+	*p = x
+	return p
+}
+
+func (x MFFile_Version) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (MFFile_Version) Descriptor() protoreflect.EnumDescriptor {
+	return file_mf_proto_enumTypes[2].Descriptor()
+}
+
+func (MFFile_Version) Type() protoreflect.EnumType {
+	return &file_mf_proto_enumTypes[2]
+}
+
+func (x MFFile_Version) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use MFFile_Version.Descriptor instead.
+func (MFFile_Version) EnumDescriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{4, 0}
+}
+
+type Timestamp struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Seconds       int64                  `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"`
+	Nanos         int32                  `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Timestamp) Reset() {
+	*x = Timestamp{}
+	mi := &file_mf_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Timestamp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Timestamp) ProtoMessage() {}
+
+func (x *Timestamp) ProtoReflect() protoreflect.Message {
+	mi := &file_mf_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Timestamp.ProtoReflect.Descriptor instead.
+func (*Timestamp) Descriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Timestamp) GetSeconds() int64 {
+	if x != nil {
+		return x.Seconds
+	}
+	return 0
+}
+
+func (x *Timestamp) GetNanos() int32 {
+	if x != nil {
+		return x.Nanos
+	}
+	return 0
+}
+
+type MFFileOuter struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// required mffile root attributes 1xx
+	Version         MFFileOuter_Version         `protobuf:"varint,101,opt,name=version,proto3,enum=MFFileOuter_Version" json:"version,omitempty"`
+	CompressionType MFFileOuter_CompressionType `protobuf:"varint,102,opt,name=compressionType,proto3,enum=MFFileOuter_CompressionType" json:"compressionType,omitempty"`
+	// these are used solely to detect corruption/truncation
+	// and not for cryptographic integrity.
+	Size         int64  `protobuf:"varint,103,opt,name=size,proto3" json:"size,omitempty"`
+	Sha256       []byte `protobuf:"bytes,104,opt,name=sha256,proto3" json:"sha256,omitempty"`
+	InnerMessage []byte `protobuf:"bytes,199,opt,name=innerMessage,proto3" json:"innerMessage,omitempty"`
+	// detached signature, ascii or binary
+	Signature []byte `protobuf:"bytes,201,opt,name=signature,proto3,oneof" json:"signature,omitempty"`
+	// full GPG key id
+	Signer []byte `protobuf:"bytes,202,opt,name=signer,proto3,oneof" json:"signer,omitempty"`
+	// full GPG signing public key, ascii or binary
+	SigningPubKey []byte `protobuf:"bytes,203,opt,name=signingPubKey,proto3,oneof" json:"signingPubKey,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *MFFileOuter) Reset() {
+	*x = MFFileOuter{}
+	mi := &file_mf_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *MFFileOuter) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MFFileOuter) ProtoMessage() {}
+
+func (x *MFFileOuter) ProtoReflect() protoreflect.Message {
+	mi := &file_mf_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MFFileOuter.ProtoReflect.Descriptor instead.
+func (*MFFileOuter) Descriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *MFFileOuter) GetVersion() MFFileOuter_Version {
+	if x != nil {
+		return x.Version
+	}
+	return MFFileOuter_VERSION_NONE
+}
+
+func (x *MFFileOuter) GetCompressionType() MFFileOuter_CompressionType {
+	if x != nil {
+		return x.CompressionType
+	}
+	return MFFileOuter_COMPRESSION_NONE
+}
+
+func (x *MFFileOuter) GetSize() int64 {
+	if x != nil {
+		return x.Size
+	}
+	return 0
+}
+
+func (x *MFFileOuter) GetSha256() []byte {
+	if x != nil {
+		return x.Sha256
+	}
+	return nil
+}
+
+func (x *MFFileOuter) GetInnerMessage() []byte {
+	if x != nil {
+		return x.InnerMessage
+	}
+	return nil
+}
+
+func (x *MFFileOuter) GetSignature() []byte {
+	if x != nil {
+		return x.Signature
+	}
+	return nil
+}
+
+func (x *MFFileOuter) GetSigner() []byte {
+	if x != nil {
+		return x.Signer
+	}
+	return nil
+}
+
+func (x *MFFileOuter) GetSigningPubKey() []byte {
+	if x != nil {
+		return x.SigningPubKey
+	}
+	return nil
+}
+
+type MFFilePath struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// required attributes:
+	Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
+	Size int64  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
+	// gotta have at least one:
+	Hashes []*MFFileChecksum `protobuf:"bytes,3,rep,name=hashes,proto3" json:"hashes,omitempty"`
+	// optional per-file metadata
+	MimeType      *string    `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"`
+	Mtime         *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
+	Ctime         *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"`
+	Atime         *Timestamp `protobuf:"bytes,304,opt,name=atime,proto3,oneof" json:"atime,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *MFFilePath) Reset() {
+	*x = MFFilePath{}
+	mi := &file_mf_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *MFFilePath) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MFFilePath) ProtoMessage() {}
+
+func (x *MFFilePath) ProtoReflect() protoreflect.Message {
+	mi := &file_mf_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MFFilePath.ProtoReflect.Descriptor instead.
+func (*MFFilePath) Descriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *MFFilePath) GetPath() string {
+	if x != nil {
+		return x.Path
+	}
+	return ""
+}
+
+func (x *MFFilePath) GetSize() int64 {
+	if x != nil {
+		return x.Size
+	}
+	return 0
+}
+
+func (x *MFFilePath) GetHashes() []*MFFileChecksum {
+	if x != nil {
+		return x.Hashes
+	}
+	return nil
+}
+
+func (x *MFFilePath) GetMimeType() string {
+	if x != nil && x.MimeType != nil {
+		return *x.MimeType
+	}
+	return ""
+}
+
+func (x *MFFilePath) GetMtime() *Timestamp {
+	if x != nil {
+		return x.Mtime
+	}
+	return nil
+}
+
+func (x *MFFilePath) GetCtime() *Timestamp {
+	if x != nil {
+		return x.Ctime
+	}
+	return nil
+}
+
+func (x *MFFilePath) GetAtime() *Timestamp {
+	if x != nil {
+		return x.Atime
+	}
+	return nil
+}
+
+type MFFileChecksum struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// 1.0 golang implementation must write a multihash here
+	// it's ok to only ever use/verify sha256 multihash
+	MultiHash     []byte `protobuf:"bytes,1,opt,name=multiHash,proto3" json:"multiHash,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *MFFileChecksum) Reset() {
+	*x = MFFileChecksum{}
+	mi := &file_mf_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *MFFileChecksum) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MFFileChecksum) ProtoMessage() {}
+
+func (x *MFFileChecksum) ProtoReflect() protoreflect.Message {
+	mi := &file_mf_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MFFileChecksum.ProtoReflect.Descriptor instead.
+func (*MFFileChecksum) Descriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *MFFileChecksum) GetMultiHash() []byte {
+	if x != nil {
+		return x.MultiHash
+	}
+	return nil
+}
+
+type MFFile struct {
+	state   protoimpl.MessageState `protogen:"open.v1"`
+	Version MFFile_Version         `protobuf:"varint,100,opt,name=version,proto3,enum=MFFile_Version" json:"version,omitempty"`
+	// required manifest attributes:
+	Files []*MFFilePath `protobuf:"bytes,101,rep,name=files,proto3" json:"files,omitempty"`
+	// optional manifest attributes 2xx:
+	CreatedAt     *Timestamp `protobuf:"bytes,201,opt,name=createdAt,proto3,oneof" json:"createdAt,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *MFFile) Reset() {
+	*x = MFFile{}
+	mi := &file_mf_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *MFFile) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*MFFile) ProtoMessage() {}
+
+func (x *MFFile) ProtoReflect() protoreflect.Message {
+	mi := &file_mf_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use MFFile.ProtoReflect.Descriptor instead.
+func (*MFFile) Descriptor() ([]byte, []int) {
+	return file_mf_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *MFFile) GetVersion() MFFile_Version {
+	if x != nil {
+		return x.Version
+	}
+	return MFFile_VERSION_NONE
+}
+
+func (x *MFFile) GetFiles() []*MFFilePath {
+	if x != nil {
+		return x.Files
+	}
+	return nil
+}
+
+func (x *MFFile) GetCreatedAt() *Timestamp {
+	if x != nil {
+		return x.CreatedAt
+	}
+	return nil
+}
+
+var File_mf_proto protoreflect.FileDescriptor
+
+const file_mf_proto_rawDesc = "" +
+	"\n" +
+	"\bmf.proto\";\n" +
+	"\tTimestamp\x12\x18\n" +
+	"\aseconds\x18\x01 \x01(\x03R\aseconds\x12\x14\n" +
+	"\x05nanos\x18\x02 \x01(\x05R\x05nanos\"\xdc\x03\n" +
+	"\vMFFileOuter\x12.\n" +
+	"\aversion\x18e \x01(\x0e2\x14.MFFileOuter.VersionR\aversion\x12F\n" +
+	"\x0fcompressionType\x18f \x01(\x0e2\x1c.MFFileOuter.CompressionTypeR\x0fcompressionType\x12\x12\n" +
+	"\x04size\x18g \x01(\x03R\x04size\x12\x16\n" +
+	"\x06sha256\x18h \x01(\fR\x06sha256\x12#\n" +
+	"\finnerMessage\x18\xc7\x01 \x01(\fR\finnerMessage\x12\"\n" +
+	"\tsignature\x18\xc9\x01 \x01(\fH\x00R\tsignature\x88\x01\x01\x12\x1c\n" +
+	"\x06signer\x18\xca\x01 \x01(\fH\x01R\x06signer\x88\x01\x01\x12*\n" +
+	"\rsigningPubKey\x18\xcb\x01 \x01(\fH\x02R\rsigningPubKey\x88\x01\x01\",\n" +
+	"\aVersion\x12\x10\n" +
+	"\fVERSION_NONE\x10\x00\x12\x0f\n" +
+	"\vVERSION_ONE\x10\x01\"=\n" +
+	"\x0fCompressionType\x12\x14\n" +
+	"\x10COMPRESSION_NONE\x10\x00\x12\x14\n" +
+	"\x10COMPRESSION_ZSTD\x10\x01B\f\n" +
+	"\n" +
+	"_signatureB\t\n" +
+	"\a_signerB\x10\n" +
+	"\x0e_signingPubKey\"\xa2\x02\n" +
+	"\n" +
+	"MFFilePath\x12\x12\n" +
+	"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
+	"\x04size\x18\x02 \x01(\x03R\x04size\x12'\n" +
+	"\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12 \n" +
+	"\bmimeType\x18\xad\x02 \x01(\tH\x00R\bmimeType\x88\x01\x01\x12&\n" +
+	"\x05mtime\x18\xae\x02 \x01(\v2\n" +
+	".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" +
+	"\x05ctime\x18\xaf\x02 \x01(\v2\n" +
+	".TimestampH\x02R\x05ctime\x88\x01\x01\x12&\n" +
+	"\x05atime\x18\xb0\x02 \x01(\v2\n" +
+	".TimestampH\x03R\x05atime\x88\x01\x01B\v\n" +
+	"\t_mimeTypeB\b\n" +
+	"\x06_mtimeB\b\n" +
+	"\x06_ctimeB\b\n" +
+	"\x06_atime\".\n" +
+	"\x0eMFFileChecksum\x12\x1c\n" +
+	"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xc2\x01\n" +
+	"\x06MFFile\x12)\n" +
+	"\aversion\x18d \x01(\x0e2\x0f.MFFile.VersionR\aversion\x12!\n" +
+	"\x05files\x18e \x03(\v2\v.MFFilePathR\x05files\x12.\n" +
+	"\tcreatedAt\x18\xc9\x01 \x01(\v2\n" +
+	".TimestampH\x00R\tcreatedAt\x88\x01\x01\",\n" +
+	"\aVersion\x12\x10\n" +
+	"\fVERSION_NONE\x10\x00\x12\x0f\n" +
+	"\vVERSION_ONE\x10\x01B\f\n" +
+	"\n" +
+	"_createdAtB\x1dZ\x1bgit.eeqj.de/sneak/mfer/mferb\x06proto3"
+
+var (
+	file_mf_proto_rawDescOnce sync.Once
+	file_mf_proto_rawDescData []byte
+)
+
+func file_mf_proto_rawDescGZIP() []byte {
+	file_mf_proto_rawDescOnce.Do(func() {
+		file_mf_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)))
+	})
+	return file_mf_proto_rawDescData
+}
+
+var file_mf_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_mf_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_mf_proto_goTypes = []any{
+	(MFFileOuter_Version)(0),         // 0: MFFileOuter.Version
+	(MFFileOuter_CompressionType)(0), // 1: MFFileOuter.CompressionType
+	(MFFile_Version)(0),              // 2: MFFile.Version
+	(*Timestamp)(nil),                // 3: Timestamp
+	(*MFFileOuter)(nil),              // 4: MFFileOuter
+	(*MFFilePath)(nil),               // 5: MFFilePath
+	(*MFFileChecksum)(nil),           // 6: MFFileChecksum
+	(*MFFile)(nil),                   // 7: MFFile
+}
+var file_mf_proto_depIdxs = []int32{
+	0, // 0: MFFileOuter.version:type_name -> MFFileOuter.Version
+	1, // 1: MFFileOuter.compressionType:type_name -> MFFileOuter.CompressionType
+	6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum
+	3, // 3: MFFilePath.mtime:type_name -> Timestamp
+	3, // 4: MFFilePath.ctime:type_name -> Timestamp
+	3, // 5: MFFilePath.atime:type_name -> Timestamp
+	2, // 6: MFFile.version:type_name -> MFFile.Version
+	5, // 7: MFFile.files:type_name -> MFFilePath
+	3, // 8: MFFile.createdAt:type_name -> Timestamp
+	9, // [9:9] is the sub-list for method output_type
+	9, // [9:9] is the sub-list for method input_type
+	9, // [9:9] is the sub-list for extension type_name
+	9, // [9:9] is the sub-list for extension extendee
+	0, // [0:9] is the sub-list for field type_name
+}
+
+func init() { file_mf_proto_init() }
+func file_mf_proto_init() {
+	if File_mf_proto != nil {
+		return
+	}
+	file_mf_proto_msgTypes[1].OneofWrappers = []any{}
+	file_mf_proto_msgTypes[2].OneofWrappers = []any{}
+	file_mf_proto_msgTypes[4].OneofWrappers = []any{}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)),
+			NumEnums:      3,
+			NumMessages:   5,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_mf_proto_goTypes,
+		DependencyIndexes: file_mf_proto_depIdxs,
+		EnumInfos:         file_mf_proto_enumTypes,
+		MessageInfos:      file_mf_proto_msgTypes,
+	}.Build()
+	File_mf_proto = out.File
+	file_mf_proto_goTypes = nil
+	file_mf_proto_depIdxs = nil
+}

mfer/mf.proto

@@ -18,7 +18,7 @@ message MFFileOuter {
 
     enum CompressionType {
         COMPRESSION_NONE = 0;
-        COMPRESSION_GZIP = 1;
+        COMPRESSION_ZSTD = 1;
     }
 
     CompressionType compressionType = 102;

mfer/serialize.go

@@ -2,11 +2,11 @@ package mfer
 
 import (
 	"bytes"
-	"compress/gzip"
 	"crypto/sha256"
 	"errors"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -56,23 +56,23 @@ func (m *manifest) generateOuter() error {
 	h.Write(innerData)
 
 	idc := new(bytes.Buffer)
-	gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
+	zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
 	if err != nil {
 		return err
 	}
-	_, err = gzw.Write(innerData)
+	_, err = zw.Write(innerData)
 	if err != nil {
 		return err
 	}
 
-	_ = gzw.Close()
+	_ = zw.Close()
 
 	o := &MFFileOuter{
 		InnerMessage:    idc.Bytes(),
 		Size:            int64(len(innerData)),
 		Sha256:          h.Sum(nil),
 		Version:         MFFileOuter_VERSION_ONE,
-		CompressionType: MFFileOuter_COMPRESSION_GZIP,
+		CompressionType: MFFileOuter_COMPRESSION_ZSTD,
 	}
 	m.pbOuter = o
 	return nil