Add UUID to manifest and verify integrity before decompression

- Add UUID field to both inner and outer manifest messages
- Generate random v4 UUID when creating manifest
- Hash compressed data (not uncompressed) for integrity check
- Verify hash before decompression to prevent malicious payloads
- Validate UUIDs are proper format and match between inner/outer
- Sign string format: MAGIC-UUID-MULTIHASH

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/apex/log v1.9.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dustin/go-humanize v1.0.1
+	github.com/google/uuid v1.1.2
 	github.com/klauspost/compress v1.18.2
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/pterm/pterm v0.12.35

go.sum

@@ -135,6 +135,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=

mfer/deserialize.go

@@ -2,9 +2,11 @@ package mfer
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"errors"
 	"io"
 
+	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/afero"
 	"google.golang.org/protobuf/proto"
@@ -12,6 +14,19 @@ import (
 	"sneak.berlin/go/mfer/internal/log"
 )
 
+// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
+func validateUUID(data []byte) error {
+	if len(data) != 16 {
+		return errors.New("invalid UUID length")
+	}
+	// Try to parse as UUID to validate format
+	_, err := uuid.FromBytes(data)
+	if err != nil {
+		return errors.New("invalid UUID format")
+	}
+	return nil
+}
+
 func (m *manifest) deserializeInner() error {
 	if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
 		return errors.New("unknown version")
@@ -20,6 +35,20 @@ func (m *manifest) deserializeInner() error {
 		return errors.New("unknown compression type")
 	}
 
+	// Validate outer UUID before any decompression
+	if err := validateUUID(m.pbOuter.Uuid); err != nil {
+		return errors.New("outer UUID invalid: " + err.Error())
+	}
+
+	// Verify hash of compressed data before decompression
+	h := sha256.New()
+	if _, err := h.Write(m.pbOuter.InnerMessage); err != nil {
+		return err
+	}
+	if !bytes.Equal(h.Sum(nil), m.pbOuter.Sha256) {
+		return errors.New("compressed data hash mismatch")
+	}
+
 	bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
 
 	zr, err := zstd.NewReader(bb)
@@ -45,6 +74,16 @@ func (m *manifest) deserializeInner() error {
 		return err
 	}
 
+	// Validate inner UUID
+	if err := validateUUID(m.pbInner.Uuid); err != nil {
+		return errors.New("inner UUID invalid: " + err.Error())
+	}
+
+	// Verify UUIDs match
+	if !bytes.Equal(m.pbOuter.Uuid, m.pbInner.Uuid) {
+		return errors.New("outer and inner UUID mismatch")
+	}
+
 	log.Infof("loaded manifest with %d files", len(m.pbInner.Files))
 	return nil
 }

mfer/mf.pb.go

@@ -218,8 +218,10 @@ type MFFileOuter struct {
 	CompressionType MFFileOuter_CompressionType `protobuf:"varint,102,opt,name=compressionType,proto3,enum=MFFileOuter_CompressionType" json:"compressionType,omitempty"`
 	// these are used solely to detect corruption/truncation
 	// and not for cryptographic integrity.
-	Size         int64  `protobuf:"varint,103,opt,name=size,proto3" json:"size,omitempty"`
+	Size   int64  `protobuf:"varint,103,opt,name=size,proto3" json:"size,omitempty"`
-	Sha256       []byte `protobuf:"bytes,104,opt,name=sha256,proto3" json:"sha256,omitempty"`
+	Sha256 []byte `protobuf:"bytes,104,opt,name=sha256,proto3" json:"sha256,omitempty"`
+	// uuid must match the uuid in the inner message
+	Uuid         []byte `protobuf:"bytes,105,opt,name=uuid,proto3" json:"uuid,omitempty"`
 	InnerMessage []byte `protobuf:"bytes,199,opt,name=innerMessage,proto3" json:"innerMessage,omitempty"`
 	// detached signature, ascii or binary
 	Signature []byte `protobuf:"bytes,201,opt,name=signature,proto3,oneof" json:"signature,omitempty"`
@@ -289,6 +291,13 @@ func (x *MFFileOuter) GetSha256() []byte {
 	return nil
 }
 
+func (x *MFFileOuter) GetUuid() []byte {
+	if x != nil {
+		return x.Uuid
+	}
+	return nil
+}
+
 func (x *MFFileOuter) GetInnerMessage() []byte {
 	if x != nil {
 		return x.InnerMessage
@@ -463,6 +472,9 @@ type MFFile struct {
 	Version MFFile_Version         `protobuf:"varint,100,opt,name=version,proto3,enum=MFFile_Version" json:"version,omitempty"`
 	// required manifest attributes:
 	Files []*MFFilePath `protobuf:"bytes,101,rep,name=files,proto3" json:"files,omitempty"`
+	// uuid is a random v4 UUID generated when creating the manifest
+	// used as part of the signature to prevent replay attacks
+	Uuid []byte `protobuf:"bytes,102,opt,name=uuid,proto3" json:"uuid,omitempty"`
 	// optional manifest attributes 2xx:
 	CreatedAt     *Timestamp `protobuf:"bytes,201,opt,name=createdAt,proto3,oneof" json:"createdAt,omitempty"`
 	unknownFields protoimpl.UnknownFields
@@ -513,6 +525,13 @@ func (x *MFFile) GetFiles() []*MFFilePath {
 	return nil
 }
 
+func (x *MFFile) GetUuid() []byte {
+	if x != nil {
+		return x.Uuid
+	}
+	return nil
+}
+
 func (x *MFFile) GetCreatedAt() *Timestamp {
 	if x != nil {
 		return x.CreatedAt
@@ -527,12 +546,13 @@ const file_mf_proto_rawDesc = "" +
 	"\bmf.proto\";\n" +
 	"\tTimestamp\x12\x18\n" +
 	"\aseconds\x18\x01 \x01(\x03R\aseconds\x12\x14\n" +
-	"\x05nanos\x18\x02 \x01(\x05R\x05nanos\"\xdc\x03\n" +
+	"\x05nanos\x18\x02 \x01(\x05R\x05nanos\"\xf0\x03\n" +
 	"\vMFFileOuter\x12.\n" +
 	"\aversion\x18e \x01(\x0e2\x14.MFFileOuter.VersionR\aversion\x12F\n" +
 	"\x0fcompressionType\x18f \x01(\x0e2\x1c.MFFileOuter.CompressionTypeR\x0fcompressionType\x12\x12\n" +
 	"\x04size\x18g \x01(\x03R\x04size\x12\x16\n" +
-	"\x06sha256\x18h \x01(\fR\x06sha256\x12#\n" +
+	"\x06sha256\x18h \x01(\fR\x06sha256\x12\x12\n" +
+	"\x04uuid\x18i \x01(\fR\x04uuid\x12#\n" +
 	"\finnerMessage\x18\xc7\x01 \x01(\fR\finnerMessage\x12\"\n" +
 	"\tsignature\x18\xc9\x01 \x01(\fH\x00R\tsignature\x88\x01\x01\x12\x1c\n" +
 	"\x06signer\x18\xca\x01 \x01(\fH\x01R\x06signer\x88\x01\x01\x12*\n" +
@@ -564,10 +584,11 @@ const file_mf_proto_rawDesc = "" +
 	"\x06_ctimeB\b\n" +
 	"\x06_atime\".\n" +
 	"\x0eMFFileChecksum\x12\x1c\n" +
-	"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xc2\x01\n" +
+	"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xd6\x01\n" +
 	"\x06MFFile\x12)\n" +
 	"\aversion\x18d \x01(\x0e2\x0f.MFFile.VersionR\aversion\x12!\n" +
-	"\x05files\x18e \x03(\v2\v.MFFilePathR\x05files\x12.\n" +
+	"\x05files\x18e \x03(\v2\v.MFFilePathR\x05files\x12\x12\n" +
+	"\x04uuid\x18f \x01(\fR\x04uuid\x12.\n" +
 	"\tcreatedAt\x18\xc9\x01 \x01(\v2\n" +
 	".TimestampH\x00R\tcreatedAt\x88\x01\x01\",\n" +
 	"\aVersion\x12\x10\n" +

mfer/mf.proto

@@ -28,6 +28,9 @@ message MFFileOuter {
     int64 size = 103;
     bytes sha256 = 104;
 
+    // uuid must match the uuid in the inner message
+    bytes uuid = 105;
+
     bytes innerMessage = 199;
     // 2xx for optional manifest root attributes
     // think we might use gosignify instead of gpg:
@@ -72,6 +75,10 @@ message MFFile {
     // required manifest attributes:
     repeated MFFilePath files = 101;
 
+    // uuid is a random v4 UUID generated when creating the manifest
+    // used as part of the signature to prevent replay attacks
+    bytes uuid = 102;
+
     // optional manifest attributes 2xx:
     optional Timestamp createdAt = 201;
 }

mfer/serialize.go

@@ -3,10 +3,12 @@ package mfer
 import (
 	"bytes"
 	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/klauspost/compress/zstd"
 	"github.com/multiformats/go-multihash"
 	"google.golang.org/protobuf/proto"
@@ -49,17 +51,17 @@ func (m *manifest) generateOuter() error {
 	if m.pbInner == nil {
 		return errors.New("internal error")
 	}
+
+	// Generate UUID and set on inner message
+	manifestUUID := uuid.New()
+	m.pbInner.Uuid = manifestUUID[:]
+
 	innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
 	if err != nil {
 		return err
 	}
 
-	h := sha256.New()
+	// Compress the inner data
-	if _, err := h.Write(innerData); err != nil {
-		return err
-	}
-	sha256Hash := h.Sum(nil)
-
 	idc := new(bytes.Buffer)
 	zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
 	if err != nil {
@@ -69,26 +71,40 @@ func (m *manifest) generateOuter() error {
 	if err != nil {
 		return err
 	}
-
 	_ = zw.Close()
 
+	compressedData := idc.Bytes()
+
+	// Hash the compressed data for integrity verification before decompression
+	h := sha256.New()
+	if _, err := h.Write(compressedData); err != nil {
+		return err
+	}
+	sha256Hash := h.Sum(nil)
+
 	o := &MFFileOuter{
-		InnerMessage:    idc.Bytes(),
+		InnerMessage:    compressedData,
 		Size:            int64(len(innerData)),
 		Sha256:          sha256Hash,
+		Uuid:            manifestUUID[:],
 		Version:         MFFileOuter_VERSION_ONE,
 		CompressionType: MFFileOuter_COMPRESSION_ZSTD,
 	}
 
 	// Sign the manifest if signing options are provided
 	if m.signingOptions != nil && m.signingOptions.KeyID != "" {
-		// Encode hash as multihash for signing
+		// Encode hash as multihash
 		mh, err := multihash.Encode(sha256Hash, multihash.SHA2_256)
 		if err != nil {
 			return fmt.Errorf("failed to encode multihash: %w", err)
 		}
 
-		sig, err := gpgSign(mh, m.signingOptions.KeyID)
+		// Build signature string: MAGIC-UUID-MULTIHASH
+		uuidStr := hex.EncodeToString(manifestUUID[:])
+		mhStr := hex.EncodeToString(mh)
+		sigString := fmt.Sprintf("%s-%s-%s", MAGIC, uuidStr, mhStr)
+
+		sig, err := gpgSign([]byte(sigString), m.signingOptions.KeyID)
 		if err != nil {
 			return fmt.Errorf("failed to sign manifest: %w", err)
 		}