mfer/mfer/deserialize.go

package mfer

import (
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"

	"github.com/google/uuid"
	"github.com/klauspost/compress/zstd"
	"github.com/spf13/afero"
	"google.golang.org/protobuf/proto"
	"sneak.berlin/go/mfer/internal/bork"
	"sneak.berlin/go/mfer/internal/log"
)

var (
	errInvalidUUIDLength    = errors.New("invalid UUID length")
	errInvalidUUIDFormat    = errors.New("invalid UUID format")
	errUnknownVersion       = errors.New("unknown version")
	errUnknownCompression   = errors.New("unknown compression type")
	errCompressedHashWrong  = errors.New("compressed data hash mismatch")
	errSignatureNoPubKey    = errors.New("signature present but no public key")
	errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
	errUUIDMismatch         = errors.New("outer and inner UUID mismatch")
	errInvalidFileFormat    = errors.New("invalid file format")
	errInvalidManifestPath  = errors.New("manifest contains invalid path")
)

// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
func validateUUID(data []byte) error {
	if len(data) != uuidLength {
		return errInvalidUUIDLength
	}
	// Try to parse as UUID to validate format
	_, err := uuid.FromBytes(data)
	if err != nil {
		return errInvalidUUIDFormat
	}

	return nil
}

// validateOuterHeader checks the outer message's version, compression
// type, and UUID.
func (m *manifest) validateOuterHeader() error {
	if m.pbOuter.GetVersion() != MFFileOuter_VERSION_ONE {
		return errUnknownVersion
	}

	if m.pbOuter.GetCompressionType() != MFFileOuter_COMPRESSION_ZSTD {
		return errUnknownCompression
	}

	// Validate outer UUID before any decompression
	err := validateUUID(m.pbOuter.GetUuid())
	if err != nil {
		return fmt.Errorf("outer UUID invalid: %w", err)
	}

	return nil
}

// verifyOuterIntegrity checks the hash of the compressed payload and,
// if a signature is present, verifies it against the embedded public key.
func (m *manifest) verifyOuterIntegrity() error {
	h := sha256.New()

	_, err := h.Write(m.pbOuter.GetInnerMessage())
	if err != nil {
		return fmt.Errorf("deserialize: hash write: %w", err)
	}

	sha256Hash := h.Sum(nil)
	if !bytes.Equal(sha256Hash, m.pbOuter.GetSha256()) {
		return errCompressedHashWrong
	}

	if len(m.pbOuter.GetSignature()) == 0 {
		return nil
	}

	if len(m.pbOuter.GetSigningPubKey()) == 0 {
		return errSignatureNoPubKey
	}

	sigString, err := m.signatureString()
	if err != nil {
		return fmt.Errorf(
			"failed to generate signature string for verification: %w", err,
		)
	}

	// Loading a manifest takes no context; gpgTimeout still bounds gpg.
	err = gpgVerify(
		context.Background(),
		[]byte(sigString),
		m.pbOuter.GetSignature(),
		m.pbOuter.GetSigningPubKey(),
	)
	if err != nil {
		return fmt.Errorf("signature verification failed: %w", err)
	}

	log.Infof("signature verified successfully")

	return nil
}

// decompressInner decompresses the inner payload, enforcing size limits
// to prevent decompression bombs.
func (m *manifest) decompressInner() ([]byte, error) {
	bb := bytes.NewBuffer(m.pbOuter.GetInnerMessage())

	// By default the decoder decodes a payload under 128 KiB in full,
	// each frame up to the decoder's limit, before the LimitReader below
	// reads any of it. Decoding synchronously and never in full makes it
	// decode only what the LimitReader asks for. It also sets aside a new
	// buffer for each frame that asks for a larger window than the frames
	// before it, even a frame holding no data. Refusing windows above
	// zstdWindowSize keeps each buffer to a little over zstdWindowSize, and
	// the buffers of frames holding no data to about 16 times it in total.
	zr, err := zstd.NewReader(bb,
		zstd.WithDecoderConcurrency(1),
		zstd.WithDecodeBuffersBelow(0),
		zstd.WithDecoderMaxWindow(zstdWindowSize))
	if err != nil {
		return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
	}
	defer zr.Close()

	// Limit decompressed size to prevent decompression bombs.
	// Use declared size + 1 byte to detect overflow, capped at MaxDecompressedSize.
	maxSize := MaxDecompressedSize
	if m.pbOuter.GetSize() > 0 && m.pbOuter.GetSize() < maxSize {
		maxSize = m.pbOuter.GetSize() + 1
	}

	limitedReader := io.LimitReader(zr, maxSize)

	dat, err := io.ReadAll(limitedReader)
	if err != nil {
		return nil, fmt.Errorf("deserialize: decompress: %w", err)
	}

	if int64(len(dat)) >= MaxDecompressedSize {
		return nil, fmt.Errorf(
			"%w of %d bytes", errDecompressedTooLarge, MaxDecompressedSize,
		)
	}

	return dat, nil
}

func (m *manifest) deserializeInner() error {
	err := m.validateOuterHeader()
	if err != nil {
		return err
	}

	err = m.verifyOuterIntegrity()
	if err != nil {
		return err
	}

	dat, err := m.decompressInner()
	if err != nil {
		return err
	}

	isize := len(dat)
	if int64(isize) != m.pbOuter.GetSize() {
		log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.GetSize())

		return bork.ErrFileTruncated
	}

	// Deserialize inner message
	m.pbInner = new(MFFile)

	err = proto.Unmarshal(dat, m.pbInner)
	if err != nil {
		return fmt.Errorf("deserialize: unmarshal inner: %w", err)
	}

	// Validate inner UUID
	err = validateUUID(m.pbInner.GetUuid())
	if err != nil {
		return fmt.Errorf("inner UUID invalid: %w", err)
	}

	// Verify UUIDs match
	if !bytes.Equal(m.pbOuter.GetUuid(), m.pbInner.GetUuid()) {
		return errUUIDMismatch
	}

	// Enforce the manifest path invariants on every entry as it is loaded,
	// so that no consumer of a manifest — Checker today, any restore or
	// extract path tomorrow — acts on a traversal or absolute path from an
	// untrusted .mf. Reject loudly on the first offender rather than
	// dropping entries, which would let a hostile manifest hide files from a
	// check.
	for _, f := range m.pbInner.GetFiles() {
		err = ValidatePath(f.GetPath())
		if err != nil {
			return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
		}
	}

	log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))

	return nil
}

func validateMagic(dat []byte) bool {
	ml := len([]byte(MAGIC))
	if len(dat) < ml {
		return false
	}

	got := dat[0:ml]
	expected := []byte(MAGIC)

	return bytes.Equal(got, expected)
}

// NewManifestFromReader reads a manifest from an io.Reader.
//
//nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromReader(input io.Reader) (*manifest, error) {
	m := &manifest{}

	dat, err := io.ReadAll(input)
	if err != nil {
		return nil, err
	}

	if !validateMagic(dat) {
		return nil, errInvalidFileFormat
	}

	// remove magic bytes prefix:
	ml := len([]byte(MAGIC))
	bb := bytes.NewBuffer(dat[ml:])
	dat = bb.Bytes()

	// deserialize outer:
	m.pbOuter = new(MFFileOuter)

	err = proto.Unmarshal(dat, m.pbOuter)
	if err != nil {
		return nil, err
	}

	// deserialize inner:
	err = m.deserializeInner()
	if err != nil {
		return nil, err
	}

	return m, nil
}

// NewManifestFromFile reads a manifest from a file path using the given filesystem.
// If fs is nil, the real filesystem (OsFs) is used.
//
//nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
	if fs == nil {
		fs = afero.NewOsFs()
	}

	f, err := fs.Open(path)
	if err != nil {
		return nil, err
	}

	defer func() { _ = f.Close() }()

	return NewManifestFromReader(f)
}