From f7886682876d163092a66b53b587d3070f588f72 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 22 Sep 2026 13:46:02 +0200 Subject: [PATCH] Remove the unused crypto path and write the blob-ID hash step once (closes #151) Production encryption and decryption already run through blobgen; the crypto package (Encryptor, Decryptor, UpdateRecipients, the fx Module) and Vaultik.GetEncryptor/GetDecryptor had no production caller. Delete crypto and route verify --deep through the same blobgen reader restore uses, parsing the age key once. The second blob-ID hash step is now one exported blobgen.DoubleSHA256; Writer.Sum256 (the double hash) becomes Writer.ContentID so it no longer collides with Reader.Sum256 (the single plaintext hash). Also delete the never-adopted internal/types newtypes and the uncalled CleanupIncompleteSnapshots and its now-dead deleteSnapshot caller, and correct ARCHITECTURE.md. No production behavior changes. Model: opus-4-8 --- ARCHITECTURE.md | 3 +- internal/blob/packer.go | 2 +- internal/blobgen/compress.go | 89 --------- internal/blobgen/compress_test.go | 80 -------- internal/blobgen/reader.go | 4 +- internal/blobgen/reader_test.go | 54 ++++++ internal/blobgen/writer.go | 32 ++-- internal/blobgen/writer_test.go | 10 +- internal/crypto/encryption.go | 233 ----------------------- internal/crypto/encryption_test.go | 198 ------------------- internal/snapshot/snapshot.go | 103 +--------- internal/types/types.go | 73 ++----- internal/vaultik/blob_fetch.go | 7 +- internal/vaultik/blob_fetch_hash_test.go | 4 +- internal/vaultik/snapshot.go | 4 +- internal/vaultik/vaultik.go | 28 --- internal/vaultik/verify.go | 106 +++++------ internal/vaultik/verify_test.go | 100 ---------- 18 files changed, 154 insertions(+), 976 deletions(-) delete mode 100644 internal/blobgen/compress.go delete mode 100644 internal/blobgen/compress_test.go create mode 100644 internal/blobgen/reader_test.go delete mode 100644 internal/crypto/encryption.go delete mode 100644 internal/crypto/encryption_test.go delete mode 100644 internal/vaultik/verify_test.go diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 17d7cf4..d44746a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -286,7 +286,6 @@ Key methods: - `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record - `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete - `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3 -- `CleanupIncompleteSnapshots(ctx, hostname)` → Remove failed snapshots ### `internal/database` SQLite database for local index. Single-writer mode for thread safety. @@ -307,7 +306,7 @@ Repository interfaces: ``` CreateSnapshot(opts) │ - ├─► CleanupIncompleteSnapshots() // Critical: avoid dedup errors + ├─► PruneDatabase() // Critical: avoid dedup errors │ ├─► SnapshotManager.CreateSnapshot() // Create DB record │ diff --git a/internal/blob/packer.go b/internal/blob/packer.go index cfbb081..d4704b5 100644 --- a/internal/blob/packer.go +++ b/internal/blob/packer.go @@ -487,7 +487,7 @@ func (p *Packer) closeBlobWriter() (string, int64, error) { return "", 0, fmt.Errorf("seeking to start: %w", err) } - finalHash := p.currentBlob.writer.Sum256() + finalHash := p.currentBlob.writer.ContentID() return hex.EncodeToString(finalHash), finalSize, nil } diff --git a/internal/blobgen/compress.go b/internal/blobgen/compress.go deleted file mode 100644 index 40be29c..0000000 --- a/internal/blobgen/compress.go +++ /dev/null @@ -1,89 +0,0 @@ -// Package blobgen implements the blob data pipeline: streaming zstd -// compression, age encryption, and SHA256 content hashing for blob -// creation, plus the matching decrypt/decompress/verify reader. -package blobgen - -import ( - "bytes" - "encoding/hex" - "fmt" - "io" -) - -// CompressResult contains the results of compression -type CompressResult struct { - Data []byte - UncompressedSize int64 - CompressedSize int64 - SHA256 string -} - -// CompressData compresses and encrypts data, returning the result with hash -func CompressData( - data []byte, compressionLevel int, recipients []string, -) (*CompressResult, error) { - var buf bytes.Buffer - - // Create writer - w, err := NewWriter(&buf, compressionLevel, recipients) - if err != nil { - return nil, fmt.Errorf("creating writer: %w", err) - } - - // Write data - _, err = w.Write(data) - if err != nil { - _ = w.Close() - - return nil, fmt.Errorf("writing data: %w", err) - } - - // Close to flush - err = w.Close() - if err != nil { - return nil, fmt.Errorf("closing writer: %w", err) - } - - return &CompressResult{ - Data: buf.Bytes(), - UncompressedSize: int64(len(data)), - CompressedSize: int64(buf.Len()), - SHA256: hex.EncodeToString(w.Sum256()), - }, nil -} - -// CompressStream compresses and encrypts from reader to writer, returning -// the number of uncompressed bytes written and the content hash. -func CompressStream( - dst io.Writer, src io.Reader, compressionLevel int, recipients []string, -) (int64, string, error) { - // Create writer - w, err := NewWriter(dst, compressionLevel, recipients) - if err != nil { - return 0, "", fmt.Errorf("creating writer: %w", err) - } - - closed := false - - defer func() { - if !closed { - _ = w.Close() - } - }() - - // Copy data - _, err = io.Copy(w, src) - if err != nil { - return 0, "", fmt.Errorf("copying data: %w", err) - } - - // Close to flush - err = w.Close() - if err != nil { - return 0, "", fmt.Errorf("closing writer: %w", err) - } - - closed = true - - return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil -} diff --git a/internal/blobgen/compress_test.go b/internal/blobgen/compress_test.go deleted file mode 100644 index b019f11..0000000 --- a/internal/blobgen/compress_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package blobgen_test - -import ( - "bytes" - "crypto/rand" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "sneak.berlin/go/vaultik/internal/blobgen" -) - -// testRecipient is a static age recipient for tests. -const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv" - -// TestCompressStreamNoDoubleClose is a regression test for issue #28. -// It verifies that CompressStream does not panic or return an error due to -// double-closing the underlying blobgen.Writer. Before the fix in PR #33, -// the explicit Close() on the happy path combined with defer Close() would -// cause a double close. -func TestCompressStreamNoDoubleClose(t *testing.T) { - t.Parallel() - - input := []byte("regression test data for issue #28 double-close fix") - - var buf bytes.Buffer - - written, hash, err := blobgen.CompressStream( - &buf, bytes.NewReader(input), 3, []string{testRecipient}) - require.NoError(t, err, "CompressStream should not return an error") - assert.Positive(t, written, "expected bytes written > 0") - assert.NotEmpty(t, hash, "expected non-empty hash") - assert.Positive(t, buf.Len(), "expected non-empty output") -} - -// TestCompressStreamLargeInput exercises CompressStream with a larger payload -// to ensure no double-close issues surface under heavier I/O. -func TestCompressStreamLargeInput(t *testing.T) { - t.Parallel() - - data := make([]byte, 512*1024) // 512 KB - _, err := rand.Read(data) - require.NoError(t, err) - - var buf bytes.Buffer - - written, hash, err := blobgen.CompressStream( - &buf, bytes.NewReader(data), 3, []string{testRecipient}) - require.NoError(t, err) - assert.Positive(t, written) - assert.NotEmpty(t, hash) -} - -// TestCompressStreamEmptyInput verifies CompressStream handles empty input -// without double-close issues. -func TestCompressStreamEmptyInput(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - - _, hash, err := blobgen.CompressStream( - &buf, strings.NewReader(""), 3, []string{testRecipient}) - require.NoError(t, err) - assert.NotEmpty(t, hash) -} - -// TestCompressDataNoDoubleClose mirrors the stream test for CompressData, -// ensuring the explicit Close + error-path Close pattern is also safe. -func TestCompressDataNoDoubleClose(t *testing.T) { - t.Parallel() - - input := []byte("CompressData regression test for double-close") - - result, err := blobgen.CompressData(input, 3, []string{testRecipient}) - require.NoError(t, err) - assert.Positive(t, result.CompressedSize) - assert.Equal(t, result.UncompressedSize, int64(len(input))) - assert.NotEmpty(t, result.SHA256) -} diff --git a/internal/blobgen/reader.go b/internal/blobgen/reader.go index 1a7dc11..94f3b90 100644 --- a/internal/blobgen/reader.go +++ b/internal/blobgen/reader.go @@ -64,7 +64,9 @@ func (r *Reader) Close() error { return nil } -// Sum256 returns the SHA256 hash of all data read +// Sum256 returns the single SHA-256 of the plaintext read so far. This is the +// first hash only; the stored object name is its double hash, which callers +// obtain by passing this digest to DoubleSHA256. func (r *Reader) Sum256() []byte { return r.hasher.Sum(nil) } diff --git a/internal/blobgen/reader_test.go b/internal/blobgen/reader_test.go new file mode 100644 index 0000000..48c4c21 --- /dev/null +++ b/internal/blobgen/reader_test.go @@ -0,0 +1,54 @@ +package blobgen_test + +import ( + "bytes" + "io" + "testing" + + "filippo.io/age" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/blobgen" +) + +// TestMultipleRecipients verifies that data written for several recipients can +// be read back by each recipient's identity. Moved from internal/crypto, which +// held the only multi-recipient test; blobgen is now the sole encryption path. +func TestMultipleRecipients(t *testing.T) { + t.Parallel() + + identities := make([]*age.X25519Identity, 3) + recipients := make([]string, 3) + + for i := range identities { + identity, err := age.GenerateX25519Identity() + require.NoError(t, err) + + identities[i] = identity + recipients[i] = identity.Recipient().String() + } + + plaintext := []byte("Secret message for multiple recipients") + + var encrypted bytes.Buffer + + writer, err := blobgen.NewWriter(&encrypted, 3, recipients) + require.NoError(t, err) + _, err = writer.Write(plaintext) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + // Every recipient's identity must recover the original plaintext. + for i, identity := range identities { + reader, err := blobgen.NewReader( + bytes.NewReader(encrypted.Bytes()), identity) + require.NoError(t, err, "recipient %d should open the reader", i+1) + + got, err := io.ReadAll(reader) + require.NoError(t, err, "recipient %d should read the plaintext", i+1) + require.NoError(t, reader.Close()) + + assert.Equal(t, plaintext, got, + "recipient %d should recover the original plaintext", i+1) + } +} diff --git a/internal/blobgen/writer.go b/internal/blobgen/writer.go index 8525df9..728d829 100644 --- a/internal/blobgen/writer.go +++ b/internal/blobgen/writer.go @@ -1,3 +1,6 @@ +// Package blobgen implements the blob data pipeline: streaming zstd +// compression, age encryption, and SHA256 content hashing for blob +// creation, plus the matching decrypt/decompress/verify reader. package blobgen import ( @@ -12,6 +15,18 @@ import ( "github.com/klauspost/compress/zstd" ) +// DoubleSHA256 returns the double SHA-256 of content whose single SHA-256 +// digest is sum: it hashes that digest once more. Stored objects are named by +// this second hash so that a name never reveals whether known content is +// present — an attacker who knows a plaintext, and thus its SHA-256, still +// cannot derive the stored name without hashing the digest again. Both a blob +// and the metadata database export are named this way. +func DoubleSHA256(sum []byte) []byte { + h := sha256.Sum256(sum) + + return h[:] +} + // Zstd compression level bounds accepted by NewWriter. const ( minCompressionLevel = 1 @@ -130,17 +145,12 @@ func (w *Writer) Close() error { return nil } -// Sum256 returns the double SHA256 hash of the uncompressed input data. -// Double hashing (SHA256(SHA256(data))) prevents information leakage about -// the plaintext - an attacker cannot confirm existence of known content -// by computing its hash and checking for a matching blob filename. -func (w *Writer) Sum256() []byte { - // First hash: SHA256(plaintext) - firstHash := w.hasher.Sum(nil) - // Second hash: SHA256(firstHash) - this is the blob ID - secondHash := sha256.Sum256(firstHash) - - return secondHash[:] +// ContentID returns the double SHA-256 of the uncompressed input data: the +// name under which this content is stored. It is the second hash of the +// running SHA-256, via DoubleSHA256; see that function for why content is +// named this way rather than by its plain SHA-256. +func (w *Writer) ContentID() []byte { + return DoubleSHA256(w.hasher.Sum(nil)) } // BytesWritten returns the number of uncompressed bytes written diff --git a/internal/blobgen/writer_test.go b/internal/blobgen/writer_test.go index 8ea56fa..57573be 100644 --- a/internal/blobgen/writer_test.go +++ b/internal/blobgen/writer_test.go @@ -12,7 +12,7 @@ import ( "sneak.berlin/go/vaultik/internal/blobgen" ) -// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns +// TestWriterHashIsDoubleHash verifies that Writer.ContentID() returns // the double hash SHA256(SHA256(plaintext)) for security. // Double hashing prevents attackers from confirming existence of known content. func TestWriterHashIsDoubleHash(t *testing.T) { @@ -43,7 +43,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) { require.NoError(t, err) // Get the hash from the writer - writerHash := hex.EncodeToString(writer.Sum256()) + writerHash := hex.EncodeToString(writer.ContentID()) // Calculate the expected double hash: SHA256(SHA256(plaintext)) firstHash := sha256.Sum256(testData) @@ -60,7 +60,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) { // The writer hash should match the double hash assert.Equal(t, expectedDoubleHash, writerHash, - "Writer.Sum256() should return SHA256(SHA256(plaintext)) for security") + "Writer.ContentID() should return SHA256(SHA256(plaintext)) for security") // Verify it's NOT the single hash (would leak information) assert.NotEqual(t, singleHashStr, writerHash, @@ -93,8 +93,8 @@ func TestWriterDeterministicHash(t *testing.T) { require.NoError(t, err) require.NoError(t, writer2.Close()) - hash1 := hex.EncodeToString(writer1.Sum256()) - hash2 := hex.EncodeToString(writer2.Sum256()) + hash1 := hex.EncodeToString(writer1.ContentID()) + hash2 := hex.EncodeToString(writer2.ContentID()) // Hashes should be identical (deterministic) assert.Equal(t, hash1, hash2, "Same input should produce same hash") diff --git a/internal/crypto/encryption.go b/internal/crypto/encryption.go deleted file mode 100644 index dd5d71d..0000000 --- a/internal/crypto/encryption.go +++ /dev/null @@ -1,233 +0,0 @@ -// Package crypto provides thread-safe age encryption and decryption -// helpers used to protect blob and metadata content. -package crypto //nolint:revive,nolintlint // stdlib crypto unused; see #76 - -import ( - "bytes" - "errors" - "fmt" - "io" - "sync" - - "filippo.io/age" - "go.uber.org/fx" -) - -// ErrNoRecipients is returned when an encryptor is created or updated -// without any recipient public keys. -var ErrNoRecipients = errors.New("at least one recipient is required") - -// errInvalidRecipient is returned when a recipient string does not parse as -// an X25519 age1... public key. It omits the value, which can be sensitive. -var errInvalidRecipient = errors.New( - "not a valid X25519 age1... recipient") - -// Encryptor provides thread-safe encryption using the age encryption library. -// It supports encrypting data for multiple recipients simultaneously, allowing -// any of the corresponding private keys to decrypt the data. This is useful -// for backup scenarios where multiple parties should be able to decrypt the data. -type Encryptor struct { - recipients []age.Recipient - mu sync.RWMutex -} - -// NewEncryptor creates a new encryptor with the given age public keys. -// Each public key should be a valid age X25519 recipient string (e.g., "age1...") -// At least one recipient must be provided. Returns an error if any of the -// public keys are invalid or if no recipients are specified. -func NewEncryptor(publicKeys []string) (*Encryptor, error) { - if len(publicKeys) == 0 { - return nil, ErrNoRecipients - } - - recipients := make([]age.Recipient, 0, len(publicKeys)) - for i, key := range publicKeys { - // The key string can be sensitive (e.g. a secret key pasted by - // mistake), so the error names its position, never its value. - recipient, err := age.ParseX25519Recipient(key) - if err != nil { - return nil, fmt.Errorf("%w: recipient %d", errInvalidRecipient, i) - } - - recipients = append(recipients, recipient) - } - - return &Encryptor{ - recipients: recipients, - }, nil -} - -// Encrypt encrypts data using age encryption for all configured recipients. -// The encrypted data can be decrypted by any of the corresponding private keys. -// This method is suitable for small to medium amounts of data that fit in memory. -// For large data streams, use EncryptStream or EncryptWriter instead. -func (e *Encryptor) Encrypt(data []byte) ([]byte, error) { - e.mu.RLock() - recipients := e.recipients - e.mu.RUnlock() - - var buf bytes.Buffer - - // Create encrypted writer for all recipients - w, err := age.Encrypt(&buf, recipients...) - if err != nil { - return nil, fmt.Errorf("creating encrypted writer: %w", err) - } - - // Write data - _, err = w.Write(data) - if err != nil { - return nil, fmt.Errorf("writing encrypted data: %w", err) - } - - // Close to flush - err = w.Close() - if err != nil { - return nil, fmt.Errorf("closing encrypted writer: %w", err) - } - - return buf.Bytes(), nil -} - -// EncryptStream encrypts data from reader to writer using age encryption. -// This method is suitable for encrypting large files or streams as it processes -// data in a streaming fashion without loading everything into memory. -// The encrypted data is written directly to the destination writer. -func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error { - e.mu.RLock() - recipients := e.recipients - e.mu.RUnlock() - - // Create encrypted writer for all recipients - w, err := age.Encrypt(dst, recipients...) - if err != nil { - return fmt.Errorf("creating encrypted writer: %w", err) - } - - // Copy data - _, err = io.Copy(w, src) - if err != nil { - return fmt.Errorf("copying encrypted data: %w", err) - } - - // Close to flush - err = w.Close() - if err != nil { - return fmt.Errorf("closing encrypted writer: %w", err) - } - - return nil -} - -// EncryptWriter creates a writer that encrypts data written to it. -// All data written to the returned WriteCloser will be encrypted and written -// to the destination writer. The caller must call Close() on the returned -// writer to ensure all encrypted data is properly flushed and finalized. -// This is useful for integrating encryption into existing writer-based pipelines. -func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) { - e.mu.RLock() - recipients := e.recipients - e.mu.RUnlock() - - // Create encrypted writer for all recipients - w, err := age.Encrypt(dst, recipients...) - if err != nil { - return nil, fmt.Errorf("creating encrypted writer: %w", err) - } - - return w, nil -} - -// UpdateRecipients updates the recipients for future encryption operations. -// This method is thread-safe and can be called while other encryption operations -// are in progress. Existing encryption operations will continue with the old -// recipients. At least one recipient must be provided. Returns an error if any -// of the public keys are invalid or if no recipients are specified. -func (e *Encryptor) UpdateRecipients(publicKeys []string) error { - if len(publicKeys) == 0 { - return ErrNoRecipients - } - - recipients := make([]age.Recipient, 0, len(publicKeys)) - for i, key := range publicKeys { - // The key string can be sensitive (e.g. a secret key pasted by - // mistake), so the error names its position, never its value. - recipient, err := age.ParseX25519Recipient(key) - if err != nil { - return fmt.Errorf("%w: recipient %d", errInvalidRecipient, i) - } - - recipients = append(recipients, recipient) - } - - e.mu.Lock() - e.recipients = recipients - e.mu.Unlock() - - return nil -} - -// Decryptor provides thread-safe decryption using the age encryption library. -// It uses a private key to decrypt data that was encrypted for the corresponding -// public key. -type Decryptor struct { - identity age.Identity - mu sync.RWMutex -} - -// NewDecryptor creates a new decryptor with the given age private key. -// The private key should be a valid age X25519 identity string. -// Returns an error if the private key is invalid. -func NewDecryptor(privateKey string) (*Decryptor, error) { - identity, err := age.ParseX25519Identity(privateKey) - if err != nil { - return nil, fmt.Errorf("parsing age identity: %w", err) - } - - return &Decryptor{ - identity: identity, - }, nil -} - -// Decrypt decrypts data using age decryption. -// This method is suitable for small to medium amounts of data that fit in memory. -// For large data streams, use DecryptStream instead. -func (d *Decryptor) Decrypt(data []byte) ([]byte, error) { - d.mu.RLock() - identity := d.identity - d.mu.RUnlock() - - r, err := age.Decrypt(bytes.NewReader(data), identity) - if err != nil { - return nil, fmt.Errorf("creating decrypted reader: %w", err) - } - - decrypted, err := io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("reading decrypted data: %w", err) - } - - return decrypted, nil -} - -// DecryptStream returns a reader that decrypts data from the provided reader. -// This method is suitable for decrypting large files or streams as it processes -// data in a streaming fashion without loading everything into memory. -// The caller should close the input reader when done. -func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) { - d.mu.RLock() - identity := d.identity - d.mu.RUnlock() - - r, err := age.Decrypt(src, identity) - if err != nil { - return nil, fmt.Errorf("creating decrypted reader: %w", err) - } - - return r, nil -} - -// Module exports the crypto module for fx dependency injection. -// -//nolint:gochecknoglobals // fx module definitions are package globals -var Module = fx.Module("crypto") diff --git a/internal/crypto/encryption_test.go b/internal/crypto/encryption_test.go deleted file mode 100644 index 90a9784..0000000 --- a/internal/crypto/encryption_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package crypto_test - -import ( - "bytes" - "strings" - "testing" - - "filippo.io/age" - "sneak.berlin/go/vaultik/internal/crypto" -) - -func TestEncryptor(t *testing.T) { - t.Parallel() - - // Generate a test key pair - identity, err := age.GenerateX25519Identity() - if err != nil { - t.Fatalf("failed to generate identity: %v", err) - } - - publicKey := identity.Recipient().String() - - // Create encryptor - enc, err := crypto.NewEncryptor([]string{publicKey}) - if err != nil { - t.Fatalf("failed to create encryptor: %v", err) - } - - // Test data - plaintext := []byte("Hello, World! This is a test message.") - - // Encrypt - ciphertext, err := enc.Encrypt(plaintext) - if err != nil { - t.Fatalf("failed to encrypt: %v", err) - } - - // Verify it's actually encrypted (should be larger and different) - if bytes.Equal(plaintext, ciphertext) { - t.Error("ciphertext equals plaintext") - } - - // Decrypt to verify - r, err := age.Decrypt(bytes.NewReader(ciphertext), identity) - if err != nil { - t.Fatalf("failed to decrypt: %v", err) - } - - var decrypted bytes.Buffer - - _, err = decrypted.ReadFrom(r) - if err != nil { - t.Fatalf("failed to read decrypted data: %v", err) - } - - if !bytes.Equal(plaintext, decrypted.Bytes()) { - t.Error("decrypted data doesn't match original") - } -} - -func TestEncryptorMultipleRecipients(t *testing.T) { - t.Parallel() - - // Generate three test key pairs - identity1, err := age.GenerateX25519Identity() - if err != nil { - t.Fatalf("failed to generate identity1: %v", err) - } - - identity2, err := age.GenerateX25519Identity() - if err != nil { - t.Fatalf("failed to generate identity2: %v", err) - } - - identity3, err := age.GenerateX25519Identity() - if err != nil { - t.Fatalf("failed to generate identity3: %v", err) - } - - publicKeys := []string{ - identity1.Recipient().String(), - identity2.Recipient().String(), - identity3.Recipient().String(), - } - - // Create encryptor with multiple recipients - enc, err := crypto.NewEncryptor(publicKeys) - if err != nil { - t.Fatalf("failed to create encryptor: %v", err) - } - - // Test data - plaintext := []byte("Secret message for multiple recipients") - - // Encrypt - ciphertext, err := enc.Encrypt(plaintext) - if err != nil { - t.Fatalf("failed to encrypt: %v", err) - } - - // Verify each recipient can decrypt - identities := []age.Identity{identity1, identity2, identity3} - for i, identity := range identities { - r, err := age.Decrypt(bytes.NewReader(ciphertext), identity) - if err != nil { - t.Fatalf("recipient %d failed to decrypt: %v", i+1, err) - } - - var decrypted bytes.Buffer - - _, err = decrypted.ReadFrom(r) - if err != nil { - t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err) - } - - if !bytes.Equal(plaintext, decrypted.Bytes()) { - t.Errorf("recipient %d: decrypted data doesn't match original", i+1) - } - } -} - -func TestEncryptorUpdateRecipients(t *testing.T) { - t.Parallel() - - // Generate two identities - identity1, _ := age.GenerateX25519Identity() - identity2, _ := age.GenerateX25519Identity() - - publicKey1 := identity1.Recipient().String() - publicKey2 := identity2.Recipient().String() - - // Create encryptor with first key - enc, err := crypto.NewEncryptor([]string{publicKey1}) - if err != nil { - t.Fatalf("failed to create encryptor: %v", err) - } - - // Encrypt with first key - plaintext := []byte("test data") - - ciphertext1, err := enc.Encrypt(plaintext) - if err != nil { - t.Fatalf("failed to encrypt: %v", err) - } - - // Update to second key - err = enc.UpdateRecipients([]string{publicKey2}) - if err != nil { - t.Fatalf("failed to update recipients: %v", err) - } - - // Encrypt with second key - ciphertext2, err := enc.Encrypt(plaintext) - if err != nil { - t.Fatalf("failed to encrypt: %v", err) - } - - // First ciphertext should only decrypt with first identity - _, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1) - if err != nil { - t.Error("failed to decrypt with identity1") - } - - _, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2) - if err == nil { - t.Error("should not decrypt with identity2") - } - - // Second ciphertext should only decrypt with second identity - _, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2) - if err != nil { - t.Error("failed to decrypt with identity2") - } - - _, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1) - if err == nil { - t.Error("should not decrypt with identity1") - } -} - -// TestNewEncryptorSecretKeyNotEchoed verifies that a secret key mistakenly -// passed as a recipient does not appear in the returned error. A recipient -// string can be sensitive, so the error must name only the position. -func TestNewEncryptorSecretKeyNotEchoed(t *testing.T) { - t.Parallel() - - secretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" + - "VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - - _, err := crypto.NewEncryptor([]string{secretKey}) - if err == nil { - t.Fatal("NewEncryptor returned nil, want error") - } - - if strings.Contains(err.Error(), secretKey) { - t.Fatalf("error echoed the recipient value: %v", err) - } -} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 2e77656..bf73bb6 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -295,68 +295,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata( return nil } -// CleanupIncompleteSnapshots removes incomplete snapshots that don't have -// metadata in S3. This is critical for data safety: incomplete snapshots -// can cause deduplication to skip files that were never successfully -// backed up, resulting in data loss. -func (sm *SnapshotManager) CleanupIncompleteSnapshots( - ctx context.Context, hostname string, -) error { - log.Info("Checking for incomplete snapshots", "hostname", hostname) - - // Get all incomplete snapshots for this hostname - incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname) - if err != nil { - return fmt.Errorf("getting incomplete snapshots: %w", err) - } - - if len(incompleteSnapshots) == 0 { - log.Debug("No incomplete snapshots found") - - return nil - } - - log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots)) - - // Check each incomplete snapshot for metadata in storage - for _, snapshot := range incompleteSnapshots { - // Check if metadata exists in storage (paths use the hashed - // remote key so we don't leak host info to the listing). - metadataKey := fmt.Sprintf("metadata/%s/db.zst", - RemoteSnapshotKey(snapshot.ID.String())) - - _, err := sm.storage.Stat(ctx, metadataKey) - if err != nil { - // Metadata doesn't exist in S3 - this is an incomplete snapshot - log.Info("Cleaning up incomplete snapshot record", - "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) - - // Delete the snapshot and all its associations - err := sm.deleteSnapshot(ctx, snapshot.ID.String()) - if err != nil { - return fmt.Errorf("deleting incomplete snapshot %s: %w", - snapshot.ID, err) - } - - log.Info("Deleted incomplete snapshot record and associated data", - "snapshot_id", snapshot.ID) - } else { - // Metadata exists - this snapshot was completed but database wasn't updated - // This shouldn't happen in normal operation, but mark it complete - log.Warn("Found snapshot with remote metadata but incomplete in database", - "snapshot_id", snapshot.ID) - - err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) - if err != nil { - log.Error("Failed to mark snapshot as complete in database", - "snapshot_id", snapshot.ID, "error", err) - } - } - } - - return nil -} - // CleanupOrphanedData removes files, chunks, and blobs that are no longer // referenced by any snapshot. This should be called periodically to clean // up data from deleted or incomplete snapshots. @@ -759,7 +697,7 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { writerClosed = true - log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256())) + log.Debug("Compression complete", "hash", hex.EncodeToString(writer.ContentID())) return nil } @@ -933,45 +871,6 @@ type ExtendedBackupStats struct { UploadDurationMs int64 // Total milliseconds spent uploading to S3 } -// deleteSnapshot removes a snapshot and all its associations from the database -func (sm *SnapshotManager) deleteSnapshot( - ctx context.Context, snapshotID string, -) error { - // Delete snapshot_files entries - err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID) - if err != nil { - return fmt.Errorf("deleting snapshot files: %w", err) - } - - // Delete snapshot_blobs entries - err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID) - if err != nil { - return fmt.Errorf("deleting snapshot blobs: %w", err) - } - - // Delete uploads entries (has foreign key to snapshots without CASCADE) - err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID) - if err != nil { - return fmt.Errorf("deleting snapshot uploads: %w", err) - } - - // Delete the snapshot itself - err = sm.repos.Snapshots.Delete(ctx, snapshotID) - if err != nil { - return fmt.Errorf("deleting snapshot: %w", err) - } - - // Clean up orphaned data - log.Debug("Cleaning up orphaned records in main database") - - err = sm.CleanupOrphanedData(ctx) - if err != nil { - return fmt.Errorf("cleaning up orphaned data: %w", err) - } - - return nil -} - // deleteOtherSnapshots deletes all snapshots except the current one func (sm *SnapshotManager) deleteOtherSnapshots( ctx context.Context, tx *sql.Tx, currentSnapshotID string, diff --git a/internal/types/types.go b/internal/types/types.go index 5310179..6f1768b 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1,7 +1,7 @@ // Package types provides custom types for better type safety across the -// vaultik codebase. Using distinct types for IDs, hashes, paths, and -// credentials prevents accidental mixing of semantically different values -// that happen to share the same underlying type. +// vaultik codebase. Using distinct types for IDs, hashes, and paths prevents +// accidental mixing of semantically different values that happen to share the +// same underlying type. package types //nolint:revive,nolintlint // rename decision tracked in #76 import ( @@ -157,34 +157,6 @@ type FilePath string // Used during restore to strip the source prefix from paths. type SourcePath string -// AgeRecipient is an age public key used for encryption. -// Format: age1... (Bech32-encoded X25519 public key) -type AgeRecipient string - -// AgeSecretKey is an age private key used for decryption. -// Format: AGE-SECRET-KEY-... (Bech32-encoded X25519 private key) -// This type should never be logged or serialized in plaintext. -type AgeSecretKey string - -// S3Endpoint is the URL of an S3-compatible storage endpoint. -type S3Endpoint string - -// BucketName is the name of an S3 bucket. -type BucketName string - -// S3Prefix is the path prefix within an S3 bucket. -type S3Prefix string - -// AWSRegion is an AWS region identifier (e.g., "us-east-1"). -type AWSRegion string - -// AWSAccessKeyID is an AWS access key ID for authentication. -type AWSAccessKeyID string - -// AWSSecretAccessKey is an AWS secret access key for authentication. -// This type should never be logged or serialized in plaintext. -type AWSSecretAccessKey string - // Hostname identifies a host machine. type Hostname string @@ -199,31 +171,14 @@ type GlobPattern string // String methods for Stringer interface -func (id FileID) String() string { return uuid.UUID(id).String() } -func (id BlobID) String() string { return uuid.UUID(id).String() } -func (id SnapshotID) String() string { return string(id) } -func (h ChunkHash) String() string { return string(h) } -func (h BlobHash) String() string { return string(h) } -func (p FilePath) String() string { return string(p) } -func (p SourcePath) String() string { return string(p) } -func (r AgeRecipient) String() string { return string(r) } -func (e S3Endpoint) String() string { return string(e) } -func (b BucketName) String() string { return string(b) } -func (p S3Prefix) String() string { return string(p) } -func (r AWSRegion) String() string { return string(r) } -func (k AWSAccessKeyID) String() string { return string(k) } -func (h Hostname) String() string { return string(h) } -func (v Version) String() string { return string(v) } -func (r GitRevision) String() string { return string(r) } -func (p GlobPattern) String() string { return string(p) } - -// Redacted String methods for sensitive types - prevents accidental logging - -func (k AgeSecretKey) String() string { return "[REDACTED]" } -func (k AWSSecretAccessKey) String() string { return "[REDACTED]" } - -// Raw returns the actual value for sensitive types when explicitly needed. -func (k AgeSecretKey) Raw() string { return string(k) } - -// Raw returns the actual value for sensitive types when explicitly needed. -func (k AWSSecretAccessKey) Raw() string { return string(k) } +func (id FileID) String() string { return uuid.UUID(id).String() } +func (id BlobID) String() string { return uuid.UUID(id).String() } +func (id SnapshotID) String() string { return string(id) } +func (h ChunkHash) String() string { return string(h) } +func (h BlobHash) String() string { return string(h) } +func (p FilePath) String() string { return string(p) } +func (p SourcePath) String() string { return string(p) } +func (h Hostname) String() string { return string(h) } +func (v Version) String() string { return string(v) } +func (r GitRevision) String() string { return string(r) } +func (p GlobPattern) String() string { return string(p) } diff --git a/internal/vaultik/blob_fetch.go b/internal/vaultik/blob_fetch.go index 045335a..8f7c001 100644 --- a/internal/vaultik/blob_fetch.go +++ b/internal/vaultik/blob_fetch.go @@ -2,7 +2,6 @@ package vaultik import ( "context" - "crypto/sha256" "encoding/hex" "errors" "fmt" @@ -57,11 +56,7 @@ func (h *hashVerifyReader) Close() error { return errBlobNotFullyRead } - firstHash := h.reader.Sum256() - secondHasher := sha256.New() - secondHasher.Write(firstHash) - - actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) + actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256())) if actualHashHex != h.blobHash { return fmt.Errorf("%w: expected %s, got %s", errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16]) diff --git a/internal/vaultik/blob_fetch_hash_test.go b/internal/vaultik/blob_fetch_hash_test.go index a9db0a3..7124942 100644 --- a/internal/vaultik/blob_fetch_hash_test.go +++ b/internal/vaultik/blob_fetch_hash_test.go @@ -40,13 +40,13 @@ func buildHashTestBlob( } // Compute the double-SHA-256 hash of the plaintext (matches - // blobgen.Writer.Sum256). + // blobgen.Writer.ContentID). firstHash := sha256.Sum256(plaintext) secondHash := sha256.Sum256(firstHash[:]) correctHash := hex.EncodeToString(secondHash[:]) // Verify our hash matches what blobgen.Writer produces - writerHash := hex.EncodeToString(writer.Sum256()) + writerHash := hex.EncodeToString(writer.ContentID()) if correctHash != writerHash { t.Fatalf("hash computation mismatch: manual=%s, writer=%s", correctHash, writerHash) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 0a24b78..9582463 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -56,8 +56,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { return err } - // Clean up incomplete snapshots FIRST, before any scanning - // This is critical for data safety - see CleanupIncompleteSnapshots for details + // Clean up incomplete snapshots FIRST, before any scanning. + // This is critical for data safety; PruneDatabase below does it. hostname := v.Config.Hostname if hostname == "" { hostname, _ = os.Hostname() diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 21783eb..7cb80ee 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -5,7 +5,6 @@ package vaultik import ( "bytes" "context" - "errors" "fmt" "io" "os" @@ -13,7 +12,6 @@ import ( "github.com/spf13/afero" "go.uber.org/fx" "sneak.berlin/go/vaultik/internal/config" - "sneak.berlin/go/vaultik/internal/crypto" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/globals" "sneak.berlin/go/vaultik/internal/snapshot" @@ -21,12 +19,6 @@ import ( "sneak.berlin/go/vaultik/internal/ui" ) -// Sentinel errors for misconfigured encryption settings. -var ( - errNoAgeRecipients = errors.New("no age recipients configured") - errNoAgeSecretKey = errors.New("no age secret key configured") -) - // Vaultik contains all dependencies needed for vaultik operations type Vaultik struct { Globals *globals.Globals @@ -141,26 +133,6 @@ func (v *Vaultik) CanDecrypt() bool { return v.Config.AgeSecretKey != "" } -// GetEncryptor creates a new Encryptor instance based on the configured age recipients -// Returns an error if no recipients are configured -func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { - if len(v.Config.AgeRecipients) == 0 { - return nil, errNoAgeRecipients - } - - return crypto.NewEncryptor(v.Config.AgeRecipients) -} - -// GetDecryptor creates a new Decryptor instance based on the configured age secret key -// Returns an error if no secret key is configured -func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) { - if v.Config.AgeSecretKey == "" { - return nil, errNoAgeSecretKey - } - - return crypto.NewDecryptor(v.Config.AgeSecretKey) -} - // GetFilesystem returns the filesystem instance used by Vaultik // //nolint:ireturn // afero.Fs is the filesystem abstraction by design diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 69ab7ed..feea3df 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -6,14 +6,14 @@ import ( "encoding/hex" "errors" "fmt" - "hash" "io" "os" "path/filepath" "time" - "github.com/klauspost/compress/zstd" + "filippo.io/age" + "sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" @@ -88,13 +88,22 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { errSecretKeyRequired.Error(), errSecretKeyRequired) } + // Parse the age secret key once, the same way restore does, and reuse + // the identity for the database and every blob. + identity, err := v.prepareRestoreIdentity() + if err != nil { + return v.deepVerifyFailure(result, opts, + fmt.Sprintf("parsing age secret key: %v", err), err) + } + log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") if !opts.JSON { v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) } - manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result) + manifest, tempDB, dbBlobs, err := v.loadVerificationData( + snapshotID, opts, result, identity) if err != nil { return err } @@ -114,7 +123,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { result.TotalSize = totalSize - err = v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize) + err = v.runVerificationSteps( + manifest, dbBlobs, tempDB, opts, result, totalSize, identity) if err != nil { return err } @@ -139,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { // loadVerificationData downloads manifest, database, and blob list for verification func (v *Vaultik) loadVerificationData( snapshotID string, opts *VerifyOptions, result *VerifyResult, + identity age.Identity, ) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { // Resolve the identifier to the snapshot's remote key. A human ID is // hashed; a remote key (or its abbreviation, as printed for a @@ -188,7 +199,7 @@ func (v *Vaultik) loadVerificationData( defer func() { _ = dbReader.Close() }() - tdb, err := v.decryptAndLoadDatabase(dbReader) + tdb, err := v.decryptAndLoadDatabase(dbReader, identity) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to decrypt database: %v", err), @@ -230,6 +241,7 @@ func (v *Vaultik) runVerificationSteps( opts *VerifyOptions, result *VerifyResult, totalSize int64, + identity age.Identity, ) error { if !opts.JSON { v.stdoutf("Verifying manifest against database...\n") @@ -256,7 +268,7 @@ func (v *Vaultik) runVerificationSteps( len(dbBlobs), ubytes(totalSize)) } - err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts) + err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identity) if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } @@ -281,26 +293,18 @@ func (t *tempDB) Close() error { } // decryptAndLoadDatabase decrypts and loads the binary SQLite database -// from the encrypted stream. -func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) { - // Get decryptor - decryptor, err := v.GetDecryptor() +// from the encrypted stream. It reads through the same blobgen reader restore +// uses, streaming the decrypted, decompressed database to a temp file. +func (v *Vaultik) decryptAndLoadDatabase( + reader io.ReadCloser, identity age.Identity, +) (*tempDB, error) { + // Decrypt and decompress through the shared blobgen reader. + blobReader, err := blobgen.NewReader(reader, identity) if err != nil { - return nil, fmt.Errorf("failed to get decryptor: %w", err) + return nil, fmt.Errorf("failed to create decryption reader: %w", err) } - // Decrypt the stream - decryptedReader, err := decryptor.DecryptStream(reader) - if err != nil { - return nil, fmt.Errorf("failed to decrypt database: %w", err) - } - - // Decompress the binary database - decompressor, err := zstd.NewReader(decryptedReader) - if err != nil { - return nil, fmt.Errorf("failed to create decompressor: %w", err) - } - defer decompressor.Close() + defer func() { _ = blobReader.Close() }() // Materialize the decrypted database inside a private (0700) temp // directory so it is never world-readable, and remove the whole @@ -330,7 +334,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) // Stream decompress directly to file log.Info("Decompressing database...") - written, err := io.Copy(tempFile, decompressor) + written, err := io.Copy(tempFile, blobReader) if err != nil { _ = tempFile.Close() @@ -355,7 +359,9 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) } // verifyBlob downloads and verifies a single blob -func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { +func (v *Vaultik) verifyBlob( + blobInfo snapshot.BlobInfo, db *sql.DB, identity age.Identity, +) error { // Download blob using shared fetch method reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize) if err != nil { @@ -364,38 +370,23 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { defer func() { _ = reader.Close() }() - // Get decryptor - decryptor, err := v.GetDecryptor() + // Decrypt and decompress through the shared blobgen reader, which hashes + // the plaintext as it is read. A blob's hash — its remote name — is the + // double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the + // encrypted bytes. + blobReader, err := blobgen.NewReader(reader, identity) if err != nil { - return fmt.Errorf("failed to get decryptor: %w", err) + return fmt.Errorf("failed to create blob reader: %w", err) } - // Decrypt blob - decryptedReader, err := decryptor.DecryptStream(reader) - if err != nil { - return fmt.Errorf("failed to decrypt: %w", err) - } + defer func() { _ = blobReader.Close() }() - // Decompress blob - decompressor, err := zstd.NewReader(decryptedReader) - if err != nil { - return fmt.Errorf("failed to decompress: %w", err) - } - defer decompressor.Close() - - // A blob's hash — its remote name — is the double SHA256 of its - // decompressed plaintext (see blobgen.Writer.Sum256), not of the - // encrypted bytes. Hash the plaintext as chunk verification streams - // it, then compare on completion. - plaintextHasher := sha256.New() - hashedStream := io.TeeReader(decompressor, plaintextHasher) - - chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream) + chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, blobReader) if err != nil { return err } - err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash) + err = v.verifyBlobFinalIntegrity(blobReader, blobInfo.Hash) if err != nil { return err } @@ -501,11 +492,12 @@ func (v *Vaultik) verifyBlobChunks( // verifyBlobFinalIntegrity checks that no trailing data exists in the // decompressed stream and that the blob hash matches the expected value. func (v *Vaultik) verifyBlobFinalIntegrity( - plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string, + blobReader *blobgen.Reader, expectedHash string, ) error { // Verify no remaining data in blob - if the chunk list is accurate, - // the blob should be fully consumed. - remaining, err := io.Copy(io.Discard, plaintext) + // the blob should be fully consumed. Draining to EOF also completes the + // reader's plaintext hash. + remaining, err := io.Copy(io.Discard, blobReader) if err != nil { return fmt.Errorf("failed to check for remaining blob data: %w", err) } @@ -514,10 +506,9 @@ func (v *Vaultik) verifyBlobFinalIntegrity( return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining) } - // The blob hash is the double SHA256 of its plaintext content. - firstHash := plaintextHasher.Sum(nil) - secondHash := sha256.Sum256(firstHash) - calculatedBlobHash := hex.EncodeToString(secondHash[:]) + // The blob hash is the double SHA-256 of its plaintext content. + calculatedBlobHash := hex.EncodeToString( + blobgen.DoubleSHA256(blobReader.Sum256())) if calculatedBlobHash != expectedHash { return fmt.Errorf("%w: calculated %s, expected %s", @@ -667,6 +658,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { // each blob using the database as source. func (v *Vaultik) performDeepVerificationFromDB( blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions, + identity age.Identity, ) error { // Calculate total bytes for ETA var totalBytesExpected int64 @@ -684,7 +676,7 @@ func (v *Vaultik) performDeepVerificationFromDB( for i, blobInfo := range blobs { // Verify individual blob - err := v.verifyBlob(blobInfo, db) + err := v.verifyBlob(blobInfo, db, identity) if err != nil { return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err) } diff --git a/internal/vaultik/verify_test.go b/internal/vaultik/verify_test.go deleted file mode 100644 index 75aab5f..0000000 --- a/internal/vaultik/verify_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package vaultik_test - -import ( - "bytes" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "io" - "testing" - - "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "sneak.berlin/go/vaultik/internal/crypto" -) - -// TestTeeReaderWithDecryption tests that TeeReader correctly hashes all encrypted -// bytes when streaming through age decryption and zstd decompression. -// This validates the verification path: hash encrypted blob -> decrypt -> decompress. -func TestTeeReaderWithDecryption(t *testing.T) { - t.Parallel() - - // Test data - use random data that doesn't compress well (5MB) - testData := make([]byte, 5*1024*1024) - _, err := rand.Read(testData) - require.NoError(t, err) - - // Compress the data - var compressedBuf bytes.Buffer - - compressor, err := zstd.NewWriter(&compressedBuf, - zstd.WithEncoderLevel(zstd.SpeedDefault)) - require.NoError(t, err) - _, err = compressor.Write(testData) - require.NoError(t, err) - err = compressor.Close() - require.NoError(t, err) - - // Encrypt the compressed data - testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" + - "mu62kv3s89gmvv" - testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" + - "T00J9CCW9986M3XY87PSGP89AQ" - - encryptor, err := crypto.NewEncryptor([]string{testRecipient}) - require.NoError(t, err) - - var encryptedBuf bytes.Buffer - - err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes())) - require.NoError(t, err) - - encryptedData := encryptedBuf.Bytes() - - // Calculate the expected hash of the encrypted data directly - expectedHash := sha256.Sum256(encryptedData) - expectedHashStr := hex.EncodeToString(expectedHash[:]) - - t.Logf("Encrypted data size: %d bytes", len(encryptedData)) - t.Logf("Expected hash: %s", expectedHashStr) - - // Now simulate what verifyBlob does: use TeeReader to hash while decrypting - decryptor, err := crypto.NewDecryptor(testSecretKey) - require.NoError(t, err) - - // Create hasher and tee reader - hasher := sha256.New() - reader := bytes.NewReader(encryptedData) - teeReader := io.TeeReader(reader, hasher) - - // Decrypt through the tee reader - decryptedReader, err := decryptor.DecryptStream(teeReader) - require.NoError(t, err) - - // Decompress - decompressor, err := zstd.NewReader(decryptedReader) - require.NoError(t, err) - - defer decompressor.Close() - - // Read all decompressed data (simulating chunk verification) - decompressedData, err := io.ReadAll(decompressor) - require.NoError(t, err) - - // Verify we got the original data back - assert.Equal(t, testData, decompressedData, "Decompressed data should match original") - - // Drain remaining decompressed data (should be 0) - remaining, err := io.Copy(io.Discard, decompressor) - require.NoError(t, err) - assert.Equal(t, int64(0), remaining, "No remaining decompressed data") - - // Calculate hash from tee reader - calculatedHashStr := hex.EncodeToString(hasher.Sum(nil)) - t.Logf("Calculated hash (before drain): %s", calculatedHashStr) - - // Verify the hash matches the direct hash of encrypted data - assert.Equal(t, expectedHashStr, calculatedHashStr, - "Hash calculated via TeeReader should match direct hash of encrypted data") -}