Remove the unused crypto path and write the blob-ID hash step once (closes #151)
check / check (pull_request) Successful in 1m22s

Production encryption and decryption already run through blobgen; the
crypto package (Encryptor, Decryptor, UpdateRecipients, the fx Module)
and Vaultik.GetEncryptor/GetDecryptor had no production caller. Delete
crypto and route verify --deep through the same blobgen reader restore
uses: it parses the age key once and reads both the database (still
streamed to a temp file) and every blob through blobgen.NewReader.

The second, blob-ID hash step is now one exported function,
blobgen.DoubleSHA256, called by the three former copies. Writer.Sum256
(the double hash) becomes Writer.ContentID so it no longer shares the
name Sum256 with Reader.Sum256 (the single plaintext hash). CompressData
and CompressStream, unused outside tests, are deleted.

Delete the unused, never-adopted secret/config newtypes in internal/types
(the redacting AgeSecretKey and AWSSecretAccessKey plus AgeRecipient,
S3Endpoint, BucketName, S3Prefix, AWSRegion, AWSAccessKeyID). Delete the
uncalled CleanupIncompleteSnapshots (and deleteSnapshot, its only caller,
now dead) and correct ARCHITECTURE.md. The only multi-recipient test
moves to blobgen; the pre-#131 encrypted-bytes hash test is removed.

Model: opus-4-8
This commit is contained in:
2026-09-22 11:17:33 +00:00
parent 548a7ae156
commit 9ec1c005df
18 changed files with 154 additions and 976 deletions
+1 -2
View File
@@ -286,7 +286,6 @@ Key methods:
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record - `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
- `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete - `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3 - `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
- `CleanupIncompleteSnapshots(ctx, hostname)` → Remove failed snapshots
### `internal/database` ### `internal/database`
SQLite database for local index. Single-writer mode for thread safety. SQLite database for local index. Single-writer mode for thread safety.
@@ -307,7 +306,7 @@ Repository interfaces:
``` ```
CreateSnapshot(opts) CreateSnapshot(opts)
├─► CleanupIncompleteSnapshots() // Critical: avoid dedup errors ├─► PruneDatabase() // Critical: avoid dedup errors
├─► SnapshotManager.CreateSnapshot() // Create DB record ├─► SnapshotManager.CreateSnapshot() // Create DB record
+1 -1
View File
@@ -487,7 +487,7 @@ func (p *Packer) closeBlobWriter() (string, int64, error) {
return "", 0, fmt.Errorf("seeking to start: %w", err) return "", 0, fmt.Errorf("seeking to start: %w", err)
} }
finalHash := p.currentBlob.writer.Sum256() finalHash := p.currentBlob.writer.ContentID()
return hex.EncodeToString(finalHash), finalSize, nil return hex.EncodeToString(finalHash), finalSize, nil
} }
-89
View File
@@ -1,89 +0,0 @@
// Package blobgen implements the blob data pipeline: streaming zstd
// compression, age encryption, and SHA256 content hashing for blob
// creation, plus the matching decrypt/decompress/verify reader.
package blobgen
import (
"bytes"
"encoding/hex"
"fmt"
"io"
)
// CompressResult contains the results of compression
type CompressResult struct {
Data []byte
UncompressedSize int64
CompressedSize int64
SHA256 string
}
// CompressData compresses and encrypts data, returning the result with hash
func CompressData(
data []byte, compressionLevel int, recipients []string,
) (*CompressResult, error) {
var buf bytes.Buffer
// Create writer
w, err := NewWriter(&buf, compressionLevel, recipients)
if err != nil {
return nil, fmt.Errorf("creating writer: %w", err)
}
// Write data
_, err = w.Write(data)
if err != nil {
_ = w.Close()
return nil, fmt.Errorf("writing data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return nil, fmt.Errorf("closing writer: %w", err)
}
return &CompressResult{
Data: buf.Bytes(),
UncompressedSize: int64(len(data)),
CompressedSize: int64(buf.Len()),
SHA256: hex.EncodeToString(w.Sum256()),
}, nil
}
// CompressStream compresses and encrypts from reader to writer, returning
// the number of uncompressed bytes written and the content hash.
func CompressStream(
dst io.Writer, src io.Reader, compressionLevel int, recipients []string,
) (int64, string, error) {
// Create writer
w, err := NewWriter(dst, compressionLevel, recipients)
if err != nil {
return 0, "", fmt.Errorf("creating writer: %w", err)
}
closed := false
defer func() {
if !closed {
_ = w.Close()
}
}()
// Copy data
_, err = io.Copy(w, src)
if err != nil {
return 0, "", fmt.Errorf("copying data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return 0, "", fmt.Errorf("closing writer: %w", err)
}
closed = true
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
}
-80
View File
@@ -1,80 +0,0 @@
package blobgen_test
import (
"bytes"
"crypto/rand"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blobgen"
)
// testRecipient is a static age recipient for tests.
const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
// TestCompressStreamNoDoubleClose is a regression test for issue #28.
// It verifies that CompressStream does not panic or return an error due to
// double-closing the underlying blobgen.Writer. Before the fix in PR #33,
// the explicit Close() on the happy path combined with defer Close() would
// cause a double close.
func TestCompressStreamNoDoubleClose(t *testing.T) {
t.Parallel()
input := []byte("regression test data for issue #28 double-close fix")
var buf bytes.Buffer
written, hash, err := blobgen.CompressStream(
&buf, bytes.NewReader(input), 3, []string{testRecipient})
require.NoError(t, err, "CompressStream should not return an error")
assert.Positive(t, written, "expected bytes written > 0")
assert.NotEmpty(t, hash, "expected non-empty hash")
assert.Positive(t, buf.Len(), "expected non-empty output")
}
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
// to ensure no double-close issues surface under heavier I/O.
func TestCompressStreamLargeInput(t *testing.T) {
t.Parallel()
data := make([]byte, 512*1024) // 512 KB
_, err := rand.Read(data)
require.NoError(t, err)
var buf bytes.Buffer
written, hash, err := blobgen.CompressStream(
&buf, bytes.NewReader(data), 3, []string{testRecipient})
require.NoError(t, err)
assert.Positive(t, written)
assert.NotEmpty(t, hash)
}
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
// without double-close issues.
func TestCompressStreamEmptyInput(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
_, hash, err := blobgen.CompressStream(
&buf, strings.NewReader(""), 3, []string{testRecipient})
require.NoError(t, err)
assert.NotEmpty(t, hash)
}
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
// ensuring the explicit Close + error-path Close pattern is also safe.
func TestCompressDataNoDoubleClose(t *testing.T) {
t.Parallel()
input := []byte("CompressData regression test for double-close")
result, err := blobgen.CompressData(input, 3, []string{testRecipient})
require.NoError(t, err)
assert.Positive(t, result.CompressedSize)
assert.Equal(t, result.UncompressedSize, int64(len(input)))
assert.NotEmpty(t, result.SHA256)
}
+3 -1
View File
@@ -64,7 +64,9 @@ func (r *Reader) Close() error {
return nil return nil
} }
// Sum256 returns the SHA256 hash of all data read // Sum256 returns the single SHA-256 of the plaintext read so far. This is the
// first hash only; the stored object name is its double hash, which callers
// obtain by passing this digest to DoubleSHA256.
func (r *Reader) Sum256() []byte { func (r *Reader) Sum256() []byte {
return r.hasher.Sum(nil) return r.hasher.Sum(nil)
} }
+54
View File
@@ -0,0 +1,54 @@
package blobgen_test
import (
"bytes"
"io"
"testing"
"filippo.io/age"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blobgen"
)
// TestMultipleRecipients verifies that data written for several recipients can
// be read back by each recipient's identity. Moved from internal/crypto, which
// held the only multi-recipient test; blobgen is now the sole encryption path.
func TestMultipleRecipients(t *testing.T) {
t.Parallel()
identities := make([]*age.X25519Identity, 3)
recipients := make([]string, 3)
for i := range identities {
identity, err := age.GenerateX25519Identity()
require.NoError(t, err)
identities[i] = identity
recipients[i] = identity.Recipient().String()
}
plaintext := []byte("Secret message for multiple recipients")
var encrypted bytes.Buffer
writer, err := blobgen.NewWriter(&encrypted, 3, recipients)
require.NoError(t, err)
_, err = writer.Write(plaintext)
require.NoError(t, err)
require.NoError(t, writer.Close())
// Every recipient's identity must recover the original plaintext.
for i, identity := range identities {
reader, err := blobgen.NewReader(
bytes.NewReader(encrypted.Bytes()), identity)
require.NoError(t, err, "recipient %d should open the reader", i+1)
got, err := io.ReadAll(reader)
require.NoError(t, err, "recipient %d should read the plaintext", i+1)
require.NoError(t, reader.Close())
assert.Equal(t, plaintext, got,
"recipient %d should recover the original plaintext", i+1)
}
}
+21 -11
View File
@@ -1,3 +1,6 @@
// Package blobgen implements the blob data pipeline: streaming zstd
// compression, age encryption, and SHA256 content hashing for blob
// creation, plus the matching decrypt/decompress/verify reader.
package blobgen package blobgen
import ( import (
@@ -12,6 +15,18 @@ import (
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
) )
// DoubleSHA256 returns the double SHA-256 of content whose single SHA-256
// digest is sum: it hashes that digest once more. Stored objects are named by
// this second hash so that a name never reveals whether known content is
// present — an attacker who knows a plaintext, and thus its SHA-256, still
// cannot derive the stored name without hashing the digest again. Both a blob
// and the metadata database export are named this way.
func DoubleSHA256(sum []byte) []byte {
h := sha256.Sum256(sum)
return h[:]
}
// Zstd compression level bounds accepted by NewWriter. // Zstd compression level bounds accepted by NewWriter.
const ( const (
minCompressionLevel = 1 minCompressionLevel = 1
@@ -130,17 +145,12 @@ func (w *Writer) Close() error {
return nil return nil
} }
// Sum256 returns the double SHA256 hash of the uncompressed input data. // ContentID returns the double SHA-256 of the uncompressed input data: the
// Double hashing (SHA256(SHA256(data))) prevents information leakage about // name under which this content is stored. It is the second hash of the
// the plaintext - an attacker cannot confirm existence of known content // running SHA-256, via DoubleSHA256; see that function for why content is
// by computing its hash and checking for a matching blob filename. // named this way rather than by its plain SHA-256.
func (w *Writer) Sum256() []byte { func (w *Writer) ContentID() []byte {
// First hash: SHA256(plaintext) return DoubleSHA256(w.hasher.Sum(nil))
firstHash := w.hasher.Sum(nil)
// Second hash: SHA256(firstHash) - this is the blob ID
secondHash := sha256.Sum256(firstHash)
return secondHash[:]
} }
// BytesWritten returns the number of uncompressed bytes written // BytesWritten returns the number of uncompressed bytes written
+5 -5
View File
@@ -12,7 +12,7 @@ import (
"sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/blobgen"
) )
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns // TestWriterHashIsDoubleHash verifies that Writer.ContentID() returns
// the double hash SHA256(SHA256(plaintext)) for security. // the double hash SHA256(SHA256(plaintext)) for security.
// Double hashing prevents attackers from confirming existence of known content. // Double hashing prevents attackers from confirming existence of known content.
func TestWriterHashIsDoubleHash(t *testing.T) { func TestWriterHashIsDoubleHash(t *testing.T) {
@@ -43,7 +43,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// Get the hash from the writer // Get the hash from the writer
writerHash := hex.EncodeToString(writer.Sum256()) writerHash := hex.EncodeToString(writer.ContentID())
// Calculate the expected double hash: SHA256(SHA256(plaintext)) // Calculate the expected double hash: SHA256(SHA256(plaintext))
firstHash := sha256.Sum256(testData) firstHash := sha256.Sum256(testData)
@@ -60,7 +60,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
// The writer hash should match the double hash // The writer hash should match the double hash
assert.Equal(t, expectedDoubleHash, writerHash, assert.Equal(t, expectedDoubleHash, writerHash,
"Writer.Sum256() should return SHA256(SHA256(plaintext)) for security") "Writer.ContentID() should return SHA256(SHA256(plaintext)) for security")
// Verify it's NOT the single hash (would leak information) // Verify it's NOT the single hash (would leak information)
assert.NotEqual(t, singleHashStr, writerHash, assert.NotEqual(t, singleHashStr, writerHash,
@@ -93,8 +93,8 @@ func TestWriterDeterministicHash(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, writer2.Close()) require.NoError(t, writer2.Close())
hash1 := hex.EncodeToString(writer1.Sum256()) hash1 := hex.EncodeToString(writer1.ContentID())
hash2 := hex.EncodeToString(writer2.Sum256()) hash2 := hex.EncodeToString(writer2.ContentID())
// Hashes should be identical (deterministic) // Hashes should be identical (deterministic)
assert.Equal(t, hash1, hash2, "Same input should produce same hash") assert.Equal(t, hash1, hash2, "Same input should produce same hash")
-233
View File
@@ -1,233 +0,0 @@
// Package crypto provides thread-safe age encryption and decryption
// helpers used to protect blob and metadata content.
package crypto //nolint:revive,nolintlint // stdlib crypto unused; see #76
import (
"bytes"
"errors"
"fmt"
"io"
"sync"
"filippo.io/age"
"go.uber.org/fx"
)
// ErrNoRecipients is returned when an encryptor is created or updated
// without any recipient public keys.
var ErrNoRecipients = errors.New("at least one recipient is required")
// errInvalidRecipient is returned when a recipient string does not parse as
// an X25519 age1... public key. It omits the value, which can be sensitive.
var errInvalidRecipient = errors.New(
"not a valid X25519 age1... recipient")
// Encryptor provides thread-safe encryption using the age encryption library.
// It supports encrypting data for multiple recipients simultaneously, allowing
// any of the corresponding private keys to decrypt the data. This is useful
// for backup scenarios where multiple parties should be able to decrypt the data.
type Encryptor struct {
recipients []age.Recipient
mu sync.RWMutex
}
// NewEncryptor creates a new encryptor with the given age public keys.
// Each public key should be a valid age X25519 recipient string (e.g., "age1...")
// At least one recipient must be provided. Returns an error if any of the
// public keys are invalid or if no recipients are specified.
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if len(publicKeys) == 0 {
return nil, ErrNoRecipients
}
recipients := make([]age.Recipient, 0, len(publicKeys))
for i, key := range publicKeys {
// The key string can be sensitive (e.g. a secret key pasted by
// mistake), so the error names its position, never its value.
recipient, err := age.ParseX25519Recipient(key)
if err != nil {
return nil, fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
}
recipients = append(recipients, recipient)
}
return &Encryptor{
recipients: recipients,
}, nil
}
// Encrypt encrypts data using age encryption for all configured recipients.
// The encrypted data can be decrypted by any of the corresponding private keys.
// This method is suitable for small to medium amounts of data that fit in memory.
// For large data streams, use EncryptStream or EncryptWriter instead.
func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
e.mu.RLock()
recipients := e.recipients
e.mu.RUnlock()
var buf bytes.Buffer
// Create encrypted writer for all recipients
w, err := age.Encrypt(&buf, recipients...)
if err != nil {
return nil, fmt.Errorf("creating encrypted writer: %w", err)
}
// Write data
_, err = w.Write(data)
if err != nil {
return nil, fmt.Errorf("writing encrypted data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return nil, fmt.Errorf("closing encrypted writer: %w", err)
}
return buf.Bytes(), nil
}
// EncryptStream encrypts data from reader to writer using age encryption.
// This method is suitable for encrypting large files or streams as it processes
// data in a streaming fashion without loading everything into memory.
// The encrypted data is written directly to the destination writer.
func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
e.mu.RLock()
recipients := e.recipients
e.mu.RUnlock()
// Create encrypted writer for all recipients
w, err := age.Encrypt(dst, recipients...)
if err != nil {
return fmt.Errorf("creating encrypted writer: %w", err)
}
// Copy data
_, err = io.Copy(w, src)
if err != nil {
return fmt.Errorf("copying encrypted data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return fmt.Errorf("closing encrypted writer: %w", err)
}
return nil
}
// EncryptWriter creates a writer that encrypts data written to it.
// All data written to the returned WriteCloser will be encrypted and written
// to the destination writer. The caller must call Close() on the returned
// writer to ensure all encrypted data is properly flushed and finalized.
// This is useful for integrating encryption into existing writer-based pipelines.
func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
e.mu.RLock()
recipients := e.recipients
e.mu.RUnlock()
// Create encrypted writer for all recipients
w, err := age.Encrypt(dst, recipients...)
if err != nil {
return nil, fmt.Errorf("creating encrypted writer: %w", err)
}
return w, nil
}
// UpdateRecipients updates the recipients for future encryption operations.
// This method is thread-safe and can be called while other encryption operations
// are in progress. Existing encryption operations will continue with the old
// recipients. At least one recipient must be provided. Returns an error if any
// of the public keys are invalid or if no recipients are specified.
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if len(publicKeys) == 0 {
return ErrNoRecipients
}
recipients := make([]age.Recipient, 0, len(publicKeys))
for i, key := range publicKeys {
// The key string can be sensitive (e.g. a secret key pasted by
// mistake), so the error names its position, never its value.
recipient, err := age.ParseX25519Recipient(key)
if err != nil {
return fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
}
recipients = append(recipients, recipient)
}
e.mu.Lock()
e.recipients = recipients
e.mu.Unlock()
return nil
}
// Decryptor provides thread-safe decryption using the age encryption library.
// It uses a private key to decrypt data that was encrypted for the corresponding
// public key.
type Decryptor struct {
identity age.Identity
mu sync.RWMutex
}
// NewDecryptor creates a new decryptor with the given age private key.
// The private key should be a valid age X25519 identity string.
// Returns an error if the private key is invalid.
func NewDecryptor(privateKey string) (*Decryptor, error) {
identity, err := age.ParseX25519Identity(privateKey)
if err != nil {
return nil, fmt.Errorf("parsing age identity: %w", err)
}
return &Decryptor{
identity: identity,
}, nil
}
// Decrypt decrypts data using age decryption.
// This method is suitable for small to medium amounts of data that fit in memory.
// For large data streams, use DecryptStream instead.
func (d *Decryptor) Decrypt(data []byte) ([]byte, error) {
d.mu.RLock()
identity := d.identity
d.mu.RUnlock()
r, err := age.Decrypt(bytes.NewReader(data), identity)
if err != nil {
return nil, fmt.Errorf("creating decrypted reader: %w", err)
}
decrypted, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("reading decrypted data: %w", err)
}
return decrypted, nil
}
// DecryptStream returns a reader that decrypts data from the provided reader.
// This method is suitable for decrypting large files or streams as it processes
// data in a streaming fashion without loading everything into memory.
// The caller should close the input reader when done.
func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
d.mu.RLock()
identity := d.identity
d.mu.RUnlock()
r, err := age.Decrypt(src, identity)
if err != nil {
return nil, fmt.Errorf("creating decrypted reader: %w", err)
}
return r, nil
}
// Module exports the crypto module for fx dependency injection.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("crypto")
-198
View File
@@ -1,198 +0,0 @@
package crypto_test
import (
"bytes"
"strings"
"testing"
"filippo.io/age"
"sneak.berlin/go/vaultik/internal/crypto"
)
func TestEncryptor(t *testing.T) {
t.Parallel()
// Generate a test key pair
identity, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity: %v", err)
}
publicKey := identity.Recipient().String()
// Create encryptor
enc, err := crypto.NewEncryptor([]string{publicKey})
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Test data
plaintext := []byte("Hello, World! This is a test message.")
// Encrypt
ciphertext, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Verify it's actually encrypted (should be larger and different)
if bytes.Equal(plaintext, ciphertext) {
t.Error("ciphertext equals plaintext")
}
// Decrypt to verify
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
if err != nil {
t.Fatalf("failed to decrypt: %v", err)
}
var decrypted bytes.Buffer
_, err = decrypted.ReadFrom(r)
if err != nil {
t.Fatalf("failed to read decrypted data: %v", err)
}
if !bytes.Equal(plaintext, decrypted.Bytes()) {
t.Error("decrypted data doesn't match original")
}
}
func TestEncryptorMultipleRecipients(t *testing.T) {
t.Parallel()
// Generate three test key pairs
identity1, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity1: %v", err)
}
identity2, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity2: %v", err)
}
identity3, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity3: %v", err)
}
publicKeys := []string{
identity1.Recipient().String(),
identity2.Recipient().String(),
identity3.Recipient().String(),
}
// Create encryptor with multiple recipients
enc, err := crypto.NewEncryptor(publicKeys)
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Test data
plaintext := []byte("Secret message for multiple recipients")
// Encrypt
ciphertext, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Verify each recipient can decrypt
identities := []age.Identity{identity1, identity2, identity3}
for i, identity := range identities {
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
if err != nil {
t.Fatalf("recipient %d failed to decrypt: %v", i+1, err)
}
var decrypted bytes.Buffer
_, err = decrypted.ReadFrom(r)
if err != nil {
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
}
if !bytes.Equal(plaintext, decrypted.Bytes()) {
t.Errorf("recipient %d: decrypted data doesn't match original", i+1)
}
}
}
func TestEncryptorUpdateRecipients(t *testing.T) {
t.Parallel()
// Generate two identities
identity1, _ := age.GenerateX25519Identity()
identity2, _ := age.GenerateX25519Identity()
publicKey1 := identity1.Recipient().String()
publicKey2 := identity2.Recipient().String()
// Create encryptor with first key
enc, err := crypto.NewEncryptor([]string{publicKey1})
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Encrypt with first key
plaintext := []byte("test data")
ciphertext1, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Update to second key
err = enc.UpdateRecipients([]string{publicKey2})
if err != nil {
t.Fatalf("failed to update recipients: %v", err)
}
// Encrypt with second key
ciphertext2, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// First ciphertext should only decrypt with first identity
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
if err != nil {
t.Error("failed to decrypt with identity1")
}
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
if err == nil {
t.Error("should not decrypt with identity2")
}
// Second ciphertext should only decrypt with second identity
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
if err != nil {
t.Error("failed to decrypt with identity2")
}
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
if err == nil {
t.Error("should not decrypt with identity1")
}
}
// TestNewEncryptorSecretKeyNotEchoed verifies that a secret key mistakenly
// passed as a recipient does not appear in the returned error. A recipient
// string can be sensitive, so the error must name only the position.
func TestNewEncryptorSecretKeyNotEchoed(t *testing.T) {
t.Parallel()
secretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
_, err := crypto.NewEncryptor([]string{secretKey})
if err == nil {
t.Fatal("NewEncryptor returned nil, want error")
}
if strings.Contains(err.Error(), secretKey) {
t.Fatalf("error echoed the recipient value: %v", err)
}
}
+1 -102
View File
@@ -295,68 +295,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(
return nil return nil
} }
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have
// metadata in S3. This is critical for data safety: incomplete snapshots
// can cause deduplication to skip files that were never successfully
// backed up, resulting in data loss.
func (sm *SnapshotManager) CleanupIncompleteSnapshots(
ctx context.Context, hostname string,
) error {
log.Info("Checking for incomplete snapshots", "hostname", hostname)
// Get all incomplete snapshots for this hostname
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
// Check each incomplete snapshot for metadata in storage
for _, snapshot := range incompleteSnapshots {
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst",
RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record",
"snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w",
snapshot.ID, err)
}
log.Info("Deleted incomplete snapshot record and associated data",
"snapshot_id", snapshot.ID)
} else {
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database",
"snapshot_id", snapshot.ID)
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database",
"snapshot_id", snapshot.ID, "error", err)
}
}
}
return nil
}
// CleanupOrphanedData removes files, chunks, and blobs that are no longer // CleanupOrphanedData removes files, chunks, and blobs that are no longer
// referenced by any snapshot. This should be called periodically to clean // referenced by any snapshot. This should be called periodically to clean
// up data from deleted or incomplete snapshots. // up data from deleted or incomplete snapshots.
@@ -759,7 +697,7 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
writerClosed = true writerClosed = true
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256())) log.Debug("Compression complete", "hash", hex.EncodeToString(writer.ContentID()))
return nil return nil
} }
@@ -933,45 +871,6 @@ type ExtendedBackupStats struct {
UploadDurationMs int64 // Total milliseconds spent uploading to S3 UploadDurationMs int64 // Total milliseconds spent uploading to S3
} }
// deleteSnapshot removes a snapshot and all its associations from the database
func (sm *SnapshotManager) deleteSnapshot(
ctx context.Context, snapshotID string,
) error {
// Delete snapshot_files entries
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot files: %w", err)
}
// Delete snapshot_blobs entries
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot blobs: %w", err)
}
// Delete uploads entries (has foreign key to snapshots without CASCADE)
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot uploads: %w", err)
}
// Delete the snapshot itself
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot: %w", err)
}
// Clean up orphaned data
log.Debug("Cleaning up orphaned records in main database")
err = sm.CleanupOrphanedData(ctx)
if err != nil {
return fmt.Errorf("cleaning up orphaned data: %w", err)
}
return nil
}
// deleteOtherSnapshots deletes all snapshots except the current one // deleteOtherSnapshots deletes all snapshots except the current one
func (sm *SnapshotManager) deleteOtherSnapshots( func (sm *SnapshotManager) deleteOtherSnapshots(
ctx context.Context, tx *sql.Tx, currentSnapshotID string, ctx context.Context, tx *sql.Tx, currentSnapshotID string,
+3 -48
View File
@@ -1,7 +1,7 @@
// Package types provides custom types for better type safety across the // Package types provides custom types for better type safety across the
// vaultik codebase. Using distinct types for IDs, hashes, paths, and // vaultik codebase. Using distinct types for IDs, hashes, and paths prevents
// credentials prevents accidental mixing of semantically different values // accidental mixing of semantically different values that happen to share the
// that happen to share the same underlying type. // same underlying type.
package types //nolint:revive,nolintlint // rename decision tracked in #76 package types //nolint:revive,nolintlint // rename decision tracked in #76
import ( import (
@@ -157,34 +157,6 @@ type FilePath string
// Used during restore to strip the source prefix from paths. // Used during restore to strip the source prefix from paths.
type SourcePath string type SourcePath string
// AgeRecipient is an age public key used for encryption.
// Format: age1... (Bech32-encoded X25519 public key)
type AgeRecipient string
// AgeSecretKey is an age private key used for decryption.
// Format: AGE-SECRET-KEY-... (Bech32-encoded X25519 private key)
// This type should never be logged or serialized in plaintext.
type AgeSecretKey string
// S3Endpoint is the URL of an S3-compatible storage endpoint.
type S3Endpoint string
// BucketName is the name of an S3 bucket.
type BucketName string
// S3Prefix is the path prefix within an S3 bucket.
type S3Prefix string
// AWSRegion is an AWS region identifier (e.g., "us-east-1").
type AWSRegion string
// AWSAccessKeyID is an AWS access key ID for authentication.
type AWSAccessKeyID string
// AWSSecretAccessKey is an AWS secret access key for authentication.
// This type should never be logged or serialized in plaintext.
type AWSSecretAccessKey string
// Hostname identifies a host machine. // Hostname identifies a host machine.
type Hostname string type Hostname string
@@ -206,24 +178,7 @@ func (h ChunkHash) String() string { return string(h) }
func (h BlobHash) String() string { return string(h) } func (h BlobHash) String() string { return string(h) }
func (p FilePath) String() string { return string(p) } func (p FilePath) String() string { return string(p) }
func (p SourcePath) String() string { return string(p) } func (p SourcePath) String() string { return string(p) }
func (r AgeRecipient) String() string { return string(r) }
func (e S3Endpoint) String() string { return string(e) }
func (b BucketName) String() string { return string(b) }
func (p S3Prefix) String() string { return string(p) }
func (r AWSRegion) String() string { return string(r) }
func (k AWSAccessKeyID) String() string { return string(k) }
func (h Hostname) String() string { return string(h) } func (h Hostname) String() string { return string(h) }
func (v Version) String() string { return string(v) } func (v Version) String() string { return string(v) }
func (r GitRevision) String() string { return string(r) } func (r GitRevision) String() string { return string(r) }
func (p GlobPattern) String() string { return string(p) } func (p GlobPattern) String() string { return string(p) }
// Redacted String methods for sensitive types - prevents accidental logging
func (k AgeSecretKey) String() string { return "[REDACTED]" }
func (k AWSSecretAccessKey) String() string { return "[REDACTED]" }
// Raw returns the actual value for sensitive types when explicitly needed.
func (k AgeSecretKey) Raw() string { return string(k) }
// Raw returns the actual value for sensitive types when explicitly needed.
func (k AWSSecretAccessKey) Raw() string { return string(k) }
+1 -6
View File
@@ -2,7 +2,6 @@ package vaultik
import ( import (
"context" "context"
"crypto/sha256"
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt" "fmt"
@@ -57,11 +56,7 @@ func (h *hashVerifyReader) Close() error {
return errBlobNotFullyRead return errBlobNotFullyRead
} }
firstHash := h.reader.Sum256() actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256()))
secondHasher := sha256.New()
secondHasher.Write(firstHash)
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
if actualHashHex != h.blobHash { if actualHashHex != h.blobHash {
return fmt.Errorf("%w: expected %s, got %s", return fmt.Errorf("%w: expected %s, got %s",
errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16]) errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16])
+2 -2
View File
@@ -40,13 +40,13 @@ func buildHashTestBlob(
} }
// Compute the double-SHA-256 hash of the plaintext (matches // Compute the double-SHA-256 hash of the plaintext (matches
// blobgen.Writer.Sum256). // blobgen.Writer.ContentID).
firstHash := sha256.Sum256(plaintext) firstHash := sha256.Sum256(plaintext)
secondHash := sha256.Sum256(firstHash[:]) secondHash := sha256.Sum256(firstHash[:])
correctHash := hex.EncodeToString(secondHash[:]) correctHash := hex.EncodeToString(secondHash[:])
// Verify our hash matches what blobgen.Writer produces // Verify our hash matches what blobgen.Writer produces
writerHash := hex.EncodeToString(writer.Sum256()) writerHash := hex.EncodeToString(writer.ContentID())
if correctHash != writerHash { if correctHash != writerHash {
t.Fatalf("hash computation mismatch: manual=%s, writer=%s", t.Fatalf("hash computation mismatch: manual=%s, writer=%s",
correctHash, writerHash) correctHash, writerHash)
+2 -2
View File
@@ -56,8 +56,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
return err return err
} }
// Clean up incomplete snapshots FIRST, before any scanning // Clean up incomplete snapshots FIRST, before any scanning.
// This is critical for data safety - see CleanupIncompleteSnapshots for details // This is critical for data safety; PruneDatabase below does it.
hostname := v.Config.Hostname hostname := v.Config.Hostname
if hostname == "" { if hostname == "" {
hostname, _ = os.Hostname() hostname, _ = os.Hostname()
-28
View File
@@ -5,7 +5,6 @@ package vaultik
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -13,7 +12,6 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/crypto"
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/globals" "sneak.berlin/go/vaultik/internal/globals"
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
@@ -21,12 +19,6 @@ import (
"sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/ui"
) )
// Sentinel errors for misconfigured encryption settings.
var (
errNoAgeRecipients = errors.New("no age recipients configured")
errNoAgeSecretKey = errors.New("no age secret key configured")
)
// Vaultik contains all dependencies needed for vaultik operations // Vaultik contains all dependencies needed for vaultik operations
type Vaultik struct { type Vaultik struct {
Globals *globals.Globals Globals *globals.Globals
@@ -141,26 +133,6 @@ func (v *Vaultik) CanDecrypt() bool {
return v.Config.AgeSecretKey != "" return v.Config.AgeSecretKey != ""
} }
// GetEncryptor creates a new Encryptor instance based on the configured age recipients
// Returns an error if no recipients are configured
func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) {
if len(v.Config.AgeRecipients) == 0 {
return nil, errNoAgeRecipients
}
return crypto.NewEncryptor(v.Config.AgeRecipients)
}
// GetDecryptor creates a new Decryptor instance based on the configured age secret key
// Returns an error if no secret key is configured
func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) {
if v.Config.AgeSecretKey == "" {
return nil, errNoAgeSecretKey
}
return crypto.NewDecryptor(v.Config.AgeSecretKey)
}
// GetFilesystem returns the filesystem instance used by Vaultik // GetFilesystem returns the filesystem instance used by Vaultik
// //
//nolint:ireturn // afero.Fs is the filesystem abstraction by design //nolint:ireturn // afero.Fs is the filesystem abstraction by design
+49 -57
View File
@@ -6,14 +6,14 @@ import (
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt" "fmt"
"hash"
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
"github.com/klauspost/compress/zstd" "filippo.io/age"
"sneak.berlin/go/vaultik/internal/blobgen"
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
@@ -88,13 +88,22 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
errSecretKeyRequired.Error(), errSecretKeyRequired) errSecretKeyRequired.Error(), errSecretKeyRequired)
} }
// Parse the age secret key once, the same way restore does, and reuse
// the identity for the database and every blob.
identity, err := v.prepareRestoreIdentity()
if err != nil {
return v.deepVerifyFailure(result, opts,
fmt.Sprintf("parsing age secret key: %v", err), err)
}
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
if !opts.JSON { if !opts.JSON {
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
} }
manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result) manifest, tempDB, dbBlobs, err := v.loadVerificationData(
snapshotID, opts, result, identity)
if err != nil { if err != nil {
return err return err
} }
@@ -114,7 +123,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
result.TotalSize = totalSize result.TotalSize = totalSize
err = v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize) err = v.runVerificationSteps(
manifest, dbBlobs, tempDB, opts, result, totalSize, identity)
if err != nil { if err != nil {
return err return err
} }
@@ -139,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
// loadVerificationData downloads manifest, database, and blob list for verification // loadVerificationData downloads manifest, database, and blob list for verification
func (v *Vaultik) loadVerificationData( func (v *Vaultik) loadVerificationData(
snapshotID string, opts *VerifyOptions, result *VerifyResult, snapshotID string, opts *VerifyOptions, result *VerifyResult,
identity age.Identity,
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { ) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
// Resolve the identifier to the snapshot's remote key. A human ID is // Resolve the identifier to the snapshot's remote key. A human ID is
// hashed; a remote key (or its abbreviation, as printed for a // hashed; a remote key (or its abbreviation, as printed for a
@@ -188,7 +199,7 @@ func (v *Vaultik) loadVerificationData(
defer func() { _ = dbReader.Close() }() defer func() { _ = dbReader.Close() }()
tdb, err := v.decryptAndLoadDatabase(dbReader) tdb, err := v.decryptAndLoadDatabase(dbReader, identity)
if err != nil { if err != nil {
return nil, nil, nil, v.deepVerifyFailure(result, opts, return nil, nil, nil, v.deepVerifyFailure(result, opts,
fmt.Sprintf("failed to decrypt database: %v", err), fmt.Sprintf("failed to decrypt database: %v", err),
@@ -230,6 +241,7 @@ func (v *Vaultik) runVerificationSteps(
opts *VerifyOptions, opts *VerifyOptions,
result *VerifyResult, result *VerifyResult,
totalSize int64, totalSize int64,
identity age.Identity,
) error { ) error {
if !opts.JSON { if !opts.JSON {
v.stdoutf("Verifying manifest against database...\n") v.stdoutf("Verifying manifest against database...\n")
@@ -256,7 +268,7 @@ func (v *Vaultik) runVerificationSteps(
len(dbBlobs), ubytes(totalSize)) len(dbBlobs), ubytes(totalSize))
} }
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts) err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identity)
if err != nil { if err != nil {
return v.deepVerifyFailure(result, opts, err.Error(), err) return v.deepVerifyFailure(result, opts, err.Error(), err)
} }
@@ -281,26 +293,18 @@ func (t *tempDB) Close() error {
} }
// decryptAndLoadDatabase decrypts and loads the binary SQLite database // decryptAndLoadDatabase decrypts and loads the binary SQLite database
// from the encrypted stream. // from the encrypted stream. It reads through the same blobgen reader restore
func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) { // uses, streaming the decrypted, decompressed database to a temp file.
// Get decryptor func (v *Vaultik) decryptAndLoadDatabase(
decryptor, err := v.GetDecryptor() reader io.ReadCloser, identity age.Identity,
) (*tempDB, error) {
// Decrypt and decompress through the shared blobgen reader.
blobReader, err := blobgen.NewReader(reader, identity)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get decryptor: %w", err) return nil, fmt.Errorf("failed to create decryption reader: %w", err)
} }
// Decrypt the stream defer func() { _ = blobReader.Close() }()
decryptedReader, err := decryptor.DecryptStream(reader)
if err != nil {
return nil, fmt.Errorf("failed to decrypt database: %w", err)
}
// Decompress the binary database
decompressor, err := zstd.NewReader(decryptedReader)
if err != nil {
return nil, fmt.Errorf("failed to create decompressor: %w", err)
}
defer decompressor.Close()
// Materialize the decrypted database inside a private (0700) temp // Materialize the decrypted database inside a private (0700) temp
// directory so it is never world-readable, and remove the whole // directory so it is never world-readable, and remove the whole
@@ -330,7 +334,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
// Stream decompress directly to file // Stream decompress directly to file
log.Info("Decompressing database...") log.Info("Decompressing database...")
written, err := io.Copy(tempFile, decompressor) written, err := io.Copy(tempFile, blobReader)
if err != nil { if err != nil {
_ = tempFile.Close() _ = tempFile.Close()
@@ -355,7 +359,9 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
} }
// verifyBlob downloads and verifies a single blob // verifyBlob downloads and verifies a single blob
func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { func (v *Vaultik) verifyBlob(
blobInfo snapshot.BlobInfo, db *sql.DB, identity age.Identity,
) error {
// Download blob using shared fetch method // Download blob using shared fetch method
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize) reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
if err != nil { if err != nil {
@@ -364,38 +370,23 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
defer func() { _ = reader.Close() }() defer func() { _ = reader.Close() }()
// Get decryptor // Decrypt and decompress through the shared blobgen reader, which hashes
decryptor, err := v.GetDecryptor() // the plaintext as it is read. A blob's hash — its remote name — is the
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
// encrypted bytes.
blobReader, err := blobgen.NewReader(reader, identity)
if err != nil { if err != nil {
return fmt.Errorf("failed to get decryptor: %w", err) return fmt.Errorf("failed to create blob reader: %w", err)
} }
// Decrypt blob defer func() { _ = blobReader.Close() }()
decryptedReader, err := decryptor.DecryptStream(reader)
if err != nil {
return fmt.Errorf("failed to decrypt: %w", err)
}
// Decompress blob chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, blobReader)
decompressor, err := zstd.NewReader(decryptedReader)
if err != nil {
return fmt.Errorf("failed to decompress: %w", err)
}
defer decompressor.Close()
// A blob's hash — its remote name — is the double SHA256 of its
// decompressed plaintext (see blobgen.Writer.Sum256), not of the
// encrypted bytes. Hash the plaintext as chunk verification streams
// it, then compare on completion.
plaintextHasher := sha256.New()
hashedStream := io.TeeReader(decompressor, plaintextHasher)
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream)
if err != nil { if err != nil {
return err return err
} }
err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash) err = v.verifyBlobFinalIntegrity(blobReader, blobInfo.Hash)
if err != nil { if err != nil {
return err return err
} }
@@ -501,11 +492,12 @@ func (v *Vaultik) verifyBlobChunks(
// verifyBlobFinalIntegrity checks that no trailing data exists in the // verifyBlobFinalIntegrity checks that no trailing data exists in the
// decompressed stream and that the blob hash matches the expected value. // decompressed stream and that the blob hash matches the expected value.
func (v *Vaultik) verifyBlobFinalIntegrity( func (v *Vaultik) verifyBlobFinalIntegrity(
plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string, blobReader *blobgen.Reader, expectedHash string,
) error { ) error {
// Verify no remaining data in blob - if the chunk list is accurate, // Verify no remaining data in blob - if the chunk list is accurate,
// the blob should be fully consumed. // the blob should be fully consumed. Draining to EOF also completes the
remaining, err := io.Copy(io.Discard, plaintext) // reader's plaintext hash.
remaining, err := io.Copy(io.Discard, blobReader)
if err != nil { if err != nil {
return fmt.Errorf("failed to check for remaining blob data: %w", err) return fmt.Errorf("failed to check for remaining blob data: %w", err)
} }
@@ -514,10 +506,9 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining) return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
} }
// The blob hash is the double SHA256 of its plaintext content. // The blob hash is the double SHA-256 of its plaintext content.
firstHash := plaintextHasher.Sum(nil) calculatedBlobHash := hex.EncodeToString(
secondHash := sha256.Sum256(firstHash) blobgen.DoubleSHA256(blobReader.Sum256()))
calculatedBlobHash := hex.EncodeToString(secondHash[:])
if calculatedBlobHash != expectedHash { if calculatedBlobHash != expectedHash {
return fmt.Errorf("%w: calculated %s, expected %s", return fmt.Errorf("%w: calculated %s, expected %s",
@@ -667,6 +658,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
// each blob using the database as source. // each blob using the database as source.
func (v *Vaultik) performDeepVerificationFromDB( func (v *Vaultik) performDeepVerificationFromDB(
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions, blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
identity age.Identity,
) error { ) error {
// Calculate total bytes for ETA // Calculate total bytes for ETA
var totalBytesExpected int64 var totalBytesExpected int64
@@ -684,7 +676,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
for i, blobInfo := range blobs { for i, blobInfo := range blobs {
// Verify individual blob // Verify individual blob
err := v.verifyBlob(blobInfo, db) err := v.verifyBlob(blobInfo, db, identity)
if err != nil { if err != nil {
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err) return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
} }
-100
View File
@@ -1,100 +0,0 @@
package vaultik_test
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"io"
"testing"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/crypto"
)
// TestTeeReaderWithDecryption tests that TeeReader correctly hashes all encrypted
// bytes when streaming through age decryption and zstd decompression.
// This validates the verification path: hash encrypted blob -> decrypt -> decompress.
func TestTeeReaderWithDecryption(t *testing.T) {
t.Parallel()
// Test data - use random data that doesn't compress well (5MB)
testData := make([]byte, 5*1024*1024)
_, err := rand.Read(testData)
require.NoError(t, err)
// Compress the data
var compressedBuf bytes.Buffer
compressor, err := zstd.NewWriter(&compressedBuf,
zstd.WithEncoderLevel(zstd.SpeedDefault))
require.NoError(t, err)
_, err = compressor.Write(testData)
require.NoError(t, err)
err = compressor.Close()
require.NoError(t, err)
// Encrypt the compressed data
testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" +
"mu62kv3s89gmvv"
testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" +
"T00J9CCW9986M3XY87PSGP89AQ"
encryptor, err := crypto.NewEncryptor([]string{testRecipient})
require.NoError(t, err)
var encryptedBuf bytes.Buffer
err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes()))
require.NoError(t, err)
encryptedData := encryptedBuf.Bytes()
// Calculate the expected hash of the encrypted data directly
expectedHash := sha256.Sum256(encryptedData)
expectedHashStr := hex.EncodeToString(expectedHash[:])
t.Logf("Encrypted data size: %d bytes", len(encryptedData))
t.Logf("Expected hash: %s", expectedHashStr)
// Now simulate what verifyBlob does: use TeeReader to hash while decrypting
decryptor, err := crypto.NewDecryptor(testSecretKey)
require.NoError(t, err)
// Create hasher and tee reader
hasher := sha256.New()
reader := bytes.NewReader(encryptedData)
teeReader := io.TeeReader(reader, hasher)
// Decrypt through the tee reader
decryptedReader, err := decryptor.DecryptStream(teeReader)
require.NoError(t, err)
// Decompress
decompressor, err := zstd.NewReader(decryptedReader)
require.NoError(t, err)
defer decompressor.Close()
// Read all decompressed data (simulating chunk verification)
decompressedData, err := io.ReadAll(decompressor)
require.NoError(t, err)
// Verify we got the original data back
assert.Equal(t, testData, decompressedData, "Decompressed data should match original")
// Drain remaining decompressed data (should be 0)
remaining, err := io.Copy(io.Discard, decompressor)
require.NoError(t, err)
assert.Equal(t, int64(0), remaining, "No remaining decompressed data")
// Calculate hash from tee reader
calculatedHashStr := hex.EncodeToString(hasher.Sum(nil))
t.Logf("Calculated hash (before drain): %s", calculatedHashStr)
// Verify the hash matches the direct hash of encrypted data
assert.Equal(t, expectedHashStr, calculatedHashStr,
"Hash calculated via TeeReader should match direct hash of encrypted data")
}