Remove the unused crypto path and write the blob-ID hash step once #191
+1
-2
@@ -286,7 +286,6 @@ Key methods:
|
||||
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
|
||||
- `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete
|
||||
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
|
||||
- `CleanupIncompleteSnapshots(ctx, hostname)` → Remove failed snapshots
|
||||
|
||||
### `internal/database`
|
||||
SQLite database for local index. Single-writer mode for thread safety.
|
||||
@@ -307,7 +306,7 @@ Repository interfaces:
|
||||
```
|
||||
CreateSnapshot(opts)
|
||||
│
|
||||
├─► CleanupIncompleteSnapshots() // Critical: avoid dedup errors
|
||||
├─► PruneDatabase() // Critical: avoid dedup errors
|
||||
│
|
||||
├─► SnapshotManager.CreateSnapshot() // Create DB record
|
||||
│
|
||||
|
||||
@@ -487,7 +487,7 @@ func (p *Packer) closeBlobWriter() (string, int64, error) {
|
||||
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
||||
}
|
||||
|
||||
finalHash := p.currentBlob.writer.Sum256()
|
||||
finalHash := p.currentBlob.writer.ContentID()
|
||||
|
||||
return hex.EncodeToString(finalHash), finalSize, nil
|
||||
}
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
// Package blobgen implements the blob data pipeline: streaming zstd
|
||||
// compression, age encryption, and SHA256 content hashing for blob
|
||||
// creation, plus the matching decrypt/decompress/verify reader.
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// CompressResult contains the results of compression
|
||||
type CompressResult struct {
|
||||
Data []byte
|
||||
UncompressedSize int64
|
||||
CompressedSize int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// CompressData compresses and encrypts data, returning the result with hash
|
||||
func CompressData(
|
||||
data []byte, compressionLevel int, recipients []string,
|
||||
) (*CompressResult, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Create writer
|
||||
w, err := NewWriter(&buf, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating writer: %w", err)
|
||||
}
|
||||
|
||||
// Write data
|
||||
_, err = w.Write(data)
|
||||
if err != nil {
|
||||
_ = w.Close()
|
||||
|
||||
return nil, fmt.Errorf("writing data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
return &CompressResult{
|
||||
Data: buf.Bytes(),
|
||||
UncompressedSize: int64(len(data)),
|
||||
CompressedSize: int64(buf.Len()),
|
||||
SHA256: hex.EncodeToString(w.Sum256()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CompressStream compresses and encrypts from reader to writer, returning
|
||||
// the number of uncompressed bytes written and the content hash.
|
||||
func CompressStream(
|
||||
dst io.Writer, src io.Reader, compressionLevel int, recipients []string,
|
||||
) (int64, string, error) {
|
||||
// Create writer
|
||||
w, err := NewWriter(dst, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("creating writer: %w", err)
|
||||
}
|
||||
|
||||
closed := false
|
||||
|
||||
defer func() {
|
||||
if !closed {
|
||||
_ = w.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Copy data
|
||||
_, err = io.Copy(w, src)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("copying data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
closed = true
|
||||
|
||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// testRecipient is a static age recipient for tests.
|
||||
const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
|
||||
|
||||
// TestCompressStreamNoDoubleClose is a regression test for issue #28.
|
||||
// It verifies that CompressStream does not panic or return an error due to
|
||||
// double-closing the underlying blobgen.Writer. Before the fix in PR #33,
|
||||
// the explicit Close() on the happy path combined with defer Close() would
|
||||
// cause a double close.
|
||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := []byte("regression test data for issue #28 double-close fix")
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := blobgen.CompressStream(
|
||||
&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||
require.NoError(t, err, "CompressStream should not return an error")
|
||||
assert.Positive(t, written, "expected bytes written > 0")
|
||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
||||
assert.Positive(t, buf.Len(), "expected non-empty output")
|
||||
}
|
||||
|
||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
||||
// to ensure no double-close issues surface under heavier I/O.
|
||||
func TestCompressStreamLargeInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := make([]byte, 512*1024) // 512 KB
|
||||
_, err := rand.Read(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := blobgen.CompressStream(
|
||||
&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.Positive(t, written)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
|
||||
// without double-close issues.
|
||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, hash, err := blobgen.CompressStream(
|
||||
&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
|
||||
// ensuring the explicit Close + error-path Close pattern is also safe.
|
||||
func TestCompressDataNoDoubleClose(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := []byte("CompressData regression test for double-close")
|
||||
|
||||
result, err := blobgen.CompressData(input, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.Positive(t, result.CompressedSize)
|
||||
assert.Equal(t, result.UncompressedSize, int64(len(input)))
|
||||
assert.NotEmpty(t, result.SHA256)
|
||||
}
|
||||
@@ -64,7 +64,9 @@ func (r *Reader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sum256 returns the SHA256 hash of all data read
|
||||
// Sum256 returns the single SHA-256 of the plaintext read so far. This is the
|
||||
// first hash only; the stored object name is its double hash, which callers
|
||||
// obtain by passing this digest to DoubleSHA256.
|
||||
func (r *Reader) Sum256() []byte {
|
||||
return r.hasher.Sum(nil)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestMultipleRecipients verifies that data written for several recipients can
|
||||
// be read back by each recipient's identity. Moved from internal/crypto, which
|
||||
// held the only multi-recipient test; blobgen is now the sole encryption path.
|
||||
func TestMultipleRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
identities := make([]*age.X25519Identity, 3)
|
||||
recipients := make([]string, 3)
|
||||
|
||||
for i := range identities {
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
identities[i] = identity
|
||||
recipients[i] = identity.Recipient().String()
|
||||
}
|
||||
|
||||
plaintext := []byte("Secret message for multiple recipients")
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
|
||||
writer, err := blobgen.NewWriter(&encrypted, 3, recipients)
|
||||
require.NoError(t, err)
|
||||
_, err = writer.Write(plaintext)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
// Every recipient's identity must recover the original plaintext.
|
||||
for i, identity := range identities {
|
||||
reader, err := blobgen.NewReader(
|
||||
bytes.NewReader(encrypted.Bytes()), identity)
|
||||
require.NoError(t, err, "recipient %d should open the reader", i+1)
|
||||
|
||||
got, err := io.ReadAll(reader)
|
||||
require.NoError(t, err, "recipient %d should read the plaintext", i+1)
|
||||
require.NoError(t, reader.Close())
|
||||
|
||||
assert.Equal(t, plaintext, got,
|
||||
"recipient %d should recover the original plaintext", i+1)
|
||||
}
|
||||
}
|
||||
+21
-11
@@ -1,3 +1,6 @@
|
||||
// Package blobgen implements the blob data pipeline: streaming zstd
|
||||
// compression, age encryption, and SHA256 content hashing for blob
|
||||
// creation, plus the matching decrypt/decompress/verify reader.
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
@@ -12,6 +15,18 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// DoubleSHA256 returns the double SHA-256 of content whose single SHA-256
|
||||
// digest is sum: it hashes that digest once more. Stored objects are named by
|
||||
// this second hash so that a name never reveals whether known content is
|
||||
// present — an attacker who knows a plaintext, and thus its SHA-256, still
|
||||
// cannot derive the stored name without hashing the digest again. Both a blob
|
||||
// and the metadata database export are named this way.
|
||||
func DoubleSHA256(sum []byte) []byte {
|
||||
h := sha256.Sum256(sum)
|
||||
|
||||
return h[:]
|
||||
}
|
||||
|
||||
// Zstd compression level bounds accepted by NewWriter.
|
||||
const (
|
||||
minCompressionLevel = 1
|
||||
@@ -130,17 +145,12 @@ func (w *Writer) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sum256 returns the double SHA256 hash of the uncompressed input data.
|
||||
// Double hashing (SHA256(SHA256(data))) prevents information leakage about
|
||||
// the plaintext - an attacker cannot confirm existence of known content
|
||||
// by computing its hash and checking for a matching blob filename.
|
||||
func (w *Writer) Sum256() []byte {
|
||||
// First hash: SHA256(plaintext)
|
||||
firstHash := w.hasher.Sum(nil)
|
||||
// Second hash: SHA256(firstHash) - this is the blob ID
|
||||
secondHash := sha256.Sum256(firstHash)
|
||||
|
||||
return secondHash[:]
|
||||
// ContentID returns the double SHA-256 of the uncompressed input data: the
|
||||
// name under which this content is stored. It is the second hash of the
|
||||
// running SHA-256, via DoubleSHA256; see that function for why content is
|
||||
// named this way rather than by its plain SHA-256.
|
||||
func (w *Writer) ContentID() []byte {
|
||||
return DoubleSHA256(w.hasher.Sum(nil))
|
||||
}
|
||||
|
||||
// BytesWritten returns the number of uncompressed bytes written
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns
|
||||
// TestWriterHashIsDoubleHash verifies that Writer.ContentID() returns
|
||||
// the double hash SHA256(SHA256(plaintext)) for security.
|
||||
// Double hashing prevents attackers from confirming existence of known content.
|
||||
func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
@@ -43,7 +43,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the hash from the writer
|
||||
writerHash := hex.EncodeToString(writer.Sum256())
|
||||
writerHash := hex.EncodeToString(writer.ContentID())
|
||||
|
||||
// Calculate the expected double hash: SHA256(SHA256(plaintext))
|
||||
firstHash := sha256.Sum256(testData)
|
||||
@@ -60,7 +60,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
|
||||
// The writer hash should match the double hash
|
||||
assert.Equal(t, expectedDoubleHash, writerHash,
|
||||
"Writer.Sum256() should return SHA256(SHA256(plaintext)) for security")
|
||||
"Writer.ContentID() should return SHA256(SHA256(plaintext)) for security")
|
||||
|
||||
// Verify it's NOT the single hash (would leak information)
|
||||
assert.NotEqual(t, singleHashStr, writerHash,
|
||||
@@ -93,8 +93,8 @@ func TestWriterDeterministicHash(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer2.Close())
|
||||
|
||||
hash1 := hex.EncodeToString(writer1.Sum256())
|
||||
hash2 := hex.EncodeToString(writer2.Sum256())
|
||||
hash1 := hex.EncodeToString(writer1.ContentID())
|
||||
hash2 := hex.EncodeToString(writer2.ContentID())
|
||||
|
||||
// Hashes should be identical (deterministic)
|
||||
assert.Equal(t, hash1, hash2, "Same input should produce same hash")
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
// Package crypto provides thread-safe age encryption and decryption
|
||||
// helpers used to protect blob and metadata content.
|
||||
package crypto //nolint:revive,nolintlint // stdlib crypto unused; see #76
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"filippo.io/age"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// ErrNoRecipients is returned when an encryptor is created or updated
|
||||
// without any recipient public keys.
|
||||
var ErrNoRecipients = errors.New("at least one recipient is required")
|
||||
|
||||
// errInvalidRecipient is returned when a recipient string does not parse as
|
||||
// an X25519 age1... public key. It omits the value, which can be sensitive.
|
||||
var errInvalidRecipient = errors.New(
|
||||
"not a valid X25519 age1... recipient")
|
||||
|
||||
// Encryptor provides thread-safe encryption using the age encryption library.
|
||||
// It supports encrypting data for multiple recipients simultaneously, allowing
|
||||
// any of the corresponding private keys to decrypt the data. This is useful
|
||||
// for backup scenarios where multiple parties should be able to decrypt the data.
|
||||
type Encryptor struct {
|
||||
recipients []age.Recipient
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewEncryptor creates a new encryptor with the given age public keys.
|
||||
// Each public key should be a valid age X25519 recipient string (e.g., "age1...")
|
||||
// At least one recipient must be provided. Returns an error if any of the
|
||||
// public keys are invalid or if no recipients are specified.
|
||||
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||
if len(publicKeys) == 0 {
|
||||
return nil, ErrNoRecipients
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
for i, key := range publicKeys {
|
||||
// The key string can be sensitive (e.g. a secret key pasted by
|
||||
// mistake), so the error names its position, never its value.
|
||||
recipient, err := age.ParseX25519Recipient(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
|
||||
}
|
||||
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
return &Encryptor{
|
||||
recipients: recipients,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Encrypt encrypts data using age encryption for all configured recipients.
|
||||
// The encrypted data can be decrypted by any of the corresponding private keys.
|
||||
// This method is suitable for small to medium amounts of data that fit in memory.
|
||||
// For large data streams, use EncryptStream or EncryptWriter instead.
|
||||
func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
|
||||
e.mu.RLock()
|
||||
recipients := e.recipients
|
||||
e.mu.RUnlock()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Create encrypted writer for all recipients
|
||||
w, err := age.Encrypt(&buf, recipients...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
// Write data
|
||||
_, err = w.Write(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("writing encrypted data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("closing encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// EncryptStream encrypts data from reader to writer using age encryption.
|
||||
// This method is suitable for encrypting large files or streams as it processes
|
||||
// data in a streaming fashion without loading everything into memory.
|
||||
// The encrypted data is written directly to the destination writer.
|
||||
func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
|
||||
e.mu.RLock()
|
||||
recipients := e.recipients
|
||||
e.mu.RUnlock()
|
||||
|
||||
// Create encrypted writer for all recipients
|
||||
w, err := age.Encrypt(dst, recipients...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
// Copy data
|
||||
_, err = io.Copy(w, src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copying encrypted data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncryptWriter creates a writer that encrypts data written to it.
|
||||
// All data written to the returned WriteCloser will be encrypted and written
|
||||
// to the destination writer. The caller must call Close() on the returned
|
||||
// writer to ensure all encrypted data is properly flushed and finalized.
|
||||
// This is useful for integrating encryption into existing writer-based pipelines.
|
||||
func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
|
||||
e.mu.RLock()
|
||||
recipients := e.recipients
|
||||
e.mu.RUnlock()
|
||||
|
||||
// Create encrypted writer for all recipients
|
||||
w, err := age.Encrypt(dst, recipients...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// UpdateRecipients updates the recipients for future encryption operations.
|
||||
// This method is thread-safe and can be called while other encryption operations
|
||||
// are in progress. Existing encryption operations will continue with the old
|
||||
// recipients. At least one recipient must be provided. Returns an error if any
|
||||
// of the public keys are invalid or if no recipients are specified.
|
||||
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||
if len(publicKeys) == 0 {
|
||||
return ErrNoRecipients
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
for i, key := range publicKeys {
|
||||
// The key string can be sensitive (e.g. a secret key pasted by
|
||||
// mistake), so the error names its position, never its value.
|
||||
recipient, err := age.ParseX25519Recipient(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
|
||||
}
|
||||
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
e.recipients = recipients
|
||||
e.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decryptor provides thread-safe decryption using the age encryption library.
|
||||
// It uses a private key to decrypt data that was encrypted for the corresponding
|
||||
// public key.
|
||||
type Decryptor struct {
|
||||
identity age.Identity
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewDecryptor creates a new decryptor with the given age private key.
|
||||
// The private key should be a valid age X25519 identity string.
|
||||
// Returns an error if the private key is invalid.
|
||||
func NewDecryptor(privateKey string) (*Decryptor, error) {
|
||||
identity, err := age.ParseX25519Identity(privateKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing age identity: %w", err)
|
||||
}
|
||||
|
||||
return &Decryptor{
|
||||
identity: identity,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Decrypt decrypts data using age decryption.
|
||||
// This method is suitable for small to medium amounts of data that fit in memory.
|
||||
// For large data streams, use DecryptStream instead.
|
||||
func (d *Decryptor) Decrypt(data []byte) ([]byte, error) {
|
||||
d.mu.RLock()
|
||||
identity := d.identity
|
||||
d.mu.RUnlock()
|
||||
|
||||
r, err := age.Decrypt(bytes.NewReader(data), identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decrypted reader: %w", err)
|
||||
}
|
||||
|
||||
decrypted, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading decrypted data: %w", err)
|
||||
}
|
||||
|
||||
return decrypted, nil
|
||||
}
|
||||
|
||||
// DecryptStream returns a reader that decrypts data from the provided reader.
|
||||
// This method is suitable for decrypting large files or streams as it processes
|
||||
// data in a streaming fashion without loading everything into memory.
|
||||
// The caller should close the input reader when done.
|
||||
func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
|
||||
d.mu.RLock()
|
||||
identity := d.identity
|
||||
d.mu.RUnlock()
|
||||
|
||||
r, err := age.Decrypt(src, identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decrypted reader: %w", err)
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Module exports the crypto module for fx dependency injection.
|
||||
//
|
||||
//nolint:gochecknoglobals // fx module definitions are package globals
|
||||
var Module = fx.Module("crypto")
|
||||
@@ -1,198 +0,0 @@
|
||||
package crypto_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"sneak.berlin/go/vaultik/internal/crypto"
|
||||
)
|
||||
|
||||
func TestEncryptor(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate a test key pair
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity: %v", err)
|
||||
}
|
||||
|
||||
publicKey := identity.Recipient().String()
|
||||
|
||||
// Create encryptor
|
||||
enc, err := crypto.NewEncryptor([]string{publicKey})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Test data
|
||||
plaintext := []byte("Hello, World! This is a test message.")
|
||||
|
||||
// Encrypt
|
||||
ciphertext, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Verify it's actually encrypted (should be larger and different)
|
||||
if bytes.Equal(plaintext, ciphertext) {
|
||||
t.Error("ciphertext equals plaintext")
|
||||
}
|
||||
|
||||
// Decrypt to verify
|
||||
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt: %v", err)
|
||||
}
|
||||
|
||||
var decrypted bytes.Buffer
|
||||
|
||||
_, err = decrypted.ReadFrom(r)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read decrypted data: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(plaintext, decrypted.Bytes()) {
|
||||
t.Error("decrypted data doesn't match original")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate three test key pairs
|
||||
identity1, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity1: %v", err)
|
||||
}
|
||||
|
||||
identity2, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity2: %v", err)
|
||||
}
|
||||
|
||||
identity3, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity3: %v", err)
|
||||
}
|
||||
|
||||
publicKeys := []string{
|
||||
identity1.Recipient().String(),
|
||||
identity2.Recipient().String(),
|
||||
identity3.Recipient().String(),
|
||||
}
|
||||
|
||||
// Create encryptor with multiple recipients
|
||||
enc, err := crypto.NewEncryptor(publicKeys)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Test data
|
||||
plaintext := []byte("Secret message for multiple recipients")
|
||||
|
||||
// Encrypt
|
||||
ciphertext, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Verify each recipient can decrypt
|
||||
identities := []age.Identity{identity1, identity2, identity3}
|
||||
for i, identity := range identities {
|
||||
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("recipient %d failed to decrypt: %v", i+1, err)
|
||||
}
|
||||
|
||||
var decrypted bytes.Buffer
|
||||
|
||||
_, err = decrypted.ReadFrom(r)
|
||||
if err != nil {
|
||||
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(plaintext, decrypted.Bytes()) {
|
||||
t.Errorf("recipient %d: decrypted data doesn't match original", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate two identities
|
||||
identity1, _ := age.GenerateX25519Identity()
|
||||
identity2, _ := age.GenerateX25519Identity()
|
||||
|
||||
publicKey1 := identity1.Recipient().String()
|
||||
publicKey2 := identity2.Recipient().String()
|
||||
|
||||
// Create encryptor with first key
|
||||
enc, err := crypto.NewEncryptor([]string{publicKey1})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Encrypt with first key
|
||||
plaintext := []byte("test data")
|
||||
|
||||
ciphertext1, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Update to second key
|
||||
err = enc.UpdateRecipients([]string{publicKey2})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update recipients: %v", err)
|
||||
}
|
||||
|
||||
// Encrypt with second key
|
||||
ciphertext2, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// First ciphertext should only decrypt with first identity
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
|
||||
if err != nil {
|
||||
t.Error("failed to decrypt with identity1")
|
||||
}
|
||||
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
|
||||
if err == nil {
|
||||
t.Error("should not decrypt with identity2")
|
||||
}
|
||||
|
||||
// Second ciphertext should only decrypt with second identity
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
|
||||
if err != nil {
|
||||
t.Error("failed to decrypt with identity2")
|
||||
}
|
||||
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
|
||||
if err == nil {
|
||||
t.Error("should not decrypt with identity1")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewEncryptorSecretKeyNotEchoed verifies that a secret key mistakenly
|
||||
// passed as a recipient does not appear in the returned error. A recipient
|
||||
// string can be sensitive, so the error must name only the position.
|
||||
func TestNewEncryptorSecretKeyNotEchoed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
secretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
|
||||
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
|
||||
_, err := crypto.NewEncryptor([]string{secretKey})
|
||||
if err == nil {
|
||||
t.Fatal("NewEncryptor returned nil, want error")
|
||||
}
|
||||
|
||||
if strings.Contains(err.Error(), secretKey) {
|
||||
t.Fatalf("error echoed the recipient value: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -295,68 +295,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have
|
||||
// metadata in S3. This is critical for data safety: incomplete snapshots
|
||||
// can cause deduplication to skip files that were never successfully
|
||||
// backed up, resulting in data loss.
|
||||
func (sm *SnapshotManager) CleanupIncompleteSnapshots(
|
||||
ctx context.Context, hostname string,
|
||||
) error {
|
||||
log.Info("Checking for incomplete snapshots", "hostname", hostname)
|
||||
|
||||
// Get all incomplete snapshots for this hostname
|
||||
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting incomplete snapshots: %w", err)
|
||||
}
|
||||
|
||||
if len(incompleteSnapshots) == 0 {
|
||||
log.Debug("No incomplete snapshots found")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
|
||||
|
||||
// Check each incomplete snapshot for metadata in storage
|
||||
for _, snapshot := range incompleteSnapshots {
|
||||
// Check if metadata exists in storage (paths use the hashed
|
||||
// remote key so we don't leak host info to the listing).
|
||||
metadataKey := fmt.Sprintf("metadata/%s/db.zst",
|
||||
RemoteSnapshotKey(snapshot.ID.String()))
|
||||
|
||||
_, err := sm.storage.Stat(ctx, metadataKey)
|
||||
if err != nil {
|
||||
// Metadata doesn't exist in S3 - this is an incomplete snapshot
|
||||
log.Info("Cleaning up incomplete snapshot record",
|
||||
"snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
|
||||
|
||||
// Delete the snapshot and all its associations
|
||||
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting incomplete snapshot %s: %w",
|
||||
snapshot.ID, err)
|
||||
}
|
||||
|
||||
log.Info("Deleted incomplete snapshot record and associated data",
|
||||
"snapshot_id", snapshot.ID)
|
||||
} else {
|
||||
// Metadata exists - this snapshot was completed but database wasn't updated
|
||||
// This shouldn't happen in normal operation, but mark it complete
|
||||
log.Warn("Found snapshot with remote metadata but incomplete in database",
|
||||
"snapshot_id", snapshot.ID)
|
||||
|
||||
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
|
||||
if err != nil {
|
||||
log.Error("Failed to mark snapshot as complete in database",
|
||||
"snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupOrphanedData removes files, chunks, and blobs that are no longer
|
||||
// referenced by any snapshot. This should be called periodically to clean
|
||||
// up data from deleted or incomplete snapshots.
|
||||
@@ -759,7 +697,7 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
|
||||
writerClosed = true
|
||||
|
||||
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
|
||||
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.ContentID()))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -933,45 +871,6 @@ type ExtendedBackupStats struct {
|
||||
UploadDurationMs int64 // Total milliseconds spent uploading to S3
|
||||
}
|
||||
|
||||
// deleteSnapshot removes a snapshot and all its associations from the database
|
||||
func (sm *SnapshotManager) deleteSnapshot(
|
||||
ctx context.Context, snapshotID string,
|
||||
) error {
|
||||
// Delete snapshot_files entries
|
||||
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot files: %w", err)
|
||||
}
|
||||
|
||||
// Delete snapshot_blobs entries
|
||||
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot blobs: %w", err)
|
||||
}
|
||||
|
||||
// Delete uploads entries (has foreign key to snapshots without CASCADE)
|
||||
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot uploads: %w", err)
|
||||
}
|
||||
|
||||
// Delete the snapshot itself
|
||||
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Clean up orphaned data
|
||||
log.Debug("Cleaning up orphaned records in main database")
|
||||
|
||||
err = sm.CleanupOrphanedData(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleaning up orphaned data: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteOtherSnapshots deletes all snapshots except the current one
|
||||
func (sm *SnapshotManager) deleteOtherSnapshots(
|
||||
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
|
||||
|
||||
+3
-48
@@ -1,7 +1,7 @@
|
||||
// Package types provides custom types for better type safety across the
|
||||
// vaultik codebase. Using distinct types for IDs, hashes, paths, and
|
||||
// credentials prevents accidental mixing of semantically different values
|
||||
// that happen to share the same underlying type.
|
||||
// vaultik codebase. Using distinct types for IDs, hashes, and paths prevents
|
||||
// accidental mixing of semantically different values that happen to share the
|
||||
// same underlying type.
|
||||
package types //nolint:revive,nolintlint // rename decision tracked in #76
|
||||
|
||||
import (
|
||||
@@ -157,34 +157,6 @@ type FilePath string
|
||||
// Used during restore to strip the source prefix from paths.
|
||||
type SourcePath string
|
||||
|
||||
// AgeRecipient is an age public key used for encryption.
|
||||
// Format: age1... (Bech32-encoded X25519 public key)
|
||||
type AgeRecipient string
|
||||
|
||||
// AgeSecretKey is an age private key used for decryption.
|
||||
// Format: AGE-SECRET-KEY-... (Bech32-encoded X25519 private key)
|
||||
// This type should never be logged or serialized in plaintext.
|
||||
type AgeSecretKey string
|
||||
|
||||
// S3Endpoint is the URL of an S3-compatible storage endpoint.
|
||||
type S3Endpoint string
|
||||
|
||||
// BucketName is the name of an S3 bucket.
|
||||
type BucketName string
|
||||
|
||||
// S3Prefix is the path prefix within an S3 bucket.
|
||||
type S3Prefix string
|
||||
|
||||
// AWSRegion is an AWS region identifier (e.g., "us-east-1").
|
||||
type AWSRegion string
|
||||
|
||||
// AWSAccessKeyID is an AWS access key ID for authentication.
|
||||
type AWSAccessKeyID string
|
||||
|
||||
// AWSSecretAccessKey is an AWS secret access key for authentication.
|
||||
// This type should never be logged or serialized in plaintext.
|
||||
type AWSSecretAccessKey string
|
||||
|
||||
// Hostname identifies a host machine.
|
||||
type Hostname string
|
||||
|
||||
@@ -206,24 +178,7 @@ func (h ChunkHash) String() string { return string(h) }
|
||||
func (h BlobHash) String() string { return string(h) }
|
||||
func (p FilePath) String() string { return string(p) }
|
||||
func (p SourcePath) String() string { return string(p) }
|
||||
func (r AgeRecipient) String() string { return string(r) }
|
||||
func (e S3Endpoint) String() string { return string(e) }
|
||||
func (b BucketName) String() string { return string(b) }
|
||||
func (p S3Prefix) String() string { return string(p) }
|
||||
func (r AWSRegion) String() string { return string(r) }
|
||||
func (k AWSAccessKeyID) String() string { return string(k) }
|
||||
func (h Hostname) String() string { return string(h) }
|
||||
func (v Version) String() string { return string(v) }
|
||||
func (r GitRevision) String() string { return string(r) }
|
||||
func (p GlobPattern) String() string { return string(p) }
|
||||
|
||||
// Redacted String methods for sensitive types - prevents accidental logging
|
||||
|
||||
func (k AgeSecretKey) String() string { return "[REDACTED]" }
|
||||
func (k AWSSecretAccessKey) String() string { return "[REDACTED]" }
|
||||
|
||||
// Raw returns the actual value for sensitive types when explicitly needed.
|
||||
func (k AgeSecretKey) Raw() string { return string(k) }
|
||||
|
||||
// Raw returns the actual value for sensitive types when explicitly needed.
|
||||
func (k AWSSecretAccessKey) Raw() string { return string(k) }
|
||||
|
||||
@@ -2,7 +2,6 @@ package vaultik
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -57,11 +56,7 @@ func (h *hashVerifyReader) Close() error {
|
||||
return errBlobNotFullyRead
|
||||
}
|
||||
|
||||
firstHash := h.reader.Sum256()
|
||||
secondHasher := sha256.New()
|
||||
secondHasher.Write(firstHash)
|
||||
|
||||
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
|
||||
actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256()))
|
||||
if actualHashHex != h.blobHash {
|
||||
return fmt.Errorf("%w: expected %s, got %s",
|
||||
errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16])
|
||||
|
||||
@@ -40,13 +40,13 @@ func buildHashTestBlob(
|
||||
}
|
||||
|
||||
// Compute the double-SHA-256 hash of the plaintext (matches
|
||||
// blobgen.Writer.Sum256).
|
||||
// blobgen.Writer.ContentID).
|
||||
firstHash := sha256.Sum256(plaintext)
|
||||
secondHash := sha256.Sum256(firstHash[:])
|
||||
correctHash := hex.EncodeToString(secondHash[:])
|
||||
|
||||
// Verify our hash matches what blobgen.Writer produces
|
||||
writerHash := hex.EncodeToString(writer.Sum256())
|
||||
writerHash := hex.EncodeToString(writer.ContentID())
|
||||
if correctHash != writerHash {
|
||||
t.Fatalf("hash computation mismatch: manual=%s, writer=%s",
|
||||
correctHash, writerHash)
|
||||
|
||||
@@ -56,8 +56,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clean up incomplete snapshots FIRST, before any scanning
|
||||
// This is critical for data safety - see CleanupIncompleteSnapshots for details
|
||||
// Clean up incomplete snapshots FIRST, before any scanning.
|
||||
// This is critical for data safety; PruneDatabase below does it.
|
||||
hostname := v.Config.Hostname
|
||||
if hostname == "" {
|
||||
hostname, _ = os.Hostname()
|
||||
|
||||
@@ -5,7 +5,6 @@ package vaultik
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/crypto"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
@@ -21,12 +19,6 @@ import (
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// Sentinel errors for misconfigured encryption settings.
|
||||
var (
|
||||
errNoAgeRecipients = errors.New("no age recipients configured")
|
||||
errNoAgeSecretKey = errors.New("no age secret key configured")
|
||||
)
|
||||
|
||||
// Vaultik contains all dependencies needed for vaultik operations
|
||||
type Vaultik struct {
|
||||
Globals *globals.Globals
|
||||
@@ -141,26 +133,6 @@ func (v *Vaultik) CanDecrypt() bool {
|
||||
return v.Config.AgeSecretKey != ""
|
||||
}
|
||||
|
||||
// GetEncryptor creates a new Encryptor instance based on the configured age recipients
|
||||
// Returns an error if no recipients are configured
|
||||
func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) {
|
||||
if len(v.Config.AgeRecipients) == 0 {
|
||||
return nil, errNoAgeRecipients
|
||||
}
|
||||
|
||||
return crypto.NewEncryptor(v.Config.AgeRecipients)
|
||||
}
|
||||
|
||||
// GetDecryptor creates a new Decryptor instance based on the configured age secret key
|
||||
// Returns an error if no secret key is configured
|
||||
func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) {
|
||||
if v.Config.AgeSecretKey == "" {
|
||||
return nil, errNoAgeSecretKey
|
||||
}
|
||||
|
||||
return crypto.NewDecryptor(v.Config.AgeSecretKey)
|
||||
}
|
||||
|
||||
// GetFilesystem returns the filesystem instance used by Vaultik
|
||||
//
|
||||
//nolint:ireturn // afero.Fs is the filesystem abstraction by design
|
||||
|
||||
+49
-57
@@ -6,14 +6,14 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"filippo.io/age"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
@@ -88,13 +88,22 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
errSecretKeyRequired.Error(), errSecretKeyRequired)
|
||||
}
|
||||
|
||||
// Parse the age secret key once, the same way restore does, and reuse
|
||||
// the identity for the database and every blob.
|
||||
identity, err := v.prepareRestoreIdentity()
|
||||
if err != nil {
|
||||
return v.deepVerifyFailure(result, opts,
|
||||
fmt.Sprintf("parsing age secret key: %v", err), err)
|
||||
}
|
||||
|
||||
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
||||
|
||||
if !opts.JSON {
|
||||
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
|
||||
}
|
||||
|
||||
manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result)
|
||||
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
|
||||
snapshotID, opts, result, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -114,7 +123,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
|
||||
result.TotalSize = totalSize
|
||||
|
||||
err = v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize)
|
||||
err = v.runVerificationSteps(
|
||||
manifest, dbBlobs, tempDB, opts, result, totalSize, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -139,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
// loadVerificationData downloads manifest, database, and blob list for verification
|
||||
func (v *Vaultik) loadVerificationData(
|
||||
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
||||
identity age.Identity,
|
||||
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||
// hashed; a remote key (or its abbreviation, as printed for a
|
||||
@@ -188,7 +199,7 @@ func (v *Vaultik) loadVerificationData(
|
||||
|
||||
defer func() { _ = dbReader.Close() }()
|
||||
|
||||
tdb, err := v.decryptAndLoadDatabase(dbReader)
|
||||
tdb, err := v.decryptAndLoadDatabase(dbReader, identity)
|
||||
if err != nil {
|
||||
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
||||
fmt.Sprintf("failed to decrypt database: %v", err),
|
||||
@@ -230,6 +241,7 @@ func (v *Vaultik) runVerificationSteps(
|
||||
opts *VerifyOptions,
|
||||
result *VerifyResult,
|
||||
totalSize int64,
|
||||
identity age.Identity,
|
||||
) error {
|
||||
if !opts.JSON {
|
||||
v.stdoutf("Verifying manifest against database...\n")
|
||||
@@ -256,7 +268,7 @@ func (v *Vaultik) runVerificationSteps(
|
||||
len(dbBlobs), ubytes(totalSize))
|
||||
}
|
||||
|
||||
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts)
|
||||
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identity)
|
||||
if err != nil {
|
||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
@@ -281,26 +293,18 @@ func (t *tempDB) Close() error {
|
||||
}
|
||||
|
||||
// decryptAndLoadDatabase decrypts and loads the binary SQLite database
|
||||
// from the encrypted stream.
|
||||
func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) {
|
||||
// Get decryptor
|
||||
decryptor, err := v.GetDecryptor()
|
||||
// from the encrypted stream. It reads through the same blobgen reader restore
|
||||
// uses, streaming the decrypted, decompressed database to a temp file.
|
||||
func (v *Vaultik) decryptAndLoadDatabase(
|
||||
reader io.ReadCloser, identity age.Identity,
|
||||
) (*tempDB, error) {
|
||||
// Decrypt and decompress through the shared blobgen reader.
|
||||
blobReader, err := blobgen.NewReader(reader, identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get decryptor: %w", err)
|
||||
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||
}
|
||||
|
||||
// Decrypt the stream
|
||||
decryptedReader, err := decryptor.DecryptStream(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt database: %w", err)
|
||||
}
|
||||
|
||||
// Decompress the binary database
|
||||
decompressor, err := zstd.NewReader(decryptedReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create decompressor: %w", err)
|
||||
}
|
||||
defer decompressor.Close()
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
|
||||
// Materialize the decrypted database inside a private (0700) temp
|
||||
// directory so it is never world-readable, and remove the whole
|
||||
@@ -330,7 +334,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
|
||||
// Stream decompress directly to file
|
||||
log.Info("Decompressing database...")
|
||||
|
||||
written, err := io.Copy(tempFile, decompressor)
|
||||
written, err := io.Copy(tempFile, blobReader)
|
||||
if err != nil {
|
||||
_ = tempFile.Close()
|
||||
|
||||
@@ -355,7 +359,9 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
|
||||
}
|
||||
|
||||
// verifyBlob downloads and verifies a single blob
|
||||
func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
||||
func (v *Vaultik) verifyBlob(
|
||||
blobInfo snapshot.BlobInfo, db *sql.DB, identity age.Identity,
|
||||
) error {
|
||||
// Download blob using shared fetch method
|
||||
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
|
||||
if err != nil {
|
||||
@@ -364,38 +370,23 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
||||
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Get decryptor
|
||||
decryptor, err := v.GetDecryptor()
|
||||
// Decrypt and decompress through the shared blobgen reader, which hashes
|
||||
// the plaintext as it is read. A blob's hash — its remote name — is the
|
||||
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
||||
// encrypted bytes.
|
||||
blobReader, err := blobgen.NewReader(reader, identity)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get decryptor: %w", err)
|
||||
return fmt.Errorf("failed to create blob reader: %w", err)
|
||||
}
|
||||
|
||||
// Decrypt blob
|
||||
decryptedReader, err := decryptor.DecryptStream(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt: %w", err)
|
||||
}
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
|
||||
// Decompress blob
|
||||
decompressor, err := zstd.NewReader(decryptedReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decompress: %w", err)
|
||||
}
|
||||
defer decompressor.Close()
|
||||
|
||||
// A blob's hash — its remote name — is the double SHA256 of its
|
||||
// decompressed plaintext (see blobgen.Writer.Sum256), not of the
|
||||
// encrypted bytes. Hash the plaintext as chunk verification streams
|
||||
// it, then compare on completion.
|
||||
plaintextHasher := sha256.New()
|
||||
hashedStream := io.TeeReader(decompressor, plaintextHasher)
|
||||
|
||||
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream)
|
||||
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, blobReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash)
|
||||
err = v.verifyBlobFinalIntegrity(blobReader, blobInfo.Hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -501,11 +492,12 @@ func (v *Vaultik) verifyBlobChunks(
|
||||
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
||||
// decompressed stream and that the blob hash matches the expected value.
|
||||
func (v *Vaultik) verifyBlobFinalIntegrity(
|
||||
plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string,
|
||||
blobReader *blobgen.Reader, expectedHash string,
|
||||
) error {
|
||||
// Verify no remaining data in blob - if the chunk list is accurate,
|
||||
// the blob should be fully consumed.
|
||||
remaining, err := io.Copy(io.Discard, plaintext)
|
||||
// the blob should be fully consumed. Draining to EOF also completes the
|
||||
// reader's plaintext hash.
|
||||
remaining, err := io.Copy(io.Discard, blobReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
||||
}
|
||||
@@ -514,10 +506,9 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
|
||||
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
||||
}
|
||||
|
||||
// The blob hash is the double SHA256 of its plaintext content.
|
||||
firstHash := plaintextHasher.Sum(nil)
|
||||
secondHash := sha256.Sum256(firstHash)
|
||||
calculatedBlobHash := hex.EncodeToString(secondHash[:])
|
||||
// The blob hash is the double SHA-256 of its plaintext content.
|
||||
calculatedBlobHash := hex.EncodeToString(
|
||||
blobgen.DoubleSHA256(blobReader.Sum256()))
|
||||
|
||||
if calculatedBlobHash != expectedHash {
|
||||
return fmt.Errorf("%w: calculated %s, expected %s",
|
||||
@@ -667,6 +658,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
|
||||
// each blob using the database as source.
|
||||
func (v *Vaultik) performDeepVerificationFromDB(
|
||||
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
|
||||
identity age.Identity,
|
||||
) error {
|
||||
// Calculate total bytes for ETA
|
||||
var totalBytesExpected int64
|
||||
@@ -684,7 +676,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
|
||||
|
||||
for i, blobInfo := range blobs {
|
||||
// Verify individual blob
|
||||
err := v.verifyBlob(blobInfo, db)
|
||||
err := v.verifyBlob(blobInfo, db, identity)
|
||||
if err != nil {
|
||||
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
|
||||
}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/crypto"
|
||||
)
|
||||
|
||||
// TestTeeReaderWithDecryption tests that TeeReader correctly hashes all encrypted
|
||||
// bytes when streaming through age decryption and zstd decompression.
|
||||
// This validates the verification path: hash encrypted blob -> decrypt -> decompress.
|
||||
func TestTeeReaderWithDecryption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test data - use random data that doesn't compress well (5MB)
|
||||
testData := make([]byte, 5*1024*1024)
|
||||
_, err := rand.Read(testData)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Compress the data
|
||||
var compressedBuf bytes.Buffer
|
||||
|
||||
compressor, err := zstd.NewWriter(&compressedBuf,
|
||||
zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
require.NoError(t, err)
|
||||
_, err = compressor.Write(testData)
|
||||
require.NoError(t, err)
|
||||
err = compressor.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Encrypt the compressed data
|
||||
testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" +
|
||||
"mu62kv3s89gmvv"
|
||||
testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" +
|
||||
"T00J9CCW9986M3XY87PSGP89AQ"
|
||||
|
||||
encryptor, err := crypto.NewEncryptor([]string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
|
||||
var encryptedBuf bytes.Buffer
|
||||
|
||||
err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes()))
|
||||
require.NoError(t, err)
|
||||
|
||||
encryptedData := encryptedBuf.Bytes()
|
||||
|
||||
// Calculate the expected hash of the encrypted data directly
|
||||
expectedHash := sha256.Sum256(encryptedData)
|
||||
expectedHashStr := hex.EncodeToString(expectedHash[:])
|
||||
|
||||
t.Logf("Encrypted data size: %d bytes", len(encryptedData))
|
||||
t.Logf("Expected hash: %s", expectedHashStr)
|
||||
|
||||
// Now simulate what verifyBlob does: use TeeReader to hash while decrypting
|
||||
decryptor, err := crypto.NewDecryptor(testSecretKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create hasher and tee reader
|
||||
hasher := sha256.New()
|
||||
reader := bytes.NewReader(encryptedData)
|
||||
teeReader := io.TeeReader(reader, hasher)
|
||||
|
||||
// Decrypt through the tee reader
|
||||
decryptedReader, err := decryptor.DecryptStream(teeReader)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Decompress
|
||||
decompressor, err := zstd.NewReader(decryptedReader)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer decompressor.Close()
|
||||
|
||||
// Read all decompressed data (simulating chunk verification)
|
||||
decompressedData, err := io.ReadAll(decompressor)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify we got the original data back
|
||||
assert.Equal(t, testData, decompressedData, "Decompressed data should match original")
|
||||
|
||||
// Drain remaining decompressed data (should be 0)
|
||||
remaining, err := io.Copy(io.Discard, decompressor)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), remaining, "No remaining decompressed data")
|
||||
|
||||
// Calculate hash from tee reader
|
||||
calculatedHashStr := hex.EncodeToString(hasher.Sum(nil))
|
||||
t.Logf("Calculated hash (before drain): %s", calculatedHashStr)
|
||||
|
||||
// Verify the hash matches the direct hash of encrypted data
|
||||
assert.Equal(t, expectedHashStr, calculatedHashStr,
|
||||
"Hash calculated via TeeReader should match direct hash of encrypted data")
|
||||
}
|
||||
Reference in New Issue
Block a user