Remove the unused crypto path and write the blob-ID hash step once #191

Merged
clawbot merged 1 commits from issue-151-remove-second-encryption-path into next 2026-09-22 13:46:02 +02:00
18 changed files with 154 additions and 976 deletions
+1 -2
View File
@@ -286,7 +286,6 @@ Key methods:
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
- `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
- `CleanupIncompleteSnapshots(ctx, hostname)` → Remove failed snapshots
### `internal/database`
SQLite database for local index. Single-writer mode for thread safety.
@@ -307,7 +306,7 @@ Repository interfaces:
```
CreateSnapshot(opts)
├─► CleanupIncompleteSnapshots() // Critical: avoid dedup errors
├─► PruneDatabase() // Critical: avoid dedup errors
├─► SnapshotManager.CreateSnapshot() // Create DB record
+1 -1
View File
@@ -487,7 +487,7 @@ func (p *Packer) closeBlobWriter() (string, int64, error) {
return "", 0, fmt.Errorf("seeking to start: %w", err)
}
finalHash := p.currentBlob.writer.Sum256()
finalHash := p.currentBlob.writer.ContentID()
return hex.EncodeToString(finalHash), finalSize, nil
}
-89
View File
@@ -1,89 +0,0 @@
// Package blobgen implements the blob data pipeline: streaming zstd
// compression, age encryption, and SHA256 content hashing for blob
// creation, plus the matching decrypt/decompress/verify reader.
package blobgen
import (
"bytes"
"encoding/hex"
"fmt"
"io"
)
// CompressResult contains the results of compression
type CompressResult struct {
Data []byte
UncompressedSize int64
CompressedSize int64
SHA256 string
}
// CompressData compresses and encrypts data, returning the result with hash
func CompressData(
data []byte, compressionLevel int, recipients []string,
) (*CompressResult, error) {
var buf bytes.Buffer
// Create writer
w, err := NewWriter(&buf, compressionLevel, recipients)
if err != nil {
return nil, fmt.Errorf("creating writer: %w", err)
}
// Write data
_, err = w.Write(data)
if err != nil {
_ = w.Close()
return nil, fmt.Errorf("writing data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return nil, fmt.Errorf("closing writer: %w", err)
}
return &CompressResult{
Data: buf.Bytes(),
UncompressedSize: int64(len(data)),
CompressedSize: int64(buf.Len()),
SHA256: hex.EncodeToString(w.Sum256()),
}, nil
}
// CompressStream compresses and encrypts from reader to writer, returning
// the number of uncompressed bytes written and the content hash.
func CompressStream(
dst io.Writer, src io.Reader, compressionLevel int, recipients []string,
) (int64, string, error) {
// Create writer
w, err := NewWriter(dst, compressionLevel, recipients)
if err != nil {
return 0, "", fmt.Errorf("creating writer: %w", err)
}
closed := false
defer func() {
if !closed {
_ = w.Close()
}
}()
// Copy data
_, err = io.Copy(w, src)
if err != nil {
return 0, "", fmt.Errorf("copying data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return 0, "", fmt.Errorf("closing writer: %w", err)
}
closed = true
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
}
-80
View File
@@ -1,80 +0,0 @@
package blobgen_test
import (
"bytes"
"crypto/rand"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blobgen"
)
// testRecipient is a static age recipient for tests.
const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
// TestCompressStreamNoDoubleClose is a regression test for issue #28.
// It verifies that CompressStream does not panic or return an error due to
// double-closing the underlying blobgen.Writer. Before the fix in PR #33,
// the explicit Close() on the happy path combined with defer Close() would
// cause a double close.
func TestCompressStreamNoDoubleClose(t *testing.T) {
t.Parallel()
input := []byte("regression test data for issue #28 double-close fix")
var buf bytes.Buffer
written, hash, err := blobgen.CompressStream(
&buf, bytes.NewReader(input), 3, []string{testRecipient})
require.NoError(t, err, "CompressStream should not return an error")
assert.Positive(t, written, "expected bytes written > 0")
assert.NotEmpty(t, hash, "expected non-empty hash")
assert.Positive(t, buf.Len(), "expected non-empty output")
}
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
// to ensure no double-close issues surface under heavier I/O.
func TestCompressStreamLargeInput(t *testing.T) {
t.Parallel()
data := make([]byte, 512*1024) // 512 KB
_, err := rand.Read(data)
require.NoError(t, err)
var buf bytes.Buffer
written, hash, err := blobgen.CompressStream(
&buf, bytes.NewReader(data), 3, []string{testRecipient})
require.NoError(t, err)
assert.Positive(t, written)
assert.NotEmpty(t, hash)
}
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
// without double-close issues.
func TestCompressStreamEmptyInput(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
_, hash, err := blobgen.CompressStream(
&buf, strings.NewReader(""), 3, []string{testRecipient})
require.NoError(t, err)
assert.NotEmpty(t, hash)
}
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
// ensuring the explicit Close + error-path Close pattern is also safe.
func TestCompressDataNoDoubleClose(t *testing.T) {
t.Parallel()
input := []byte("CompressData regression test for double-close")
result, err := blobgen.CompressData(input, 3, []string{testRecipient})
require.NoError(t, err)
assert.Positive(t, result.CompressedSize)
assert.Equal(t, result.UncompressedSize, int64(len(input)))
assert.NotEmpty(t, result.SHA256)
}
+3 -1
View File
@@ -64,7 +64,9 @@ func (r *Reader) Close() error {
return nil
}
// Sum256 returns the SHA256 hash of all data read
// Sum256 returns the single SHA-256 of the plaintext read so far. This is the
// first hash only; the stored object name is its double hash, which callers
// obtain by passing this digest to DoubleSHA256.
func (r *Reader) Sum256() []byte {
return r.hasher.Sum(nil)
}
+54
View File
@@ -0,0 +1,54 @@
package blobgen_test
import (
"bytes"
"io"
"testing"
"filippo.io/age"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blobgen"
)
// TestMultipleRecipients verifies that data written for several recipients can
// be read back by each recipient's identity. Moved from internal/crypto, which
// held the only multi-recipient test; blobgen is now the sole encryption path.
func TestMultipleRecipients(t *testing.T) {
t.Parallel()
identities := make([]*age.X25519Identity, 3)
recipients := make([]string, 3)
for i := range identities {
identity, err := age.GenerateX25519Identity()
require.NoError(t, err)
identities[i] = identity
recipients[i] = identity.Recipient().String()
}
plaintext := []byte("Secret message for multiple recipients")
var encrypted bytes.Buffer
writer, err := blobgen.NewWriter(&encrypted, 3, recipients)
require.NoError(t, err)
_, err = writer.Write(plaintext)
require.NoError(t, err)
require.NoError(t, writer.Close())
// Every recipient's identity must recover the original plaintext.
for i, identity := range identities {
reader, err := blobgen.NewReader(
bytes.NewReader(encrypted.Bytes()), identity)
require.NoError(t, err, "recipient %d should open the reader", i+1)
got, err := io.ReadAll(reader)
require.NoError(t, err, "recipient %d should read the plaintext", i+1)
require.NoError(t, reader.Close())
assert.Equal(t, plaintext, got,
"recipient %d should recover the original plaintext", i+1)
}
}
+21 -11
View File
@@ -1,3 +1,6 @@
// Package blobgen implements the blob data pipeline: streaming zstd
// compression, age encryption, and SHA256 content hashing for blob
// creation, plus the matching decrypt/decompress/verify reader.
package blobgen
import (
@@ -12,6 +15,18 @@ import (
"github.com/klauspost/compress/zstd"
)
// DoubleSHA256 returns the double SHA-256 of content whose single SHA-256
// digest is sum: it hashes that digest once more. Stored objects are named by
// this second hash so that a name never reveals whether known content is
// present — an attacker who knows a plaintext, and thus its SHA-256, still
// cannot derive the stored name without hashing the digest again. Both a blob
// and the metadata database export are named this way.
func DoubleSHA256(sum []byte) []byte {
h := sha256.Sum256(sum)
return h[:]
}
// Zstd compression level bounds accepted by NewWriter.
const (
minCompressionLevel = 1
@@ -130,17 +145,12 @@ func (w *Writer) Close() error {
return nil
}
// Sum256 returns the double SHA256 hash of the uncompressed input data.
// Double hashing (SHA256(SHA256(data))) prevents information leakage about
// the plaintext - an attacker cannot confirm existence of known content
// by computing its hash and checking for a matching blob filename.
func (w *Writer) Sum256() []byte {
// First hash: SHA256(plaintext)
firstHash := w.hasher.Sum(nil)
// Second hash: SHA256(firstHash) - this is the blob ID
secondHash := sha256.Sum256(firstHash)
return secondHash[:]
// ContentID returns the double SHA-256 of the uncompressed input data: the
// name under which this content is stored. It is the second hash of the
// running SHA-256, via DoubleSHA256; see that function for why content is
// named this way rather than by its plain SHA-256.
func (w *Writer) ContentID() []byte {
return DoubleSHA256(w.hasher.Sum(nil))
}
// BytesWritten returns the number of uncompressed bytes written
+5 -5
View File
@@ -12,7 +12,7 @@ import (
"sneak.berlin/go/vaultik/internal/blobgen"
)
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns
// TestWriterHashIsDoubleHash verifies that Writer.ContentID() returns
// the double hash SHA256(SHA256(plaintext)) for security.
// Double hashing prevents attackers from confirming existence of known content.
func TestWriterHashIsDoubleHash(t *testing.T) {
@@ -43,7 +43,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
require.NoError(t, err)
// Get the hash from the writer
writerHash := hex.EncodeToString(writer.Sum256())
writerHash := hex.EncodeToString(writer.ContentID())
// Calculate the expected double hash: SHA256(SHA256(plaintext))
firstHash := sha256.Sum256(testData)
@@ -60,7 +60,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
// The writer hash should match the double hash
assert.Equal(t, expectedDoubleHash, writerHash,
"Writer.Sum256() should return SHA256(SHA256(plaintext)) for security")
"Writer.ContentID() should return SHA256(SHA256(plaintext)) for security")
// Verify it's NOT the single hash (would leak information)
assert.NotEqual(t, singleHashStr, writerHash,
@@ -93,8 +93,8 @@ func TestWriterDeterministicHash(t *testing.T) {
require.NoError(t, err)
require.NoError(t, writer2.Close())
hash1 := hex.EncodeToString(writer1.Sum256())
hash2 := hex.EncodeToString(writer2.Sum256())
hash1 := hex.EncodeToString(writer1.ContentID())
hash2 := hex.EncodeToString(writer2.ContentID())
// Hashes should be identical (deterministic)
assert.Equal(t, hash1, hash2, "Same input should produce same hash")
-233
View File
@@ -1,233 +0,0 @@
// Package crypto provides thread-safe age encryption and decryption
// helpers used to protect blob and metadata content.
package crypto //nolint:revive,nolintlint // stdlib crypto unused; see #76
import (
"bytes"
"errors"
"fmt"
"io"
"sync"
"filippo.io/age"
"go.uber.org/fx"
)
// ErrNoRecipients is returned when an encryptor is created or updated
// without any recipient public keys.
var ErrNoRecipients = errors.New("at least one recipient is required")
// errInvalidRecipient is returned when a recipient string does not parse as
// an X25519 age1... public key. It omits the value, which can be sensitive.
var errInvalidRecipient = errors.New(
"not a valid X25519 age1... recipient")
// Encryptor provides thread-safe encryption using the age encryption library.
// It supports encrypting data for multiple recipients simultaneously, allowing
// any of the corresponding private keys to decrypt the data. This is useful
// for backup scenarios where multiple parties should be able to decrypt the data.
type Encryptor struct {
recipients []age.Recipient
mu sync.RWMutex
}
// NewEncryptor creates a new encryptor with the given age public keys.
// Each public key should be a valid age X25519 recipient string (e.g., "age1...")
// At least one recipient must be provided. Returns an error if any of the
// public keys are invalid or if no recipients are specified.
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if len(publicKeys) == 0 {
return nil, ErrNoRecipients
}
recipients := make([]age.Recipient, 0, len(publicKeys))
for i, key := range publicKeys {
// The key string can be sensitive (e.g. a secret key pasted by
// mistake), so the error names its position, never its value.
recipient, err := age.ParseX25519Recipient(key)
if err != nil {
return nil, fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
}
recipients = append(recipients, recipient)
}
return &Encryptor{
recipients: recipients,
}, nil
}
// Encrypt encrypts data using age encryption for all configured recipients.
// The encrypted data can be decrypted by any of the corresponding private keys.
// This method is suitable for small to medium amounts of data that fit in memory.
// For large data streams, use EncryptStream or EncryptWriter instead.
func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
e.mu.RLock()
recipients := e.recipients
e.mu.RUnlock()
var buf bytes.Buffer
// Create encrypted writer for all recipients
w, err := age.Encrypt(&buf, recipients...)
if err != nil {
return nil, fmt.Errorf("creating encrypted writer: %w", err)
}
// Write data
_, err = w.Write(data)
if err != nil {
return nil, fmt.Errorf("writing encrypted data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return nil, fmt.Errorf("closing encrypted writer: %w", err)
}
return buf.Bytes(), nil
}
// EncryptStream encrypts data from reader to writer using age encryption.
// This method is suitable for encrypting large files or streams as it processes
// data in a streaming fashion without loading everything into memory.
// The encrypted data is written directly to the destination writer.
func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
e.mu.RLock()
recipients := e.recipients
e.mu.RUnlock()
// Create encrypted writer for all recipients
w, err := age.Encrypt(dst, recipients...)
if err != nil {
return fmt.Errorf("creating encrypted writer: %w", err)
}
// Copy data
_, err = io.Copy(w, src)
if err != nil {
return fmt.Errorf("copying encrypted data: %w", err)
}
// Close to flush
err = w.Close()
if err != nil {
return fmt.Errorf("closing encrypted writer: %w", err)
}
return nil
}
// EncryptWriter creates a writer that encrypts data written to it.
// All data written to the returned WriteCloser will be encrypted and written
// to the destination writer. The caller must call Close() on the returned
// writer to ensure all encrypted data is properly flushed and finalized.
// This is useful for integrating encryption into existing writer-based pipelines.
func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
e.mu.RLock()
recipients := e.recipients
e.mu.RUnlock()
// Create encrypted writer for all recipients
w, err := age.Encrypt(dst, recipients...)
if err != nil {
return nil, fmt.Errorf("creating encrypted writer: %w", err)
}
return w, nil
}
// UpdateRecipients updates the recipients for future encryption operations.
// This method is thread-safe and can be called while other encryption operations
// are in progress. Existing encryption operations will continue with the old
// recipients. At least one recipient must be provided. Returns an error if any
// of the public keys are invalid or if no recipients are specified.
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if len(publicKeys) == 0 {
return ErrNoRecipients
}
recipients := make([]age.Recipient, 0, len(publicKeys))
for i, key := range publicKeys {
// The key string can be sensitive (e.g. a secret key pasted by
// mistake), so the error names its position, never its value.
recipient, err := age.ParseX25519Recipient(key)
if err != nil {
return fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
}
recipients = append(recipients, recipient)
}
e.mu.Lock()
e.recipients = recipients
e.mu.Unlock()
return nil
}
// Decryptor provides thread-safe decryption using the age encryption library.
// It uses a private key to decrypt data that was encrypted for the corresponding
// public key.
type Decryptor struct {
identity age.Identity
mu sync.RWMutex
}
// NewDecryptor creates a new decryptor with the given age private key.
// The private key should be a valid age X25519 identity string.
// Returns an error if the private key is invalid.
func NewDecryptor(privateKey string) (*Decryptor, error) {
identity, err := age.ParseX25519Identity(privateKey)
if err != nil {
return nil, fmt.Errorf("parsing age identity: %w", err)
}
return &Decryptor{
identity: identity,
}, nil
}
// Decrypt decrypts data using age decryption.
// This method is suitable for small to medium amounts of data that fit in memory.
// For large data streams, use DecryptStream instead.
func (d *Decryptor) Decrypt(data []byte) ([]byte, error) {
d.mu.RLock()
identity := d.identity
d.mu.RUnlock()
r, err := age.Decrypt(bytes.NewReader(data), identity)
if err != nil {
return nil, fmt.Errorf("creating decrypted reader: %w", err)
}
decrypted, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("reading decrypted data: %w", err)
}
return decrypted, nil
}
// DecryptStream returns a reader that decrypts data from the provided reader.
// This method is suitable for decrypting large files or streams as it processes
// data in a streaming fashion without loading everything into memory.
// The caller should close the input reader when done.
func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
d.mu.RLock()
identity := d.identity
d.mu.RUnlock()
r, err := age.Decrypt(src, identity)
if err != nil {
return nil, fmt.Errorf("creating decrypted reader: %w", err)
}
return r, nil
}
// Module exports the crypto module for fx dependency injection.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("crypto")
-198
View File
@@ -1,198 +0,0 @@
package crypto_test
import (
"bytes"
"strings"
"testing"
"filippo.io/age"
"sneak.berlin/go/vaultik/internal/crypto"
)
func TestEncryptor(t *testing.T) {
t.Parallel()
// Generate a test key pair
identity, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity: %v", err)
}
publicKey := identity.Recipient().String()
// Create encryptor
enc, err := crypto.NewEncryptor([]string{publicKey})
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Test data
plaintext := []byte("Hello, World! This is a test message.")
// Encrypt
ciphertext, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Verify it's actually encrypted (should be larger and different)
if bytes.Equal(plaintext, ciphertext) {
t.Error("ciphertext equals plaintext")
}
// Decrypt to verify
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
if err != nil {
t.Fatalf("failed to decrypt: %v", err)
}
var decrypted bytes.Buffer
_, err = decrypted.ReadFrom(r)
if err != nil {
t.Fatalf("failed to read decrypted data: %v", err)
}
if !bytes.Equal(plaintext, decrypted.Bytes()) {
t.Error("decrypted data doesn't match original")
}
}
func TestEncryptorMultipleRecipients(t *testing.T) {
t.Parallel()
// Generate three test key pairs
identity1, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity1: %v", err)
}
identity2, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity2: %v", err)
}
identity3, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity3: %v", err)
}
publicKeys := []string{
identity1.Recipient().String(),
identity2.Recipient().String(),
identity3.Recipient().String(),
}
// Create encryptor with multiple recipients
enc, err := crypto.NewEncryptor(publicKeys)
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Test data
plaintext := []byte("Secret message for multiple recipients")
// Encrypt
ciphertext, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Verify each recipient can decrypt
identities := []age.Identity{identity1, identity2, identity3}
for i, identity := range identities {
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
if err != nil {
t.Fatalf("recipient %d failed to decrypt: %v", i+1, err)
}
var decrypted bytes.Buffer
_, err = decrypted.ReadFrom(r)
if err != nil {
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
}
if !bytes.Equal(plaintext, decrypted.Bytes()) {
t.Errorf("recipient %d: decrypted data doesn't match original", i+1)
}
}
}
func TestEncryptorUpdateRecipients(t *testing.T) {
t.Parallel()
// Generate two identities
identity1, _ := age.GenerateX25519Identity()
identity2, _ := age.GenerateX25519Identity()
publicKey1 := identity1.Recipient().String()
publicKey2 := identity2.Recipient().String()
// Create encryptor with first key
enc, err := crypto.NewEncryptor([]string{publicKey1})
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Encrypt with first key
plaintext := []byte("test data")
ciphertext1, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Update to second key
err = enc.UpdateRecipients([]string{publicKey2})
if err != nil {
t.Fatalf("failed to update recipients: %v", err)
}
// Encrypt with second key
ciphertext2, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// First ciphertext should only decrypt with first identity
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
if err != nil {
t.Error("failed to decrypt with identity1")
}
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
if err == nil {
t.Error("should not decrypt with identity2")
}
// Second ciphertext should only decrypt with second identity
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
if err != nil {
t.Error("failed to decrypt with identity2")
}
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
if err == nil {
t.Error("should not decrypt with identity1")
}
}
// TestNewEncryptorSecretKeyNotEchoed verifies that a secret key mistakenly
// passed as a recipient does not appear in the returned error. A recipient
// string can be sensitive, so the error must name only the position.
func TestNewEncryptorSecretKeyNotEchoed(t *testing.T) {
t.Parallel()
secretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
_, err := crypto.NewEncryptor([]string{secretKey})
if err == nil {
t.Fatal("NewEncryptor returned nil, want error")
}
if strings.Contains(err.Error(), secretKey) {
t.Fatalf("error echoed the recipient value: %v", err)
}
}
+1 -102
View File
@@ -295,68 +295,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(
return nil
}
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have
// metadata in S3. This is critical for data safety: incomplete snapshots
// can cause deduplication to skip files that were never successfully
// backed up, resulting in data loss.
func (sm *SnapshotManager) CleanupIncompleteSnapshots(
ctx context.Context, hostname string,
) error {
log.Info("Checking for incomplete snapshots", "hostname", hostname)
// Get all incomplete snapshots for this hostname
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
// Check each incomplete snapshot for metadata in storage
for _, snapshot := range incompleteSnapshots {
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst",
RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record",
"snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w",
snapshot.ID, err)
}
log.Info("Deleted incomplete snapshot record and associated data",
"snapshot_id", snapshot.ID)
} else {
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database",
"snapshot_id", snapshot.ID)
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database",
"snapshot_id", snapshot.ID, "error", err)
}
}
}
return nil
}
// CleanupOrphanedData removes files, chunks, and blobs that are no longer
// referenced by any snapshot. This should be called periodically to clean
// up data from deleted or incomplete snapshots.
@@ -759,7 +697,7 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
writerClosed = true
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.ContentID()))
return nil
}
@@ -933,45 +871,6 @@ type ExtendedBackupStats struct {
UploadDurationMs int64 // Total milliseconds spent uploading to S3
}
// deleteSnapshot removes a snapshot and all its associations from the database
func (sm *SnapshotManager) deleteSnapshot(
ctx context.Context, snapshotID string,
) error {
// Delete snapshot_files entries
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot files: %w", err)
}
// Delete snapshot_blobs entries
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot blobs: %w", err)
}
// Delete uploads entries (has foreign key to snapshots without CASCADE)
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot uploads: %w", err)
}
// Delete the snapshot itself
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot: %w", err)
}
// Clean up orphaned data
log.Debug("Cleaning up orphaned records in main database")
err = sm.CleanupOrphanedData(ctx)
if err != nil {
return fmt.Errorf("cleaning up orphaned data: %w", err)
}
return nil
}
// deleteOtherSnapshots deletes all snapshots except the current one
func (sm *SnapshotManager) deleteOtherSnapshots(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
+14 -59
View File
@@ -1,7 +1,7 @@
// Package types provides custom types for better type safety across the
// vaultik codebase. Using distinct types for IDs, hashes, paths, and
// credentials prevents accidental mixing of semantically different values
// that happen to share the same underlying type.
// vaultik codebase. Using distinct types for IDs, hashes, and paths prevents
// accidental mixing of semantically different values that happen to share the
// same underlying type.
package types //nolint:revive,nolintlint // rename decision tracked in #76
import (
@@ -157,34 +157,6 @@ type FilePath string
// Used during restore to strip the source prefix from paths.
type SourcePath string
// AgeRecipient is an age public key used for encryption.
// Format: age1... (Bech32-encoded X25519 public key)
type AgeRecipient string
// AgeSecretKey is an age private key used for decryption.
// Format: AGE-SECRET-KEY-... (Bech32-encoded X25519 private key)
// This type should never be logged or serialized in plaintext.
type AgeSecretKey string
// S3Endpoint is the URL of an S3-compatible storage endpoint.
type S3Endpoint string
// BucketName is the name of an S3 bucket.
type BucketName string
// S3Prefix is the path prefix within an S3 bucket.
type S3Prefix string
// AWSRegion is an AWS region identifier (e.g., "us-east-1").
type AWSRegion string
// AWSAccessKeyID is an AWS access key ID for authentication.
type AWSAccessKeyID string
// AWSSecretAccessKey is an AWS secret access key for authentication.
// This type should never be logged or serialized in plaintext.
type AWSSecretAccessKey string
// Hostname identifies a host machine.
type Hostname string
@@ -199,31 +171,14 @@ type GlobPattern string
// String methods for Stringer interface
func (id FileID) String() string { return uuid.UUID(id).String() }
func (id BlobID) String() string { return uuid.UUID(id).String() }
func (id SnapshotID) String() string { return string(id) }
func (h ChunkHash) String() string { return string(h) }
func (h BlobHash) String() string { return string(h) }
func (p FilePath) String() string { return string(p) }
func (p SourcePath) String() string { return string(p) }
func (r AgeRecipient) String() string { return string(r) }
func (e S3Endpoint) String() string { return string(e) }
func (b BucketName) String() string { return string(b) }
func (p S3Prefix) String() string { return string(p) }
func (r AWSRegion) String() string { return string(r) }
func (k AWSAccessKeyID) String() string { return string(k) }
func (h Hostname) String() string { return string(h) }
func (v Version) String() string { return string(v) }
func (r GitRevision) String() string { return string(r) }
func (p GlobPattern) String() string { return string(p) }
// Redacted String methods for sensitive types - prevents accidental logging
func (k AgeSecretKey) String() string { return "[REDACTED]" }
func (k AWSSecretAccessKey) String() string { return "[REDACTED]" }
// Raw returns the actual value for sensitive types when explicitly needed.
func (k AgeSecretKey) Raw() string { return string(k) }
// Raw returns the actual value for sensitive types when explicitly needed.
func (k AWSSecretAccessKey) Raw() string { return string(k) }
func (id FileID) String() string { return uuid.UUID(id).String() }
func (id BlobID) String() string { return uuid.UUID(id).String() }
func (id SnapshotID) String() string { return string(id) }
func (h ChunkHash) String() string { return string(h) }
func (h BlobHash) String() string { return string(h) }
func (p FilePath) String() string { return string(p) }
func (p SourcePath) String() string { return string(p) }
func (h Hostname) String() string { return string(h) }
func (v Version) String() string { return string(v) }
func (r GitRevision) String() string { return string(r) }
func (p GlobPattern) String() string { return string(p) }
+1 -6
View File
@@ -2,7 +2,6 @@ package vaultik
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
@@ -57,11 +56,7 @@ func (h *hashVerifyReader) Close() error {
return errBlobNotFullyRead
}
firstHash := h.reader.Sum256()
secondHasher := sha256.New()
secondHasher.Write(firstHash)
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256()))
if actualHashHex != h.blobHash {
return fmt.Errorf("%w: expected %s, got %s",
errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16])
+2 -2
View File
@@ -40,13 +40,13 @@ func buildHashTestBlob(
}
// Compute the double-SHA-256 hash of the plaintext (matches
// blobgen.Writer.Sum256).
// blobgen.Writer.ContentID).
firstHash := sha256.Sum256(plaintext)
secondHash := sha256.Sum256(firstHash[:])
correctHash := hex.EncodeToString(secondHash[:])
// Verify our hash matches what blobgen.Writer produces
writerHash := hex.EncodeToString(writer.Sum256())
writerHash := hex.EncodeToString(writer.ContentID())
if correctHash != writerHash {
t.Fatalf("hash computation mismatch: manual=%s, writer=%s",
correctHash, writerHash)
+2 -2
View File
@@ -56,8 +56,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
return err
}
// Clean up incomplete snapshots FIRST, before any scanning
// This is critical for data safety - see CleanupIncompleteSnapshots for details
// Clean up incomplete snapshots FIRST, before any scanning.
// This is critical for data safety; PruneDatabase below does it.
hostname := v.Config.Hostname
if hostname == "" {
hostname, _ = os.Hostname()
-28
View File
@@ -5,7 +5,6 @@ package vaultik
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
@@ -13,7 +12,6 @@ import (
"github.com/spf13/afero"
"go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/crypto"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/globals"
"sneak.berlin/go/vaultik/internal/snapshot"
@@ -21,12 +19,6 @@ import (
"sneak.berlin/go/vaultik/internal/ui"
)
// Sentinel errors for misconfigured encryption settings.
var (
errNoAgeRecipients = errors.New("no age recipients configured")
errNoAgeSecretKey = errors.New("no age secret key configured")
)
// Vaultik contains all dependencies needed for vaultik operations
type Vaultik struct {
Globals *globals.Globals
@@ -141,26 +133,6 @@ func (v *Vaultik) CanDecrypt() bool {
return v.Config.AgeSecretKey != ""
}
// GetEncryptor creates a new Encryptor instance based on the configured age recipients
// Returns an error if no recipients are configured
func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) {
if len(v.Config.AgeRecipients) == 0 {
return nil, errNoAgeRecipients
}
return crypto.NewEncryptor(v.Config.AgeRecipients)
}
// GetDecryptor creates a new Decryptor instance based on the configured age secret key
// Returns an error if no secret key is configured
func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) {
if v.Config.AgeSecretKey == "" {
return nil, errNoAgeSecretKey
}
return crypto.NewDecryptor(v.Config.AgeSecretKey)
}
// GetFilesystem returns the filesystem instance used by Vaultik
//
//nolint:ireturn // afero.Fs is the filesystem abstraction by design
+49 -57
View File
@@ -6,14 +6,14 @@ import (
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"os"
"path/filepath"
"time"
"github.com/klauspost/compress/zstd"
"filippo.io/age"
"sneak.berlin/go/vaultik/internal/blobgen"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
@@ -88,13 +88,22 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
errSecretKeyRequired.Error(), errSecretKeyRequired)
}
// Parse the age secret key once, the same way restore does, and reuse
// the identity for the database and every blob.
identity, err := v.prepareRestoreIdentity()
if err != nil {
return v.deepVerifyFailure(result, opts,
fmt.Sprintf("parsing age secret key: %v", err), err)
}
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
if !opts.JSON {
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
}
manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result)
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
snapshotID, opts, result, identity)
if err != nil {
return err
}
@@ -114,7 +123,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
result.TotalSize = totalSize
err = v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize)
err = v.runVerificationSteps(
manifest, dbBlobs, tempDB, opts, result, totalSize, identity)
if err != nil {
return err
}
@@ -139,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
// loadVerificationData downloads manifest, database, and blob list for verification
func (v *Vaultik) loadVerificationData(
snapshotID string, opts *VerifyOptions, result *VerifyResult,
identity age.Identity,
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
// Resolve the identifier to the snapshot's remote key. A human ID is
// hashed; a remote key (or its abbreviation, as printed for a
@@ -188,7 +199,7 @@ func (v *Vaultik) loadVerificationData(
defer func() { _ = dbReader.Close() }()
tdb, err := v.decryptAndLoadDatabase(dbReader)
tdb, err := v.decryptAndLoadDatabase(dbReader, identity)
if err != nil {
return nil, nil, nil, v.deepVerifyFailure(result, opts,
fmt.Sprintf("failed to decrypt database: %v", err),
@@ -230,6 +241,7 @@ func (v *Vaultik) runVerificationSteps(
opts *VerifyOptions,
result *VerifyResult,
totalSize int64,
identity age.Identity,
) error {
if !opts.JSON {
v.stdoutf("Verifying manifest against database...\n")
@@ -256,7 +268,7 @@ func (v *Vaultik) runVerificationSteps(
len(dbBlobs), ubytes(totalSize))
}
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts)
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identity)
if err != nil {
return v.deepVerifyFailure(result, opts, err.Error(), err)
}
@@ -281,26 +293,18 @@ func (t *tempDB) Close() error {
}
// decryptAndLoadDatabase decrypts and loads the binary SQLite database
// from the encrypted stream.
func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) {
// Get decryptor
decryptor, err := v.GetDecryptor()
// from the encrypted stream. It reads through the same blobgen reader restore
// uses, streaming the decrypted, decompressed database to a temp file.
func (v *Vaultik) decryptAndLoadDatabase(
reader io.ReadCloser, identity age.Identity,
) (*tempDB, error) {
// Decrypt and decompress through the shared blobgen reader.
blobReader, err := blobgen.NewReader(reader, identity)
if err != nil {
return nil, fmt.Errorf("failed to get decryptor: %w", err)
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
}
// Decrypt the stream
decryptedReader, err := decryptor.DecryptStream(reader)
if err != nil {
return nil, fmt.Errorf("failed to decrypt database: %w", err)
}
// Decompress the binary database
decompressor, err := zstd.NewReader(decryptedReader)
if err != nil {
return nil, fmt.Errorf("failed to create decompressor: %w", err)
}
defer decompressor.Close()
defer func() { _ = blobReader.Close() }()
// Materialize the decrypted database inside a private (0700) temp
// directory so it is never world-readable, and remove the whole
@@ -330,7 +334,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
// Stream decompress directly to file
log.Info("Decompressing database...")
written, err := io.Copy(tempFile, decompressor)
written, err := io.Copy(tempFile, blobReader)
if err != nil {
_ = tempFile.Close()
@@ -355,7 +359,9 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
}
// verifyBlob downloads and verifies a single blob
func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
func (v *Vaultik) verifyBlob(
blobInfo snapshot.BlobInfo, db *sql.DB, identity age.Identity,
) error {
// Download blob using shared fetch method
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
if err != nil {
@@ -364,38 +370,23 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
defer func() { _ = reader.Close() }()
// Get decryptor
decryptor, err := v.GetDecryptor()
// Decrypt and decompress through the shared blobgen reader, which hashes
// the plaintext as it is read. A blob's hash — its remote name — is the
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
// encrypted bytes.
blobReader, err := blobgen.NewReader(reader, identity)
if err != nil {
return fmt.Errorf("failed to get decryptor: %w", err)
return fmt.Errorf("failed to create blob reader: %w", err)
}
// Decrypt blob
decryptedReader, err := decryptor.DecryptStream(reader)
if err != nil {
return fmt.Errorf("failed to decrypt: %w", err)
}
defer func() { _ = blobReader.Close() }()
// Decompress blob
decompressor, err := zstd.NewReader(decryptedReader)
if err != nil {
return fmt.Errorf("failed to decompress: %w", err)
}
defer decompressor.Close()
// A blob's hash — its remote name — is the double SHA256 of its
// decompressed plaintext (see blobgen.Writer.Sum256), not of the
// encrypted bytes. Hash the plaintext as chunk verification streams
// it, then compare on completion.
plaintextHasher := sha256.New()
hashedStream := io.TeeReader(decompressor, plaintextHasher)
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream)
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, blobReader)
if err != nil {
return err
}
err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash)
err = v.verifyBlobFinalIntegrity(blobReader, blobInfo.Hash)
if err != nil {
return err
}
@@ -501,11 +492,12 @@ func (v *Vaultik) verifyBlobChunks(
// verifyBlobFinalIntegrity checks that no trailing data exists in the
// decompressed stream and that the blob hash matches the expected value.
func (v *Vaultik) verifyBlobFinalIntegrity(
plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string,
blobReader *blobgen.Reader, expectedHash string,
) error {
// Verify no remaining data in blob - if the chunk list is accurate,
// the blob should be fully consumed.
remaining, err := io.Copy(io.Discard, plaintext)
// the blob should be fully consumed. Draining to EOF also completes the
// reader's plaintext hash.
remaining, err := io.Copy(io.Discard, blobReader)
if err != nil {
return fmt.Errorf("failed to check for remaining blob data: %w", err)
}
@@ -514,10 +506,9 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
}
// The blob hash is the double SHA256 of its plaintext content.
firstHash := plaintextHasher.Sum(nil)
secondHash := sha256.Sum256(firstHash)
calculatedBlobHash := hex.EncodeToString(secondHash[:])
// The blob hash is the double SHA-256 of its plaintext content.
calculatedBlobHash := hex.EncodeToString(
blobgen.DoubleSHA256(blobReader.Sum256()))
if calculatedBlobHash != expectedHash {
return fmt.Errorf("%w: calculated %s, expected %s",
@@ -667,6 +658,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
// each blob using the database as source.
func (v *Vaultik) performDeepVerificationFromDB(
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
identity age.Identity,
) error {
// Calculate total bytes for ETA
var totalBytesExpected int64
@@ -684,7 +676,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
for i, blobInfo := range blobs {
// Verify individual blob
err := v.verifyBlob(blobInfo, db)
err := v.verifyBlob(blobInfo, db, identity)
if err != nil {
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
}
-100
View File
@@ -1,100 +0,0 @@
package vaultik_test
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"io"
"testing"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/crypto"
)
// TestTeeReaderWithDecryption tests that TeeReader correctly hashes all encrypted
// bytes when streaming through age decryption and zstd decompression.
// This validates the verification path: hash encrypted blob -> decrypt -> decompress.
func TestTeeReaderWithDecryption(t *testing.T) {
t.Parallel()
// Test data - use random data that doesn't compress well (5MB)
testData := make([]byte, 5*1024*1024)
_, err := rand.Read(testData)
require.NoError(t, err)
// Compress the data
var compressedBuf bytes.Buffer
compressor, err := zstd.NewWriter(&compressedBuf,
zstd.WithEncoderLevel(zstd.SpeedDefault))
require.NoError(t, err)
_, err = compressor.Write(testData)
require.NoError(t, err)
err = compressor.Close()
require.NoError(t, err)
// Encrypt the compressed data
testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" +
"mu62kv3s89gmvv"
testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" +
"T00J9CCW9986M3XY87PSGP89AQ"
encryptor, err := crypto.NewEncryptor([]string{testRecipient})
require.NoError(t, err)
var encryptedBuf bytes.Buffer
err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes()))
require.NoError(t, err)
encryptedData := encryptedBuf.Bytes()
// Calculate the expected hash of the encrypted data directly
expectedHash := sha256.Sum256(encryptedData)
expectedHashStr := hex.EncodeToString(expectedHash[:])
t.Logf("Encrypted data size: %d bytes", len(encryptedData))
t.Logf("Expected hash: %s", expectedHashStr)
// Now simulate what verifyBlob does: use TeeReader to hash while decrypting
decryptor, err := crypto.NewDecryptor(testSecretKey)
require.NoError(t, err)
// Create hasher and tee reader
hasher := sha256.New()
reader := bytes.NewReader(encryptedData)
teeReader := io.TeeReader(reader, hasher)
// Decrypt through the tee reader
decryptedReader, err := decryptor.DecryptStream(teeReader)
require.NoError(t, err)
// Decompress
decompressor, err := zstd.NewReader(decryptedReader)
require.NoError(t, err)
defer decompressor.Close()
// Read all decompressed data (simulating chunk verification)
decompressedData, err := io.ReadAll(decompressor)
require.NoError(t, err)
// Verify we got the original data back
assert.Equal(t, testData, decompressedData, "Decompressed data should match original")
// Drain remaining decompressed data (should be 0)
remaining, err := io.Copy(io.Discard, decompressor)
require.NoError(t, err)
assert.Equal(t, int64(0), remaining, "No remaining decompressed data")
// Calculate hash from tee reader
calculatedHashStr := hex.EncodeToString(hasher.Sum(nil))
t.Logf("Calculated hash (before drain): %s", calculatedHashStr)
// Verify the hash matches the direct hash of encrypted data
assert.Equal(t, expectedHashStr, calculatedHashStr,
"Hash calculated via TeeReader should match direct hash of encrypted data")
}