Remove the unused crypto path and write the blob-ID hash step once (closes #151)
Production encryption and decryption already run through blobgen; the crypto package (Encryptor, Decryptor, UpdateRecipients, the fx Module) and Vaultik.GetEncryptor/GetDecryptor had no production caller. Delete crypto and route verify --deep through the same blobgen reader restore uses, parsing the age key once. The second blob-ID hash step is now one exported blobgen.DoubleSHA256; Writer.Sum256 (the double hash) becomes Writer.ContentID so it no longer collides with Reader.Sum256 (the single plaintext hash). Also delete the never-adopted internal/types newtypes and the uncalled CleanupIncompleteSnapshots and its now-dead deleteSnapshot caller, and correct ARCHITECTURE.md. No production behavior changes. Model: opus-4-8
This commit was merged in pull request #191.
This commit is contained in:
+1
-2
@@ -286,7 +286,6 @@ Key methods:
|
|||||||
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
|
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
|
||||||
- `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete
|
- `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete
|
||||||
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
|
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
|
||||||
- `CleanupIncompleteSnapshots(ctx, hostname)` → Remove failed snapshots
|
|
||||||
|
|
||||||
### `internal/database`
|
### `internal/database`
|
||||||
SQLite database for local index. Single-writer mode for thread safety.
|
SQLite database for local index. Single-writer mode for thread safety.
|
||||||
@@ -307,7 +306,7 @@ Repository interfaces:
|
|||||||
```
|
```
|
||||||
CreateSnapshot(opts)
|
CreateSnapshot(opts)
|
||||||
│
|
│
|
||||||
├─► CleanupIncompleteSnapshots() // Critical: avoid dedup errors
|
├─► PruneDatabase() // Critical: avoid dedup errors
|
||||||
│
|
│
|
||||||
├─► SnapshotManager.CreateSnapshot() // Create DB record
|
├─► SnapshotManager.CreateSnapshot() // Create DB record
|
||||||
│
|
│
|
||||||
|
|||||||
@@ -487,7 +487,7 @@ func (p *Packer) closeBlobWriter() (string, int64, error) {
|
|||||||
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
finalHash := p.currentBlob.writer.Sum256()
|
finalHash := p.currentBlob.writer.ContentID()
|
||||||
|
|
||||||
return hex.EncodeToString(finalHash), finalSize, nil
|
return hex.EncodeToString(finalHash), finalSize, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,89 +0,0 @@
|
|||||||
// Package blobgen implements the blob data pipeline: streaming zstd
|
|
||||||
// compression, age encryption, and SHA256 content hashing for blob
|
|
||||||
// creation, plus the matching decrypt/decompress/verify reader.
|
|
||||||
package blobgen
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CompressResult contains the results of compression
|
|
||||||
type CompressResult struct {
|
|
||||||
Data []byte
|
|
||||||
UncompressedSize int64
|
|
||||||
CompressedSize int64
|
|
||||||
SHA256 string
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompressData compresses and encrypts data, returning the result with hash
|
|
||||||
func CompressData(
|
|
||||||
data []byte, compressionLevel int, recipients []string,
|
|
||||||
) (*CompressResult, error) {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
// Create writer
|
|
||||||
w, err := NewWriter(&buf, compressionLevel, recipients)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("creating writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write data
|
|
||||||
_, err = w.Write(data)
|
|
||||||
if err != nil {
|
|
||||||
_ = w.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("writing data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close to flush
|
|
||||||
err = w.Close()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("closing writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &CompressResult{
|
|
||||||
Data: buf.Bytes(),
|
|
||||||
UncompressedSize: int64(len(data)),
|
|
||||||
CompressedSize: int64(buf.Len()),
|
|
||||||
SHA256: hex.EncodeToString(w.Sum256()),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompressStream compresses and encrypts from reader to writer, returning
|
|
||||||
// the number of uncompressed bytes written and the content hash.
|
|
||||||
func CompressStream(
|
|
||||||
dst io.Writer, src io.Reader, compressionLevel int, recipients []string,
|
|
||||||
) (int64, string, error) {
|
|
||||||
// Create writer
|
|
||||||
w, err := NewWriter(dst, compressionLevel, recipients)
|
|
||||||
if err != nil {
|
|
||||||
return 0, "", fmt.Errorf("creating writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
closed := false
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if !closed {
|
|
||||||
_ = w.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Copy data
|
|
||||||
_, err = io.Copy(w, src)
|
|
||||||
if err != nil {
|
|
||||||
return 0, "", fmt.Errorf("copying data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close to flush
|
|
||||||
err = w.Close()
|
|
||||||
if err != nil {
|
|
||||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
closed = true
|
|
||||||
|
|
||||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
package blobgen_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/rand"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testRecipient is a static age recipient for tests.
|
|
||||||
const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
|
|
||||||
|
|
||||||
// TestCompressStreamNoDoubleClose is a regression test for issue #28.
|
|
||||||
// It verifies that CompressStream does not panic or return an error due to
|
|
||||||
// double-closing the underlying blobgen.Writer. Before the fix in PR #33,
|
|
||||||
// the explicit Close() on the happy path combined with defer Close() would
|
|
||||||
// cause a double close.
|
|
||||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
input := []byte("regression test data for issue #28 double-close fix")
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
written, hash, err := blobgen.CompressStream(
|
|
||||||
&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
|
||||||
require.NoError(t, err, "CompressStream should not return an error")
|
|
||||||
assert.Positive(t, written, "expected bytes written > 0")
|
|
||||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
|
||||||
assert.Positive(t, buf.Len(), "expected non-empty output")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
|
||||||
// to ensure no double-close issues surface under heavier I/O.
|
|
||||||
func TestCompressStreamLargeInput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
data := make([]byte, 512*1024) // 512 KB
|
|
||||||
_, err := rand.Read(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
written, hash, err := blobgen.CompressStream(
|
|
||||||
&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Positive(t, written)
|
|
||||||
assert.NotEmpty(t, hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
|
|
||||||
// without double-close issues.
|
|
||||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
_, hash, err := blobgen.CompressStream(
|
|
||||||
&buf, strings.NewReader(""), 3, []string{testRecipient})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotEmpty(t, hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
|
|
||||||
// ensuring the explicit Close + error-path Close pattern is also safe.
|
|
||||||
func TestCompressDataNoDoubleClose(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
input := []byte("CompressData regression test for double-close")
|
|
||||||
|
|
||||||
result, err := blobgen.CompressData(input, 3, []string{testRecipient})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Positive(t, result.CompressedSize)
|
|
||||||
assert.Equal(t, result.UncompressedSize, int64(len(input)))
|
|
||||||
assert.NotEmpty(t, result.SHA256)
|
|
||||||
}
|
|
||||||
@@ -64,7 +64,9 @@ func (r *Reader) Close() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sum256 returns the SHA256 hash of all data read
|
// Sum256 returns the single SHA-256 of the plaintext read so far. This is the
|
||||||
|
// first hash only; the stored object name is its double hash, which callers
|
||||||
|
// obtain by passing this digest to DoubleSHA256.
|
||||||
func (r *Reader) Sum256() []byte {
|
func (r *Reader) Sum256() []byte {
|
||||||
return r.hasher.Sum(nil)
|
return r.hasher.Sum(nil)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package blobgen_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"filippo.io/age"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestMultipleRecipients verifies that data written for several recipients can
|
||||||
|
// be read back by each recipient's identity. Moved from internal/crypto, which
|
||||||
|
// held the only multi-recipient test; blobgen is now the sole encryption path.
|
||||||
|
func TestMultipleRecipients(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
identities := make([]*age.X25519Identity, 3)
|
||||||
|
recipients := make([]string, 3)
|
||||||
|
|
||||||
|
for i := range identities {
|
||||||
|
identity, err := age.GenerateX25519Identity()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
identities[i] = identity
|
||||||
|
recipients[i] = identity.Recipient().String()
|
||||||
|
}
|
||||||
|
|
||||||
|
plaintext := []byte("Secret message for multiple recipients")
|
||||||
|
|
||||||
|
var encrypted bytes.Buffer
|
||||||
|
|
||||||
|
writer, err := blobgen.NewWriter(&encrypted, 3, recipients)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = writer.Write(plaintext)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
// Every recipient's identity must recover the original plaintext.
|
||||||
|
for i, identity := range identities {
|
||||||
|
reader, err := blobgen.NewReader(
|
||||||
|
bytes.NewReader(encrypted.Bytes()), identity)
|
||||||
|
require.NoError(t, err, "recipient %d should open the reader", i+1)
|
||||||
|
|
||||||
|
got, err := io.ReadAll(reader)
|
||||||
|
require.NoError(t, err, "recipient %d should read the plaintext", i+1)
|
||||||
|
require.NoError(t, reader.Close())
|
||||||
|
|
||||||
|
assert.Equal(t, plaintext, got,
|
||||||
|
"recipient %d should recover the original plaintext", i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
+21
-11
@@ -1,3 +1,6 @@
|
|||||||
|
// Package blobgen implements the blob data pipeline: streaming zstd
|
||||||
|
// compression, age encryption, and SHA256 content hashing for blob
|
||||||
|
// creation, plus the matching decrypt/decompress/verify reader.
|
||||||
package blobgen
|
package blobgen
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -12,6 +15,18 @@ import (
|
|||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// DoubleSHA256 returns the double SHA-256 of content whose single SHA-256
|
||||||
|
// digest is sum: it hashes that digest once more. Stored objects are named by
|
||||||
|
// this second hash so that a name never reveals whether known content is
|
||||||
|
// present — an attacker who knows a plaintext, and thus its SHA-256, still
|
||||||
|
// cannot derive the stored name without hashing the digest again. Both a blob
|
||||||
|
// and the metadata database export are named this way.
|
||||||
|
func DoubleSHA256(sum []byte) []byte {
|
||||||
|
h := sha256.Sum256(sum)
|
||||||
|
|
||||||
|
return h[:]
|
||||||
|
}
|
||||||
|
|
||||||
// Zstd compression level bounds accepted by NewWriter.
|
// Zstd compression level bounds accepted by NewWriter.
|
||||||
const (
|
const (
|
||||||
minCompressionLevel = 1
|
minCompressionLevel = 1
|
||||||
@@ -130,17 +145,12 @@ func (w *Writer) Close() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sum256 returns the double SHA256 hash of the uncompressed input data.
|
// ContentID returns the double SHA-256 of the uncompressed input data: the
|
||||||
// Double hashing (SHA256(SHA256(data))) prevents information leakage about
|
// name under which this content is stored. It is the second hash of the
|
||||||
// the plaintext - an attacker cannot confirm existence of known content
|
// running SHA-256, via DoubleSHA256; see that function for why content is
|
||||||
// by computing its hash and checking for a matching blob filename.
|
// named this way rather than by its plain SHA-256.
|
||||||
func (w *Writer) Sum256() []byte {
|
func (w *Writer) ContentID() []byte {
|
||||||
// First hash: SHA256(plaintext)
|
return DoubleSHA256(w.hasher.Sum(nil))
|
||||||
firstHash := w.hasher.Sum(nil)
|
|
||||||
// Second hash: SHA256(firstHash) - this is the blob ID
|
|
||||||
secondHash := sha256.Sum256(firstHash)
|
|
||||||
|
|
||||||
return secondHash[:]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BytesWritten returns the number of uncompressed bytes written
|
// BytesWritten returns the number of uncompressed bytes written
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns
|
// TestWriterHashIsDoubleHash verifies that Writer.ContentID() returns
|
||||||
// the double hash SHA256(SHA256(plaintext)) for security.
|
// the double hash SHA256(SHA256(plaintext)) for security.
|
||||||
// Double hashing prevents attackers from confirming existence of known content.
|
// Double hashing prevents attackers from confirming existence of known content.
|
||||||
func TestWriterHashIsDoubleHash(t *testing.T) {
|
func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||||
@@ -43,7 +43,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Get the hash from the writer
|
// Get the hash from the writer
|
||||||
writerHash := hex.EncodeToString(writer.Sum256())
|
writerHash := hex.EncodeToString(writer.ContentID())
|
||||||
|
|
||||||
// Calculate the expected double hash: SHA256(SHA256(plaintext))
|
// Calculate the expected double hash: SHA256(SHA256(plaintext))
|
||||||
firstHash := sha256.Sum256(testData)
|
firstHash := sha256.Sum256(testData)
|
||||||
@@ -60,7 +60,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
|||||||
|
|
||||||
// The writer hash should match the double hash
|
// The writer hash should match the double hash
|
||||||
assert.Equal(t, expectedDoubleHash, writerHash,
|
assert.Equal(t, expectedDoubleHash, writerHash,
|
||||||
"Writer.Sum256() should return SHA256(SHA256(plaintext)) for security")
|
"Writer.ContentID() should return SHA256(SHA256(plaintext)) for security")
|
||||||
|
|
||||||
// Verify it's NOT the single hash (would leak information)
|
// Verify it's NOT the single hash (would leak information)
|
||||||
assert.NotEqual(t, singleHashStr, writerHash,
|
assert.NotEqual(t, singleHashStr, writerHash,
|
||||||
@@ -93,8 +93,8 @@ func TestWriterDeterministicHash(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, writer2.Close())
|
require.NoError(t, writer2.Close())
|
||||||
|
|
||||||
hash1 := hex.EncodeToString(writer1.Sum256())
|
hash1 := hex.EncodeToString(writer1.ContentID())
|
||||||
hash2 := hex.EncodeToString(writer2.Sum256())
|
hash2 := hex.EncodeToString(writer2.ContentID())
|
||||||
|
|
||||||
// Hashes should be identical (deterministic)
|
// Hashes should be identical (deterministic)
|
||||||
assert.Equal(t, hash1, hash2, "Same input should produce same hash")
|
assert.Equal(t, hash1, hash2, "Same input should produce same hash")
|
||||||
|
|||||||
@@ -1,233 +0,0 @@
|
|||||||
// Package crypto provides thread-safe age encryption and decryption
|
|
||||||
// helpers used to protect blob and metadata content.
|
|
||||||
package crypto //nolint:revive,nolintlint // stdlib crypto unused; see #76
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"filippo.io/age"
|
|
||||||
"go.uber.org/fx"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrNoRecipients is returned when an encryptor is created or updated
|
|
||||||
// without any recipient public keys.
|
|
||||||
var ErrNoRecipients = errors.New("at least one recipient is required")
|
|
||||||
|
|
||||||
// errInvalidRecipient is returned when a recipient string does not parse as
|
|
||||||
// an X25519 age1... public key. It omits the value, which can be sensitive.
|
|
||||||
var errInvalidRecipient = errors.New(
|
|
||||||
"not a valid X25519 age1... recipient")
|
|
||||||
|
|
||||||
// Encryptor provides thread-safe encryption using the age encryption library.
|
|
||||||
// It supports encrypting data for multiple recipients simultaneously, allowing
|
|
||||||
// any of the corresponding private keys to decrypt the data. This is useful
|
|
||||||
// for backup scenarios where multiple parties should be able to decrypt the data.
|
|
||||||
type Encryptor struct {
|
|
||||||
recipients []age.Recipient
|
|
||||||
mu sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewEncryptor creates a new encryptor with the given age public keys.
|
|
||||||
// Each public key should be a valid age X25519 recipient string (e.g., "age1...")
|
|
||||||
// At least one recipient must be provided. Returns an error if any of the
|
|
||||||
// public keys are invalid or if no recipients are specified.
|
|
||||||
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
|
||||||
if len(publicKeys) == 0 {
|
|
||||||
return nil, ErrNoRecipients
|
|
||||||
}
|
|
||||||
|
|
||||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
|
||||||
for i, key := range publicKeys {
|
|
||||||
// The key string can be sensitive (e.g. a secret key pasted by
|
|
||||||
// mistake), so the error names its position, never its value.
|
|
||||||
recipient, err := age.ParseX25519Recipient(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
|
|
||||||
}
|
|
||||||
|
|
||||||
recipients = append(recipients, recipient)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Encryptor{
|
|
||||||
recipients: recipients,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encrypt encrypts data using age encryption for all configured recipients.
|
|
||||||
// The encrypted data can be decrypted by any of the corresponding private keys.
|
|
||||||
// This method is suitable for small to medium amounts of data that fit in memory.
|
|
||||||
// For large data streams, use EncryptStream or EncryptWriter instead.
|
|
||||||
func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
|
|
||||||
e.mu.RLock()
|
|
||||||
recipients := e.recipients
|
|
||||||
e.mu.RUnlock()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
// Create encrypted writer for all recipients
|
|
||||||
w, err := age.Encrypt(&buf, recipients...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("creating encrypted writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write data
|
|
||||||
_, err = w.Write(data)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("writing encrypted data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close to flush
|
|
||||||
err = w.Close()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("closing encrypted writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return buf.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EncryptStream encrypts data from reader to writer using age encryption.
|
|
||||||
// This method is suitable for encrypting large files or streams as it processes
|
|
||||||
// data in a streaming fashion without loading everything into memory.
|
|
||||||
// The encrypted data is written directly to the destination writer.
|
|
||||||
func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
|
|
||||||
e.mu.RLock()
|
|
||||||
recipients := e.recipients
|
|
||||||
e.mu.RUnlock()
|
|
||||||
|
|
||||||
// Create encrypted writer for all recipients
|
|
||||||
w, err := age.Encrypt(dst, recipients...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating encrypted writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Copy data
|
|
||||||
_, err = io.Copy(w, src)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("copying encrypted data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close to flush
|
|
||||||
err = w.Close()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("closing encrypted writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EncryptWriter creates a writer that encrypts data written to it.
|
|
||||||
// All data written to the returned WriteCloser will be encrypted and written
|
|
||||||
// to the destination writer. The caller must call Close() on the returned
|
|
||||||
// writer to ensure all encrypted data is properly flushed and finalized.
|
|
||||||
// This is useful for integrating encryption into existing writer-based pipelines.
|
|
||||||
func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
|
|
||||||
e.mu.RLock()
|
|
||||||
recipients := e.recipients
|
|
||||||
e.mu.RUnlock()
|
|
||||||
|
|
||||||
// Create encrypted writer for all recipients
|
|
||||||
w, err := age.Encrypt(dst, recipients...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("creating encrypted writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateRecipients updates the recipients for future encryption operations.
|
|
||||||
// This method is thread-safe and can be called while other encryption operations
|
|
||||||
// are in progress. Existing encryption operations will continue with the old
|
|
||||||
// recipients. At least one recipient must be provided. Returns an error if any
|
|
||||||
// of the public keys are invalid or if no recipients are specified.
|
|
||||||
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
|
||||||
if len(publicKeys) == 0 {
|
|
||||||
return ErrNoRecipients
|
|
||||||
}
|
|
||||||
|
|
||||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
|
||||||
for i, key := range publicKeys {
|
|
||||||
// The key string can be sensitive (e.g. a secret key pasted by
|
|
||||||
// mistake), so the error names its position, never its value.
|
|
||||||
recipient, err := age.ParseX25519Recipient(key)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
|
|
||||||
}
|
|
||||||
|
|
||||||
recipients = append(recipients, recipient)
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mu.Lock()
|
|
||||||
e.recipients = recipients
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decryptor provides thread-safe decryption using the age encryption library.
|
|
||||||
// It uses a private key to decrypt data that was encrypted for the corresponding
|
|
||||||
// public key.
|
|
||||||
type Decryptor struct {
|
|
||||||
identity age.Identity
|
|
||||||
mu sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewDecryptor creates a new decryptor with the given age private key.
|
|
||||||
// The private key should be a valid age X25519 identity string.
|
|
||||||
// Returns an error if the private key is invalid.
|
|
||||||
func NewDecryptor(privateKey string) (*Decryptor, error) {
|
|
||||||
identity, err := age.ParseX25519Identity(privateKey)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing age identity: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Decryptor{
|
|
||||||
identity: identity,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decrypt decrypts data using age decryption.
|
|
||||||
// This method is suitable for small to medium amounts of data that fit in memory.
|
|
||||||
// For large data streams, use DecryptStream instead.
|
|
||||||
func (d *Decryptor) Decrypt(data []byte) ([]byte, error) {
|
|
||||||
d.mu.RLock()
|
|
||||||
identity := d.identity
|
|
||||||
d.mu.RUnlock()
|
|
||||||
|
|
||||||
r, err := age.Decrypt(bytes.NewReader(data), identity)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("creating decrypted reader: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
decrypted, err := io.ReadAll(r)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("reading decrypted data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return decrypted, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DecryptStream returns a reader that decrypts data from the provided reader.
|
|
||||||
// This method is suitable for decrypting large files or streams as it processes
|
|
||||||
// data in a streaming fashion without loading everything into memory.
|
|
||||||
// The caller should close the input reader when done.
|
|
||||||
func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
|
|
||||||
d.mu.RLock()
|
|
||||||
identity := d.identity
|
|
||||||
d.mu.RUnlock()
|
|
||||||
|
|
||||||
r, err := age.Decrypt(src, identity)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("creating decrypted reader: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Module exports the crypto module for fx dependency injection.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // fx module definitions are package globals
|
|
||||||
var Module = fx.Module("crypto")
|
|
||||||
@@ -1,198 +0,0 @@
|
|||||||
package crypto_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"filippo.io/age"
|
|
||||||
"sneak.berlin/go/vaultik/internal/crypto"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEncryptor(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Generate a test key pair
|
|
||||||
identity, err := age.GenerateX25519Identity()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to generate identity: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
publicKey := identity.Recipient().String()
|
|
||||||
|
|
||||||
// Create encryptor
|
|
||||||
enc, err := crypto.NewEncryptor([]string{publicKey})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create encryptor: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test data
|
|
||||||
plaintext := []byte("Hello, World! This is a test message.")
|
|
||||||
|
|
||||||
// Encrypt
|
|
||||||
ciphertext, err := enc.Encrypt(plaintext)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to encrypt: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify it's actually encrypted (should be larger and different)
|
|
||||||
if bytes.Equal(plaintext, ciphertext) {
|
|
||||||
t.Error("ciphertext equals plaintext")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decrypt to verify
|
|
||||||
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to decrypt: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decrypted bytes.Buffer
|
|
||||||
|
|
||||||
_, err = decrypted.ReadFrom(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read decrypted data: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !bytes.Equal(plaintext, decrypted.Bytes()) {
|
|
||||||
t.Error("decrypted data doesn't match original")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEncryptorMultipleRecipients(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Generate three test key pairs
|
|
||||||
identity1, err := age.GenerateX25519Identity()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to generate identity1: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
identity2, err := age.GenerateX25519Identity()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to generate identity2: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
identity3, err := age.GenerateX25519Identity()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to generate identity3: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
publicKeys := []string{
|
|
||||||
identity1.Recipient().String(),
|
|
||||||
identity2.Recipient().String(),
|
|
||||||
identity3.Recipient().String(),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create encryptor with multiple recipients
|
|
||||||
enc, err := crypto.NewEncryptor(publicKeys)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create encryptor: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test data
|
|
||||||
plaintext := []byte("Secret message for multiple recipients")
|
|
||||||
|
|
||||||
// Encrypt
|
|
||||||
ciphertext, err := enc.Encrypt(plaintext)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to encrypt: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify each recipient can decrypt
|
|
||||||
identities := []age.Identity{identity1, identity2, identity3}
|
|
||||||
for i, identity := range identities {
|
|
||||||
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("recipient %d failed to decrypt: %v", i+1, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decrypted bytes.Buffer
|
|
||||||
|
|
||||||
_, err = decrypted.ReadFrom(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !bytes.Equal(plaintext, decrypted.Bytes()) {
|
|
||||||
t.Errorf("recipient %d: decrypted data doesn't match original", i+1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEncryptorUpdateRecipients(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Generate two identities
|
|
||||||
identity1, _ := age.GenerateX25519Identity()
|
|
||||||
identity2, _ := age.GenerateX25519Identity()
|
|
||||||
|
|
||||||
publicKey1 := identity1.Recipient().String()
|
|
||||||
publicKey2 := identity2.Recipient().String()
|
|
||||||
|
|
||||||
// Create encryptor with first key
|
|
||||||
enc, err := crypto.NewEncryptor([]string{publicKey1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create encryptor: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encrypt with first key
|
|
||||||
plaintext := []byte("test data")
|
|
||||||
|
|
||||||
ciphertext1, err := enc.Encrypt(plaintext)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to encrypt: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update to second key
|
|
||||||
err = enc.UpdateRecipients([]string{publicKey2})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to update recipients: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encrypt with second key
|
|
||||||
ciphertext2, err := enc.Encrypt(plaintext)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to encrypt: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// First ciphertext should only decrypt with first identity
|
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
|
|
||||||
if err != nil {
|
|
||||||
t.Error("failed to decrypt with identity1")
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
|
|
||||||
if err == nil {
|
|
||||||
t.Error("should not decrypt with identity2")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Second ciphertext should only decrypt with second identity
|
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
|
|
||||||
if err != nil {
|
|
||||||
t.Error("failed to decrypt with identity2")
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
|
|
||||||
if err == nil {
|
|
||||||
t.Error("should not decrypt with identity1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewEncryptorSecretKeyNotEchoed verifies that a secret key mistakenly
|
|
||||||
// passed as a recipient does not appear in the returned error. A recipient
|
|
||||||
// string can be sensitive, so the error must name only the position.
|
|
||||||
func TestNewEncryptorSecretKeyNotEchoed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
secretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
|
|
||||||
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
|
||||||
|
|
||||||
_, err := crypto.NewEncryptor([]string{secretKey})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("NewEncryptor returned nil, want error")
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(err.Error(), secretKey) {
|
|
||||||
t.Fatalf("error echoed the recipient value: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -295,68 +295,6 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have
|
|
||||||
// metadata in S3. This is critical for data safety: incomplete snapshots
|
|
||||||
// can cause deduplication to skip files that were never successfully
|
|
||||||
// backed up, resulting in data loss.
|
|
||||||
func (sm *SnapshotManager) CleanupIncompleteSnapshots(
|
|
||||||
ctx context.Context, hostname string,
|
|
||||||
) error {
|
|
||||||
log.Info("Checking for incomplete snapshots", "hostname", hostname)
|
|
||||||
|
|
||||||
// Get all incomplete snapshots for this hostname
|
|
||||||
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("getting incomplete snapshots: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(incompleteSnapshots) == 0 {
|
|
||||||
log.Debug("No incomplete snapshots found")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
|
|
||||||
|
|
||||||
// Check each incomplete snapshot for metadata in storage
|
|
||||||
for _, snapshot := range incompleteSnapshots {
|
|
||||||
// Check if metadata exists in storage (paths use the hashed
|
|
||||||
// remote key so we don't leak host info to the listing).
|
|
||||||
metadataKey := fmt.Sprintf("metadata/%s/db.zst",
|
|
||||||
RemoteSnapshotKey(snapshot.ID.String()))
|
|
||||||
|
|
||||||
_, err := sm.storage.Stat(ctx, metadataKey)
|
|
||||||
if err != nil {
|
|
||||||
// Metadata doesn't exist in S3 - this is an incomplete snapshot
|
|
||||||
log.Info("Cleaning up incomplete snapshot record",
|
|
||||||
"snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
|
|
||||||
|
|
||||||
// Delete the snapshot and all its associations
|
|
||||||
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting incomplete snapshot %s: %w",
|
|
||||||
snapshot.ID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("Deleted incomplete snapshot record and associated data",
|
|
||||||
"snapshot_id", snapshot.ID)
|
|
||||||
} else {
|
|
||||||
// Metadata exists - this snapshot was completed but database wasn't updated
|
|
||||||
// This shouldn't happen in normal operation, but mark it complete
|
|
||||||
log.Warn("Found snapshot with remote metadata but incomplete in database",
|
|
||||||
"snapshot_id", snapshot.ID)
|
|
||||||
|
|
||||||
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to mark snapshot as complete in database",
|
|
||||||
"snapshot_id", snapshot.ID, "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanupOrphanedData removes files, chunks, and blobs that are no longer
|
// CleanupOrphanedData removes files, chunks, and blobs that are no longer
|
||||||
// referenced by any snapshot. This should be called periodically to clean
|
// referenced by any snapshot. This should be called periodically to clean
|
||||||
// up data from deleted or incomplete snapshots.
|
// up data from deleted or incomplete snapshots.
|
||||||
@@ -759,7 +697,7 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
|||||||
|
|
||||||
writerClosed = true
|
writerClosed = true
|
||||||
|
|
||||||
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
|
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.ContentID()))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -933,45 +871,6 @@ type ExtendedBackupStats struct {
|
|||||||
UploadDurationMs int64 // Total milliseconds spent uploading to S3
|
UploadDurationMs int64 // Total milliseconds spent uploading to S3
|
||||||
}
|
}
|
||||||
|
|
||||||
// deleteSnapshot removes a snapshot and all its associations from the database
|
|
||||||
func (sm *SnapshotManager) deleteSnapshot(
|
|
||||||
ctx context.Context, snapshotID string,
|
|
||||||
) error {
|
|
||||||
// Delete snapshot_files entries
|
|
||||||
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting snapshot files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete snapshot_blobs entries
|
|
||||||
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting snapshot blobs: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete uploads entries (has foreign key to snapshots without CASCADE)
|
|
||||||
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting snapshot uploads: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete the snapshot itself
|
|
||||||
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting snapshot: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean up orphaned data
|
|
||||||
log.Debug("Cleaning up orphaned records in main database")
|
|
||||||
|
|
||||||
err = sm.CleanupOrphanedData(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cleaning up orphaned data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// deleteOtherSnapshots deletes all snapshots except the current one
|
// deleteOtherSnapshots deletes all snapshots except the current one
|
||||||
func (sm *SnapshotManager) deleteOtherSnapshots(
|
func (sm *SnapshotManager) deleteOtherSnapshots(
|
||||||
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
|
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
|
||||||
|
|||||||
+14
-59
@@ -1,7 +1,7 @@
|
|||||||
// Package types provides custom types for better type safety across the
|
// Package types provides custom types for better type safety across the
|
||||||
// vaultik codebase. Using distinct types for IDs, hashes, paths, and
|
// vaultik codebase. Using distinct types for IDs, hashes, and paths prevents
|
||||||
// credentials prevents accidental mixing of semantically different values
|
// accidental mixing of semantically different values that happen to share the
|
||||||
// that happen to share the same underlying type.
|
// same underlying type.
|
||||||
package types //nolint:revive,nolintlint // rename decision tracked in #76
|
package types //nolint:revive,nolintlint // rename decision tracked in #76
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -157,34 +157,6 @@ type FilePath string
|
|||||||
// Used during restore to strip the source prefix from paths.
|
// Used during restore to strip the source prefix from paths.
|
||||||
type SourcePath string
|
type SourcePath string
|
||||||
|
|
||||||
// AgeRecipient is an age public key used for encryption.
|
|
||||||
// Format: age1... (Bech32-encoded X25519 public key)
|
|
||||||
type AgeRecipient string
|
|
||||||
|
|
||||||
// AgeSecretKey is an age private key used for decryption.
|
|
||||||
// Format: AGE-SECRET-KEY-... (Bech32-encoded X25519 private key)
|
|
||||||
// This type should never be logged or serialized in plaintext.
|
|
||||||
type AgeSecretKey string
|
|
||||||
|
|
||||||
// S3Endpoint is the URL of an S3-compatible storage endpoint.
|
|
||||||
type S3Endpoint string
|
|
||||||
|
|
||||||
// BucketName is the name of an S3 bucket.
|
|
||||||
type BucketName string
|
|
||||||
|
|
||||||
// S3Prefix is the path prefix within an S3 bucket.
|
|
||||||
type S3Prefix string
|
|
||||||
|
|
||||||
// AWSRegion is an AWS region identifier (e.g., "us-east-1").
|
|
||||||
type AWSRegion string
|
|
||||||
|
|
||||||
// AWSAccessKeyID is an AWS access key ID for authentication.
|
|
||||||
type AWSAccessKeyID string
|
|
||||||
|
|
||||||
// AWSSecretAccessKey is an AWS secret access key for authentication.
|
|
||||||
// This type should never be logged or serialized in plaintext.
|
|
||||||
type AWSSecretAccessKey string
|
|
||||||
|
|
||||||
// Hostname identifies a host machine.
|
// Hostname identifies a host machine.
|
||||||
type Hostname string
|
type Hostname string
|
||||||
|
|
||||||
@@ -199,31 +171,14 @@ type GlobPattern string
|
|||||||
|
|
||||||
// String methods for Stringer interface
|
// String methods for Stringer interface
|
||||||
|
|
||||||
func (id FileID) String() string { return uuid.UUID(id).String() }
|
func (id FileID) String() string { return uuid.UUID(id).String() }
|
||||||
func (id BlobID) String() string { return uuid.UUID(id).String() }
|
func (id BlobID) String() string { return uuid.UUID(id).String() }
|
||||||
func (id SnapshotID) String() string { return string(id) }
|
func (id SnapshotID) String() string { return string(id) }
|
||||||
func (h ChunkHash) String() string { return string(h) }
|
func (h ChunkHash) String() string { return string(h) }
|
||||||
func (h BlobHash) String() string { return string(h) }
|
func (h BlobHash) String() string { return string(h) }
|
||||||
func (p FilePath) String() string { return string(p) }
|
func (p FilePath) String() string { return string(p) }
|
||||||
func (p SourcePath) String() string { return string(p) }
|
func (p SourcePath) String() string { return string(p) }
|
||||||
func (r AgeRecipient) String() string { return string(r) }
|
func (h Hostname) String() string { return string(h) }
|
||||||
func (e S3Endpoint) String() string { return string(e) }
|
func (v Version) String() string { return string(v) }
|
||||||
func (b BucketName) String() string { return string(b) }
|
func (r GitRevision) String() string { return string(r) }
|
||||||
func (p S3Prefix) String() string { return string(p) }
|
func (p GlobPattern) String() string { return string(p) }
|
||||||
func (r AWSRegion) String() string { return string(r) }
|
|
||||||
func (k AWSAccessKeyID) String() string { return string(k) }
|
|
||||||
func (h Hostname) String() string { return string(h) }
|
|
||||||
func (v Version) String() string { return string(v) }
|
|
||||||
func (r GitRevision) String() string { return string(r) }
|
|
||||||
func (p GlobPattern) String() string { return string(p) }
|
|
||||||
|
|
||||||
// Redacted String methods for sensitive types - prevents accidental logging
|
|
||||||
|
|
||||||
func (k AgeSecretKey) String() string { return "[REDACTED]" }
|
|
||||||
func (k AWSSecretAccessKey) String() string { return "[REDACTED]" }
|
|
||||||
|
|
||||||
// Raw returns the actual value for sensitive types when explicitly needed.
|
|
||||||
func (k AgeSecretKey) Raw() string { return string(k) }
|
|
||||||
|
|
||||||
// Raw returns the actual value for sensitive types when explicitly needed.
|
|
||||||
func (k AWSSecretAccessKey) Raw() string { return string(k) }
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package vaultik
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -57,11 +56,7 @@ func (h *hashVerifyReader) Close() error {
|
|||||||
return errBlobNotFullyRead
|
return errBlobNotFullyRead
|
||||||
}
|
}
|
||||||
|
|
||||||
firstHash := h.reader.Sum256()
|
actualHashHex := hex.EncodeToString(blobgen.DoubleSHA256(h.reader.Sum256()))
|
||||||
secondHasher := sha256.New()
|
|
||||||
secondHasher.Write(firstHash)
|
|
||||||
|
|
||||||
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
|
|
||||||
if actualHashHex != h.blobHash {
|
if actualHashHex != h.blobHash {
|
||||||
return fmt.Errorf("%w: expected %s, got %s",
|
return fmt.Errorf("%w: expected %s, got %s",
|
||||||
errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16])
|
errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16])
|
||||||
|
|||||||
@@ -40,13 +40,13 @@ func buildHashTestBlob(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Compute the double-SHA-256 hash of the plaintext (matches
|
// Compute the double-SHA-256 hash of the plaintext (matches
|
||||||
// blobgen.Writer.Sum256).
|
// blobgen.Writer.ContentID).
|
||||||
firstHash := sha256.Sum256(plaintext)
|
firstHash := sha256.Sum256(plaintext)
|
||||||
secondHash := sha256.Sum256(firstHash[:])
|
secondHash := sha256.Sum256(firstHash[:])
|
||||||
correctHash := hex.EncodeToString(secondHash[:])
|
correctHash := hex.EncodeToString(secondHash[:])
|
||||||
|
|
||||||
// Verify our hash matches what blobgen.Writer produces
|
// Verify our hash matches what blobgen.Writer produces
|
||||||
writerHash := hex.EncodeToString(writer.Sum256())
|
writerHash := hex.EncodeToString(writer.ContentID())
|
||||||
if correctHash != writerHash {
|
if correctHash != writerHash {
|
||||||
t.Fatalf("hash computation mismatch: manual=%s, writer=%s",
|
t.Fatalf("hash computation mismatch: manual=%s, writer=%s",
|
||||||
correctHash, writerHash)
|
correctHash, writerHash)
|
||||||
|
|||||||
@@ -56,8 +56,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up incomplete snapshots FIRST, before any scanning
|
// Clean up incomplete snapshots FIRST, before any scanning.
|
||||||
// This is critical for data safety - see CleanupIncompleteSnapshots for details
|
// This is critical for data safety; PruneDatabase below does it.
|
||||||
hostname := v.Config.Hostname
|
hostname := v.Config.Hostname
|
||||||
if hostname == "" {
|
if hostname == "" {
|
||||||
hostname, _ = os.Hostname()
|
hostname, _ = os.Hostname()
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ package vaultik
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
@@ -13,7 +12,6 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
"sneak.berlin/go/vaultik/internal/crypto"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/globals"
|
"sneak.berlin/go/vaultik/internal/globals"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
@@ -21,12 +19,6 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/ui"
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sentinel errors for misconfigured encryption settings.
|
|
||||||
var (
|
|
||||||
errNoAgeRecipients = errors.New("no age recipients configured")
|
|
||||||
errNoAgeSecretKey = errors.New("no age secret key configured")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Vaultik contains all dependencies needed for vaultik operations
|
// Vaultik contains all dependencies needed for vaultik operations
|
||||||
type Vaultik struct {
|
type Vaultik struct {
|
||||||
Globals *globals.Globals
|
Globals *globals.Globals
|
||||||
@@ -141,26 +133,6 @@ func (v *Vaultik) CanDecrypt() bool {
|
|||||||
return v.Config.AgeSecretKey != ""
|
return v.Config.AgeSecretKey != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetEncryptor creates a new Encryptor instance based on the configured age recipients
|
|
||||||
// Returns an error if no recipients are configured
|
|
||||||
func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) {
|
|
||||||
if len(v.Config.AgeRecipients) == 0 {
|
|
||||||
return nil, errNoAgeRecipients
|
|
||||||
}
|
|
||||||
|
|
||||||
return crypto.NewEncryptor(v.Config.AgeRecipients)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetDecryptor creates a new Decryptor instance based on the configured age secret key
|
|
||||||
// Returns an error if no secret key is configured
|
|
||||||
func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) {
|
|
||||||
if v.Config.AgeSecretKey == "" {
|
|
||||||
return nil, errNoAgeSecretKey
|
|
||||||
}
|
|
||||||
|
|
||||||
return crypto.NewDecryptor(v.Config.AgeSecretKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetFilesystem returns the filesystem instance used by Vaultik
|
// GetFilesystem returns the filesystem instance used by Vaultik
|
||||||
//
|
//
|
||||||
//nolint:ireturn // afero.Fs is the filesystem abstraction by design
|
//nolint:ireturn // afero.Fs is the filesystem abstraction by design
|
||||||
|
|||||||
+49
-57
@@ -6,14 +6,14 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"hash"
|
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/klauspost/compress/zstd"
|
"filippo.io/age"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
@@ -88,13 +88,22 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
errSecretKeyRequired.Error(), errSecretKeyRequired)
|
errSecretKeyRequired.Error(), errSecretKeyRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Parse the age secret key once, the same way restore does, and reuse
|
||||||
|
// the identity for the database and every blob.
|
||||||
|
identity, err := v.prepareRestoreIdentity()
|
||||||
|
if err != nil {
|
||||||
|
return v.deepVerifyFailure(result, opts,
|
||||||
|
fmt.Sprintf("parsing age secret key: %v", err), err)
|
||||||
|
}
|
||||||
|
|
||||||
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
||||||
|
|
||||||
if !opts.JSON {
|
if !opts.JSON {
|
||||||
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
|
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result)
|
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
|
||||||
|
snapshotID, opts, result, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -114,7 +123,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
|
|
||||||
result.TotalSize = totalSize
|
result.TotalSize = totalSize
|
||||||
|
|
||||||
err = v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize)
|
err = v.runVerificationSteps(
|
||||||
|
manifest, dbBlobs, tempDB, opts, result, totalSize, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -139,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
// loadVerificationData downloads manifest, database, and blob list for verification
|
// loadVerificationData downloads manifest, database, and blob list for verification
|
||||||
func (v *Vaultik) loadVerificationData(
|
func (v *Vaultik) loadVerificationData(
|
||||||
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
||||||
|
identity age.Identity,
|
||||||
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
||||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||||
// hashed; a remote key (or its abbreviation, as printed for a
|
// hashed; a remote key (or its abbreviation, as printed for a
|
||||||
@@ -188,7 +199,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
|
|
||||||
defer func() { _ = dbReader.Close() }()
|
defer func() { _ = dbReader.Close() }()
|
||||||
|
|
||||||
tdb, err := v.decryptAndLoadDatabase(dbReader)
|
tdb, err := v.decryptAndLoadDatabase(dbReader, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
||||||
fmt.Sprintf("failed to decrypt database: %v", err),
|
fmt.Sprintf("failed to decrypt database: %v", err),
|
||||||
@@ -230,6 +241,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
opts *VerifyOptions,
|
opts *VerifyOptions,
|
||||||
result *VerifyResult,
|
result *VerifyResult,
|
||||||
totalSize int64,
|
totalSize int64,
|
||||||
|
identity age.Identity,
|
||||||
) error {
|
) error {
|
||||||
if !opts.JSON {
|
if !opts.JSON {
|
||||||
v.stdoutf("Verifying manifest against database...\n")
|
v.stdoutf("Verifying manifest against database...\n")
|
||||||
@@ -256,7 +268,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
len(dbBlobs), ubytes(totalSize))
|
len(dbBlobs), ubytes(totalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts)
|
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||||
}
|
}
|
||||||
@@ -281,26 +293,18 @@ func (t *tempDB) Close() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// decryptAndLoadDatabase decrypts and loads the binary SQLite database
|
// decryptAndLoadDatabase decrypts and loads the binary SQLite database
|
||||||
// from the encrypted stream.
|
// from the encrypted stream. It reads through the same blobgen reader restore
|
||||||
func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) {
|
// uses, streaming the decrypted, decompressed database to a temp file.
|
||||||
// Get decryptor
|
func (v *Vaultik) decryptAndLoadDatabase(
|
||||||
decryptor, err := v.GetDecryptor()
|
reader io.ReadCloser, identity age.Identity,
|
||||||
|
) (*tempDB, error) {
|
||||||
|
// Decrypt and decompress through the shared blobgen reader.
|
||||||
|
blobReader, err := blobgen.NewReader(reader, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to get decryptor: %w", err)
|
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decrypt the stream
|
defer func() { _ = blobReader.Close() }()
|
||||||
decryptedReader, err := decryptor.DecryptStream(reader)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to decrypt database: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decompress the binary database
|
|
||||||
decompressor, err := zstd.NewReader(decryptedReader)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create decompressor: %w", err)
|
|
||||||
}
|
|
||||||
defer decompressor.Close()
|
|
||||||
|
|
||||||
// Materialize the decrypted database inside a private (0700) temp
|
// Materialize the decrypted database inside a private (0700) temp
|
||||||
// directory so it is never world-readable, and remove the whole
|
// directory so it is never world-readable, and remove the whole
|
||||||
@@ -330,7 +334,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
|
|||||||
// Stream decompress directly to file
|
// Stream decompress directly to file
|
||||||
log.Info("Decompressing database...")
|
log.Info("Decompressing database...")
|
||||||
|
|
||||||
written, err := io.Copy(tempFile, decompressor)
|
written, err := io.Copy(tempFile, blobReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tempFile.Close()
|
_ = tempFile.Close()
|
||||||
|
|
||||||
@@ -355,7 +359,9 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// verifyBlob downloads and verifies a single blob
|
// verifyBlob downloads and verifies a single blob
|
||||||
func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
func (v *Vaultik) verifyBlob(
|
||||||
|
blobInfo snapshot.BlobInfo, db *sql.DB, identity age.Identity,
|
||||||
|
) error {
|
||||||
// Download blob using shared fetch method
|
// Download blob using shared fetch method
|
||||||
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
|
reader, _, err := v.FetchBlob(v.ctx, blobInfo.Hash, blobInfo.CompressedSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -364,38 +370,23 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
|||||||
|
|
||||||
defer func() { _ = reader.Close() }()
|
defer func() { _ = reader.Close() }()
|
||||||
|
|
||||||
// Get decryptor
|
// Decrypt and decompress through the shared blobgen reader, which hashes
|
||||||
decryptor, err := v.GetDecryptor()
|
// the plaintext as it is read. A blob's hash — its remote name — is the
|
||||||
|
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
||||||
|
// encrypted bytes.
|
||||||
|
blobReader, err := blobgen.NewReader(reader, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get decryptor: %w", err)
|
return fmt.Errorf("failed to create blob reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decrypt blob
|
defer func() { _ = blobReader.Close() }()
|
||||||
decryptedReader, err := decryptor.DecryptStream(reader)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to decrypt: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decompress blob
|
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, blobReader)
|
||||||
decompressor, err := zstd.NewReader(decryptedReader)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to decompress: %w", err)
|
|
||||||
}
|
|
||||||
defer decompressor.Close()
|
|
||||||
|
|
||||||
// A blob's hash — its remote name — is the double SHA256 of its
|
|
||||||
// decompressed plaintext (see blobgen.Writer.Sum256), not of the
|
|
||||||
// encrypted bytes. Hash the plaintext as chunk verification streams
|
|
||||||
// it, then compare on completion.
|
|
||||||
plaintextHasher := sha256.New()
|
|
||||||
hashedStream := io.TeeReader(decompressor, plaintextHasher)
|
|
||||||
|
|
||||||
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash)
|
err = v.verifyBlobFinalIntegrity(blobReader, blobInfo.Hash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -501,11 +492,12 @@ func (v *Vaultik) verifyBlobChunks(
|
|||||||
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
||||||
// decompressed stream and that the blob hash matches the expected value.
|
// decompressed stream and that the blob hash matches the expected value.
|
||||||
func (v *Vaultik) verifyBlobFinalIntegrity(
|
func (v *Vaultik) verifyBlobFinalIntegrity(
|
||||||
plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string,
|
blobReader *blobgen.Reader, expectedHash string,
|
||||||
) error {
|
) error {
|
||||||
// Verify no remaining data in blob - if the chunk list is accurate,
|
// Verify no remaining data in blob - if the chunk list is accurate,
|
||||||
// the blob should be fully consumed.
|
// the blob should be fully consumed. Draining to EOF also completes the
|
||||||
remaining, err := io.Copy(io.Discard, plaintext)
|
// reader's plaintext hash.
|
||||||
|
remaining, err := io.Copy(io.Discard, blobReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
||||||
}
|
}
|
||||||
@@ -514,10 +506,9 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
|
|||||||
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The blob hash is the double SHA256 of its plaintext content.
|
// The blob hash is the double SHA-256 of its plaintext content.
|
||||||
firstHash := plaintextHasher.Sum(nil)
|
calculatedBlobHash := hex.EncodeToString(
|
||||||
secondHash := sha256.Sum256(firstHash)
|
blobgen.DoubleSHA256(blobReader.Sum256()))
|
||||||
calculatedBlobHash := hex.EncodeToString(secondHash[:])
|
|
||||||
|
|
||||||
if calculatedBlobHash != expectedHash {
|
if calculatedBlobHash != expectedHash {
|
||||||
return fmt.Errorf("%w: calculated %s, expected %s",
|
return fmt.Errorf("%w: calculated %s, expected %s",
|
||||||
@@ -667,6 +658,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
|
|||||||
// each blob using the database as source.
|
// each blob using the database as source.
|
||||||
func (v *Vaultik) performDeepVerificationFromDB(
|
func (v *Vaultik) performDeepVerificationFromDB(
|
||||||
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
|
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
|
||||||
|
identity age.Identity,
|
||||||
) error {
|
) error {
|
||||||
// Calculate total bytes for ETA
|
// Calculate total bytes for ETA
|
||||||
var totalBytesExpected int64
|
var totalBytesExpected int64
|
||||||
@@ -684,7 +676,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
|
|||||||
|
|
||||||
for i, blobInfo := range blobs {
|
for i, blobInfo := range blobs {
|
||||||
// Verify individual blob
|
// Verify individual blob
|
||||||
err := v.verifyBlob(blobInfo, db)
|
err := v.verifyBlob(blobInfo, db, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
|
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"io"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/klauspost/compress/zstd"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/crypto"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestTeeReaderWithDecryption tests that TeeReader correctly hashes all encrypted
|
|
||||||
// bytes when streaming through age decryption and zstd decompression.
|
|
||||||
// This validates the verification path: hash encrypted blob -> decrypt -> decompress.
|
|
||||||
func TestTeeReaderWithDecryption(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Test data - use random data that doesn't compress well (5MB)
|
|
||||||
testData := make([]byte, 5*1024*1024)
|
|
||||||
_, err := rand.Read(testData)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Compress the data
|
|
||||||
var compressedBuf bytes.Buffer
|
|
||||||
|
|
||||||
compressor, err := zstd.NewWriter(&compressedBuf,
|
|
||||||
zstd.WithEncoderLevel(zstd.SpeedDefault))
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = compressor.Write(testData)
|
|
||||||
require.NoError(t, err)
|
|
||||||
err = compressor.Close()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Encrypt the compressed data
|
|
||||||
testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" +
|
|
||||||
"mu62kv3s89gmvv"
|
|
||||||
testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" +
|
|
||||||
"T00J9CCW9986M3XY87PSGP89AQ"
|
|
||||||
|
|
||||||
encryptor, err := crypto.NewEncryptor([]string{testRecipient})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var encryptedBuf bytes.Buffer
|
|
||||||
|
|
||||||
err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes()))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
encryptedData := encryptedBuf.Bytes()
|
|
||||||
|
|
||||||
// Calculate the expected hash of the encrypted data directly
|
|
||||||
expectedHash := sha256.Sum256(encryptedData)
|
|
||||||
expectedHashStr := hex.EncodeToString(expectedHash[:])
|
|
||||||
|
|
||||||
t.Logf("Encrypted data size: %d bytes", len(encryptedData))
|
|
||||||
t.Logf("Expected hash: %s", expectedHashStr)
|
|
||||||
|
|
||||||
// Now simulate what verifyBlob does: use TeeReader to hash while decrypting
|
|
||||||
decryptor, err := crypto.NewDecryptor(testSecretKey)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Create hasher and tee reader
|
|
||||||
hasher := sha256.New()
|
|
||||||
reader := bytes.NewReader(encryptedData)
|
|
||||||
teeReader := io.TeeReader(reader, hasher)
|
|
||||||
|
|
||||||
// Decrypt through the tee reader
|
|
||||||
decryptedReader, err := decryptor.DecryptStream(teeReader)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Decompress
|
|
||||||
decompressor, err := zstd.NewReader(decryptedReader)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer decompressor.Close()
|
|
||||||
|
|
||||||
// Read all decompressed data (simulating chunk verification)
|
|
||||||
decompressedData, err := io.ReadAll(decompressor)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Verify we got the original data back
|
|
||||||
assert.Equal(t, testData, decompressedData, "Decompressed data should match original")
|
|
||||||
|
|
||||||
// Drain remaining decompressed data (should be 0)
|
|
||||||
remaining, err := io.Copy(io.Discard, decompressor)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, int64(0), remaining, "No remaining decompressed data")
|
|
||||||
|
|
||||||
// Calculate hash from tee reader
|
|
||||||
calculatedHashStr := hex.EncodeToString(hasher.Sum(nil))
|
|
||||||
t.Logf("Calculated hash (before drain): %s", calculatedHashStr)
|
|
||||||
|
|
||||||
// Verify the hash matches the direct hash of encrypted data
|
|
||||||
assert.Equal(t, expectedHashStr, calculatedHashStr,
|
|
||||||
"Hash calculated via TeeReader should match direct hash of encrypted data")
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user