sneak
bb38f8c5d6
- Add afero.Fs field to Vaultik struct for filesystem operations - Vaultik now owns and manages the filesystem instance - SnapshotManager receives filesystem via SetFilesystem() setter - Update blob packer to use afero for temporary files - Convert all filesystem operations to use afero abstraction - Remove filesystem module - Vaultik manages filesystem directly - Update tests: remove symlink test (unsupported by afero memfs) - Fix TestMultipleFileChanges to handle scanner examining directories This enables full end-to-end testing without touching disk by using memory-backed filesystems. Database operations continue using real filesystem as SQLite requires actual files.
385 lines
9.7 KiB
Go
385 lines
9.7 KiB
Go
package blob
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"io"
|
|
"testing"
|
|
|
|
"filippo.io/age"
|
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/spf13/afero"
|
|
)
|
|
|
|
const (
|
|
// Test key from test/insecure-integration-test.key
|
|
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
|
testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
|
)
|
|
|
|
func TestPacker(t *testing.T) {
|
|
// Initialize logger for tests
|
|
log.Initialize(log.Config{})
|
|
|
|
// Parse test identity
|
|
identity, err := age.ParseX25519Identity(testPrivateKey)
|
|
if err != nil {
|
|
t.Fatalf("failed to parse test identity: %v", err)
|
|
}
|
|
|
|
t.Run("single chunk creates single blob", func(t *testing.T) {
|
|
// Create test database
|
|
db, err := database.NewTestDB()
|
|
if err != nil {
|
|
t.Fatalf("failed to create test db: %v", err)
|
|
}
|
|
defer func() { _ = db.Close() }()
|
|
repos := database.NewRepositories(db)
|
|
|
|
cfg := PackerConfig{
|
|
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
|
CompressionLevel: 3,
|
|
Recipients: []string{testPublicKey},
|
|
Repositories: repos,
|
|
Fs: afero.NewMemMapFs(),
|
|
}
|
|
packer, err := NewPacker(cfg)
|
|
if err != nil {
|
|
t.Fatalf("failed to create packer: %v", err)
|
|
}
|
|
|
|
// Create a chunk
|
|
data := []byte("Hello, World!")
|
|
hash := sha256.Sum256(data)
|
|
hashStr := hex.EncodeToString(hash[:])
|
|
|
|
// Create chunk in database first
|
|
dbChunk := &database.Chunk{
|
|
ChunkHash: hashStr,
|
|
Size: int64(len(data)),
|
|
}
|
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create chunk in db: %v", err)
|
|
}
|
|
|
|
chunk := &ChunkRef{
|
|
Hash: hashStr,
|
|
Data: data,
|
|
}
|
|
|
|
// Add chunk
|
|
if err := packer.AddChunk(chunk); err != nil {
|
|
t.Fatalf("failed to add chunk: %v", err)
|
|
}
|
|
|
|
// Flush
|
|
if err := packer.Flush(); err != nil {
|
|
t.Fatalf("failed to flush: %v", err)
|
|
}
|
|
|
|
// Get finished blobs
|
|
blobs := packer.GetFinishedBlobs()
|
|
if len(blobs) != 1 {
|
|
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
|
}
|
|
|
|
blob := blobs[0]
|
|
if len(blob.Chunks) != 1 {
|
|
t.Errorf("expected 1 chunk in blob, got %d", len(blob.Chunks))
|
|
}
|
|
|
|
// Note: Very small data may not compress well
|
|
t.Logf("Compression: %d -> %d bytes", blob.Uncompressed, blob.Compressed)
|
|
|
|
// Decrypt the blob data
|
|
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
|
|
if err != nil {
|
|
t.Fatalf("failed to decrypt blob: %v", err)
|
|
}
|
|
|
|
// Decompress the decrypted data
|
|
reader, err := zstd.NewReader(decrypted)
|
|
if err != nil {
|
|
t.Fatalf("failed to create decompressor: %v", err)
|
|
}
|
|
defer reader.Close()
|
|
|
|
var decompressed bytes.Buffer
|
|
if _, err := io.Copy(&decompressed, reader); err != nil {
|
|
t.Fatalf("failed to decompress: %v", err)
|
|
}
|
|
|
|
if !bytes.Equal(decompressed.Bytes(), data) {
|
|
t.Error("decompressed data doesn't match original")
|
|
}
|
|
})
|
|
|
|
t.Run("multiple chunks packed together", func(t *testing.T) {
|
|
// Create test database
|
|
db, err := database.NewTestDB()
|
|
if err != nil {
|
|
t.Fatalf("failed to create test db: %v", err)
|
|
}
|
|
defer func() { _ = db.Close() }()
|
|
repos := database.NewRepositories(db)
|
|
|
|
cfg := PackerConfig{
|
|
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
|
CompressionLevel: 3,
|
|
Recipients: []string{testPublicKey},
|
|
Repositories: repos,
|
|
Fs: afero.NewMemMapFs(),
|
|
}
|
|
packer, err := NewPacker(cfg)
|
|
if err != nil {
|
|
t.Fatalf("failed to create packer: %v", err)
|
|
}
|
|
|
|
// Create multiple small chunks
|
|
chunks := make([]*ChunkRef, 10)
|
|
for i := 0; i < 10; i++ {
|
|
data := bytes.Repeat([]byte{byte(i)}, 1000)
|
|
hash := sha256.Sum256(data)
|
|
hashStr := hex.EncodeToString(hash[:])
|
|
|
|
// Create chunk in database first
|
|
dbChunk := &database.Chunk{
|
|
ChunkHash: hashStr,
|
|
Size: int64(len(data)),
|
|
}
|
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create chunk in db: %v", err)
|
|
}
|
|
|
|
chunks[i] = &ChunkRef{
|
|
Hash: hashStr,
|
|
Data: data,
|
|
}
|
|
}
|
|
|
|
// Add all chunks
|
|
for _, chunk := range chunks {
|
|
err := packer.AddChunk(chunk)
|
|
if err != nil {
|
|
t.Fatalf("failed to add chunk: %v", err)
|
|
}
|
|
}
|
|
|
|
// Flush
|
|
if err := packer.Flush(); err != nil {
|
|
t.Fatalf("failed to flush: %v", err)
|
|
}
|
|
|
|
// Should have one blob with all chunks
|
|
blobs := packer.GetFinishedBlobs()
|
|
if len(blobs) != 1 {
|
|
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
|
}
|
|
|
|
if len(blobs[0].Chunks) != 10 {
|
|
t.Errorf("expected 10 chunks in blob, got %d", len(blobs[0].Chunks))
|
|
}
|
|
|
|
// Verify offsets are correct
|
|
expectedOffset := int64(0)
|
|
for i, chunkRef := range blobs[0].Chunks {
|
|
if chunkRef.Offset != expectedOffset {
|
|
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
|
|
}
|
|
if chunkRef.Length != 1000 {
|
|
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
|
|
}
|
|
expectedOffset += chunkRef.Length
|
|
}
|
|
})
|
|
|
|
t.Run("blob size limit enforced", func(t *testing.T) {
|
|
// Create test database
|
|
db, err := database.NewTestDB()
|
|
if err != nil {
|
|
t.Fatalf("failed to create test db: %v", err)
|
|
}
|
|
defer func() { _ = db.Close() }()
|
|
repos := database.NewRepositories(db)
|
|
|
|
// Small blob size limit to force multiple blobs
|
|
cfg := PackerConfig{
|
|
MaxBlobSize: 5000, // 5KB max
|
|
CompressionLevel: 3,
|
|
Recipients: []string{testPublicKey},
|
|
Repositories: repos,
|
|
Fs: afero.NewMemMapFs(),
|
|
}
|
|
packer, err := NewPacker(cfg)
|
|
if err != nil {
|
|
t.Fatalf("failed to create packer: %v", err)
|
|
}
|
|
|
|
// Create chunks that will exceed the limit
|
|
chunks := make([]*ChunkRef, 10)
|
|
for i := 0; i < 10; i++ {
|
|
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
|
|
hash := sha256.Sum256(data)
|
|
hashStr := hex.EncodeToString(hash[:])
|
|
|
|
// Create chunk in database first
|
|
dbChunk := &database.Chunk{
|
|
ChunkHash: hashStr,
|
|
Size: int64(len(data)),
|
|
}
|
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create chunk in db: %v", err)
|
|
}
|
|
|
|
chunks[i] = &ChunkRef{
|
|
Hash: hashStr,
|
|
Data: data,
|
|
}
|
|
}
|
|
|
|
blobCount := 0
|
|
|
|
// Add chunks and handle size limit errors
|
|
for _, chunk := range chunks {
|
|
err := packer.AddChunk(chunk)
|
|
if err == ErrBlobSizeLimitExceeded {
|
|
// Finalize current blob
|
|
if err := packer.FinalizeBlob(); err != nil {
|
|
t.Fatalf("failed to finalize blob: %v", err)
|
|
}
|
|
blobCount++
|
|
// Retry adding the chunk
|
|
if err := packer.AddChunk(chunk); err != nil {
|
|
t.Fatalf("failed to add chunk after finalize: %v", err)
|
|
}
|
|
} else if err != nil {
|
|
t.Fatalf("failed to add chunk: %v", err)
|
|
}
|
|
}
|
|
|
|
// Flush remaining
|
|
if err := packer.Flush(); err != nil {
|
|
t.Fatalf("failed to flush: %v", err)
|
|
}
|
|
|
|
// Get all blobs
|
|
blobs := packer.GetFinishedBlobs()
|
|
totalBlobs := blobCount + len(blobs)
|
|
|
|
// Should have multiple blobs due to size limit
|
|
if totalBlobs < 2 {
|
|
t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs)
|
|
}
|
|
|
|
// Verify each blob respects size limit (approximately)
|
|
for _, blob := range blobs {
|
|
if blob.Compressed > 6000 { // Allow some overhead
|
|
t.Errorf("blob size %d exceeds limit", blob.Compressed)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("with encryption", func(t *testing.T) {
|
|
// Create test database
|
|
db, err := database.NewTestDB()
|
|
if err != nil {
|
|
t.Fatalf("failed to create test db: %v", err)
|
|
}
|
|
defer func() { _ = db.Close() }()
|
|
repos := database.NewRepositories(db)
|
|
|
|
// Generate test identity (using the one from parent test)
|
|
cfg := PackerConfig{
|
|
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
|
CompressionLevel: 3,
|
|
Recipients: []string{testPublicKey},
|
|
Repositories: repos,
|
|
Fs: afero.NewMemMapFs(),
|
|
}
|
|
packer, err := NewPacker(cfg)
|
|
if err != nil {
|
|
t.Fatalf("failed to create packer: %v", err)
|
|
}
|
|
|
|
// Create test data
|
|
data := bytes.Repeat([]byte("Test data for encryption!"), 100)
|
|
hash := sha256.Sum256(data)
|
|
hashStr := hex.EncodeToString(hash[:])
|
|
|
|
// Create chunk in database first
|
|
dbChunk := &database.Chunk{
|
|
ChunkHash: hashStr,
|
|
Size: int64(len(data)),
|
|
}
|
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create chunk in db: %v", err)
|
|
}
|
|
|
|
chunk := &ChunkRef{
|
|
Hash: hashStr,
|
|
Data: data,
|
|
}
|
|
|
|
// Add chunk and flush
|
|
if err := packer.AddChunk(chunk); err != nil {
|
|
t.Fatalf("failed to add chunk: %v", err)
|
|
}
|
|
if err := packer.Flush(); err != nil {
|
|
t.Fatalf("failed to flush: %v", err)
|
|
}
|
|
|
|
// Get blob
|
|
blobs := packer.GetFinishedBlobs()
|
|
if len(blobs) != 1 {
|
|
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
|
}
|
|
|
|
blob := blobs[0]
|
|
|
|
// Decrypt the blob
|
|
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
|
|
if err != nil {
|
|
t.Fatalf("failed to decrypt blob: %v", err)
|
|
}
|
|
|
|
var decryptedData bytes.Buffer
|
|
if _, err := decryptedData.ReadFrom(decrypted); err != nil {
|
|
t.Fatalf("failed to read decrypted data: %v", err)
|
|
}
|
|
|
|
// Decompress
|
|
reader, err := zstd.NewReader(&decryptedData)
|
|
if err != nil {
|
|
t.Fatalf("failed to create decompressor: %v", err)
|
|
}
|
|
defer reader.Close()
|
|
|
|
var decompressed bytes.Buffer
|
|
if _, err := decompressed.ReadFrom(reader); err != nil {
|
|
t.Fatalf("failed to decompress: %v", err)
|
|
}
|
|
|
|
// Verify data
|
|
if !bytes.Equal(decompressed.Bytes(), data) {
|
|
t.Error("decrypted and decompressed data doesn't match original")
|
|
}
|
|
})
|
|
}
|