clawbot
e496aa334b
All checks were successful
check / check (push) Successful in 5s
Clears the final 80 golangci-lint findings under the canonical .golangci.yml (sha256 021cc83f4e6fc7c31b95b34b846723dfcf20b66b7baeea1dc40406e643346bcb), taking the repo from red to green: script/cibuild exits 0. - wsl_v5 (60): blank line above defer/go statements sharing no variable with the line above; blank-line-only diff. - sqlclosecheck (10): the package-local CloseRows helper hid the close from the analyzer. Helper removed; all 18 call sites now defer an inline rows.Close(), preserving the fatal-on-close-error path. No resource leak existed - the rows were always being closed. - prealloc (3): append targets given a starting capacity. - revive (3): package-name findings suppressed with per-site directives pending the naming decision tracked in #76. No gosec suppressions are needed under the pinned linter. .golangci.yml, Dockerfile, Makefile, .gitea/ and script/ are byte-identical to main. Verified with script/cibuild (digest-pinned golangci-lint v2.12.2), not make check - the latter resolves the linter from PATH and is not a trustworthy gate here; see #78. Closes #59.
1038 lines
30 KiB
Go
1038 lines
30 KiB
Go
package vaultik_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/database"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/types"
|
|
"sneak.berlin/go/vaultik/internal/ui"
|
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
)
|
|
|
|
// Shared fixture values used across the vaultik integration tests.
|
|
const (
|
|
testLabel = "test"
|
|
testHostname = "test-host"
|
|
testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05g" +
|
|
"l0sjq9q9wjg"
|
|
testAgeSecretKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKU" +
|
|
"T68TXSFPK7APHXA2QS2NJA5"
|
|
)
|
|
|
|
// MockStorer implements storage.Storer for testing
|
|
type MockStorer struct {
|
|
mu sync.Mutex
|
|
data map[string][]byte
|
|
calls []string
|
|
}
|
|
|
|
func NewMockStorer() *MockStorer {
|
|
return &MockStorer{
|
|
data: make(map[string][]byte),
|
|
calls: make([]string, 0),
|
|
}
|
|
}
|
|
|
|
func (m *MockStorer) Put(_ context.Context, key string, reader io.Reader) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
m.calls = append(m.calls, "Put:"+key)
|
|
|
|
data, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
m.data[key] = data
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *MockStorer) PutWithProgress(
|
|
ctx context.Context, key string, reader io.Reader,
|
|
_ int64, _ storage.ProgressCallback,
|
|
) error {
|
|
return m.Put(ctx, key, reader)
|
|
}
|
|
|
|
func (m *MockStorer) Get(_ context.Context, key string) (io.ReadCloser, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
m.calls = append(m.calls, "Get:"+key)
|
|
|
|
data, exists := m.data[key]
|
|
if !exists {
|
|
return nil, storage.ErrNotFound
|
|
}
|
|
|
|
return io.NopCloser(bytes.NewReader(data)), nil
|
|
}
|
|
|
|
func (m *MockStorer) Stat(_ context.Context, key string) (*storage.ObjectInfo, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
m.calls = append(m.calls, "Stat:"+key)
|
|
|
|
data, exists := m.data[key]
|
|
if !exists {
|
|
return nil, storage.ErrNotFound
|
|
}
|
|
|
|
return &storage.ObjectInfo{
|
|
Key: key,
|
|
Size: int64(len(data)),
|
|
}, nil
|
|
}
|
|
|
|
func (m *MockStorer) Delete(_ context.Context, key string) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
m.calls = append(m.calls, "Delete:"+key)
|
|
delete(m.data, key)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *MockStorer) List(_ context.Context, prefix string) ([]string, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
m.calls = append(m.calls, "List:"+prefix)
|
|
|
|
var keys []string
|
|
|
|
for key := range m.data {
|
|
if len(prefix) == 0 ||
|
|
(len(key) >= len(prefix) && key[:len(prefix)] == prefix) {
|
|
keys = append(keys, key)
|
|
}
|
|
}
|
|
|
|
return keys, nil
|
|
}
|
|
|
|
func (m *MockStorer) ListStream(
|
|
_ context.Context, prefix string,
|
|
) <-chan storage.ObjectInfo {
|
|
ch := make(chan storage.ObjectInfo)
|
|
|
|
go func() {
|
|
defer close(ch)
|
|
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
for key, data := range m.data {
|
|
if len(prefix) == 0 ||
|
|
(len(key) >= len(prefix) && key[:len(prefix)] == prefix) {
|
|
ch <- storage.ObjectInfo{
|
|
Key: key,
|
|
Size: int64(len(data)),
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
return ch
|
|
}
|
|
|
|
func (m *MockStorer) Info() storage.Info {
|
|
return storage.Info{
|
|
Type: "mock",
|
|
Location: "memory",
|
|
}
|
|
}
|
|
|
|
// GetCalls returns the list of operations that were called
|
|
func (m *MockStorer) GetCalls() []string {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
calls := make([]string, len(m.calls))
|
|
copy(calls, m.calls)
|
|
|
|
return calls
|
|
}
|
|
|
|
// GetStorageSize returns the number of objects in storage
|
|
func (m *MockStorer) GetStorageSize() int {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
return len(m.data)
|
|
}
|
|
|
|
// writeTestFileTree creates each file (and its parent directory) in fs.
|
|
func writeTestFileTree(t *testing.T, fs afero.Fs, files map[string]string) {
|
|
t.Helper()
|
|
|
|
for path, content := range files {
|
|
dir := filepath.Dir(path)
|
|
|
|
err := fs.MkdirAll(dir, 0755)
|
|
if err != nil {
|
|
t.Fatalf("failed to create directory %s: %v", dir, err)
|
|
}
|
|
|
|
err = afero.WriteFile(fs, path, []byte(content), 0644)
|
|
if err != nil {
|
|
t.Fatalf("failed to create test file %s: %v", path, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// createTestSnapshotRecord inserts a snapshot row so scans have a
|
|
// snapshot to attach to.
|
|
func createTestSnapshotRecord(
|
|
t *testing.T, repos *database.Repositories, snapshotID string,
|
|
) {
|
|
t.Helper()
|
|
|
|
ctx := context.Background()
|
|
|
|
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
|
snap := &database.Snapshot{
|
|
ID: types.SnapshotID(snapshotID),
|
|
Hostname: testHostname,
|
|
VaultikVersion: "test-version",
|
|
StartedAt: time.Now(),
|
|
}
|
|
|
|
return repos.Snapshots.Create(ctx, tx, snap)
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// TestEndToEndBackup tests the full backup workflow with mocked dependencies
|
|
// verifyEndToEndBackupState checks storage upload calls, database file
|
|
// rows, and chunk mappings after the end-to-end backup scan.
|
|
func verifyEndToEndBackupState(
|
|
ctx context.Context, t *testing.T,
|
|
repos *database.Repositories, mockStorage *MockStorer,
|
|
) {
|
|
t.Helper()
|
|
|
|
// Verify storage operations
|
|
calls := mockStorage.GetCalls()
|
|
t.Logf("Storage operations performed: %v", calls)
|
|
|
|
// Should have uploaded at least one blob
|
|
blobUploads := 0
|
|
|
|
for _, call := range calls {
|
|
if len(call) > 4 && call[:4] == "Put:" {
|
|
if len(call) > 10 && call[4:10] == "blobs/" {
|
|
blobUploads++
|
|
}
|
|
}
|
|
}
|
|
|
|
assert.Positive(t, blobUploads, "Should upload at least one blob")
|
|
|
|
// Verify files in database
|
|
files, err := repos.Files.ListByPrefix(ctx, "/home/user")
|
|
require.NoError(t, err)
|
|
// Count only regular files (not directories)
|
|
regularFiles := 0
|
|
|
|
for _, f := range files {
|
|
if f.Mode&0x80000000 == 0 { // Check if regular file (not directory)
|
|
regularFiles++
|
|
}
|
|
}
|
|
|
|
assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database")
|
|
|
|
// Verify chunks were created by checking a specific file
|
|
fileChunks, err := repos.FileChunks.GetByPath(ctx,
|
|
"/home/user/documents/file1.txt")
|
|
require.NoError(t, err)
|
|
assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt")
|
|
}
|
|
|
|
// newEndToEndTestConfig builds the standard config used by the
|
|
// end-to-end backup test.
|
|
func newEndToEndTestConfig() *config.Config {
|
|
return &config.Config{
|
|
Snapshots: map[string]config.SnapshotConfig{
|
|
testLabel: {
|
|
Paths: []string{"/home/user"},
|
|
},
|
|
},
|
|
Exclude: []string{"*.tmp", "*.log"},
|
|
ChunkSize: config.Size(16 * 1024), // 16KB chunks
|
|
BlobSizeLimit: config.Size(100 * 1024), // 100KB blobs
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{testAgePublicKey},
|
|
AgeSecretKey: testAgeSecretKey,
|
|
S3: config.S3Config{
|
|
Endpoint: "http://localhost:9000", // MinIO endpoint for testing
|
|
Region: "us-east-1",
|
|
Bucket: "test-bucket",
|
|
AccessKeyID: "test-access",
|
|
SecretAccessKey: "test-secret",
|
|
},
|
|
IndexPath: ":memory:", // In-memory SQLite database
|
|
}
|
|
}
|
|
|
|
func TestEndToEndBackup(t *testing.T) {
|
|
// Initialize logger
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
// Create in-memory filesystem
|
|
fs := afero.NewMemMapFs()
|
|
|
|
// Create test directory structure and files
|
|
testFiles := map[string]string{
|
|
"/home/user/documents/file1.txt": "This is file 1 content",
|
|
"/home/user/documents/file2.txt": "This is file 2 content with more data",
|
|
"/home/user/pictures/photo1.jpg": "Binary photo data here...",
|
|
"/home/user/code/main.go": "package main\n\nfunc main() {\n" +
|
|
"\tprintln(\"Hello, World!\")\n}",
|
|
}
|
|
|
|
writeTestFileTree(t, fs, testFiles)
|
|
|
|
// Create mock storage
|
|
mockStorage := NewMockStorer()
|
|
|
|
cfg := newEndToEndTestConfig()
|
|
|
|
// For a true end-to-end test, we'll create a simpler test that focuses on
|
|
// the core backup logic using the scanner directly with our mock storage
|
|
ctx := context.Background()
|
|
|
|
// Create in-memory database
|
|
db, err := database.New(ctx, ":memory:")
|
|
|
|
require.NoError(t, err)
|
|
|
|
defer func() {
|
|
err := db.Close()
|
|
if err != nil {
|
|
t.Errorf("failed to close database: %v", err)
|
|
}
|
|
}()
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
// Create scanner with mock storage
|
|
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
|
FS: fs,
|
|
ChunkSize: cfg.ChunkSize.Int64(),
|
|
Repositories: repos,
|
|
Storage: mockStorage,
|
|
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
|
|
CompressionLevel: cfg.CompressionLevel,
|
|
AgeRecipients: cfg.AgeRecipients,
|
|
EnableProgress: false,
|
|
})
|
|
|
|
// Create a snapshot record
|
|
snapshotID := "test-snapshot-001"
|
|
createTestSnapshotRecord(t, repos, snapshotID)
|
|
|
|
// Run the backup scan
|
|
result, err := scanner.Scan(ctx, "/home/user", snapshotID)
|
|
require.NoError(t, err)
|
|
|
|
// Verify scan results. The scanner counts both files and
|
|
// directories: 4 files + directories (/home, /home/user,
|
|
// /home/user/documents, /home/user/pictures, /home/user/code).
|
|
assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files")
|
|
assert.Positive(t, result.BytesScanned, "Should scan some bytes")
|
|
assert.Positive(t, result.ChunksCreated, "Should create chunks")
|
|
assert.Positive(t, result.BlobsCreated, "Should create blobs")
|
|
|
|
verifyEndToEndBackupState(ctx, t, repos, mockStorage)
|
|
|
|
// Verify blobs were uploaded to storage
|
|
assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage")
|
|
|
|
// Complete the snapshot - just verify we got results
|
|
// In a real integration test, we'd update the snapshot record
|
|
|
|
// Create snapshot manager to test metadata export
|
|
snapshotManager := &snapshot.SnapshotManager{}
|
|
snapshotManager.SetFilesystem(fs)
|
|
|
|
// Note: We can't fully test snapshot metadata export without a proper S3 client mock
|
|
// that implements all required methods. This would require refactoring the S3 client
|
|
// interface to be more testable.
|
|
|
|
t.Logf("Backup completed successfully:")
|
|
t.Logf(" Files scanned: %d", result.FilesScanned)
|
|
t.Logf(" Bytes scanned: %d", result.BytesScanned)
|
|
t.Logf(" Chunks created: %d", result.ChunksCreated)
|
|
t.Logf(" Blobs created: %d", result.BlobsCreated)
|
|
t.Logf(" Storage size: %d objects", mockStorage.GetStorageSize())
|
|
}
|
|
|
|
// TestBackupAndVerify tests backing up files and verifying the blobs
|
|
func TestBackupAndVerify(t *testing.T) {
|
|
// Initialize logger
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
// Create in-memory filesystem
|
|
fs := afero.NewMemMapFs()
|
|
|
|
// Create test files
|
|
testContent := "This is a test file with some content that should be backed up"
|
|
err := fs.MkdirAll("/data", 0755)
|
|
require.NoError(t, err)
|
|
err = afero.WriteFile(fs, "/data/test.txt", []byte(testContent), 0644)
|
|
require.NoError(t, err)
|
|
|
|
// Create mock storage
|
|
mockStorage := NewMockStorer()
|
|
|
|
// Create test database
|
|
ctx := context.Background()
|
|
db, err := database.New(ctx, ":memory:")
|
|
|
|
require.NoError(t, err)
|
|
|
|
defer func() {
|
|
err := db.Close()
|
|
if err != nil {
|
|
t.Errorf("failed to close database: %v", err)
|
|
}
|
|
}()
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
// Create scanner
|
|
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
|
FS: fs,
|
|
ChunkSize: int64(1024 * 16), // 16KB chunks
|
|
Repositories: repos,
|
|
Storage: mockStorage,
|
|
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{testAgePublicKey},
|
|
})
|
|
|
|
// Create a snapshot
|
|
snapshotID := "test-snapshot-001"
|
|
createTestSnapshotRecord(t, repos, snapshotID)
|
|
|
|
// Run the backup
|
|
result, err := scanner.Scan(ctx, "/data", snapshotID)
|
|
require.NoError(t, err)
|
|
|
|
// Verify backup created blobs
|
|
assert.Positive(t, result.BlobsCreated, "Should create at least one blob")
|
|
assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated,
|
|
"Storage should have the blobs")
|
|
|
|
// Verify we can retrieve the blob from storage
|
|
objects, err := mockStorage.List(ctx, "blobs/")
|
|
require.NoError(t, err)
|
|
assert.Len(t, objects, result.BlobsCreated,
|
|
"Should have correct number of blobs in storage")
|
|
|
|
// Get the first blob and verify it exists
|
|
if len(objects) > 0 {
|
|
blobKey := objects[0]
|
|
t.Logf("Verifying blob: %s", blobKey)
|
|
|
|
// Get blob info
|
|
blobInfo, err := mockStorage.Stat(ctx, blobKey)
|
|
require.NoError(t, err)
|
|
assert.Positive(t, blobInfo.Size, "Blob should have content")
|
|
|
|
// Get blob content
|
|
reader, err := mockStorage.Get(ctx, blobKey)
|
|
require.NoError(t, err)
|
|
|
|
defer func() { _ = reader.Close() }()
|
|
|
|
// Verify blob data is encrypted (should not contain plaintext)
|
|
blobData, err := io.ReadAll(reader)
|
|
require.NoError(t, err)
|
|
assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted")
|
|
assert.NotEmpty(t, blobData, "Blob should have data")
|
|
}
|
|
|
|
t.Logf("Backup and verify test completed successfully")
|
|
}
|
|
|
|
// TestBackupAndRestore tests the full backup and restore workflow
|
|
// This test verifies that the restore code correctly handles the binary SQLite
|
|
// database format that is exported by the snapshot manager.
|
|
// runBackupPhase performs the backup half of the round-trip test:
|
|
// create the snapshot, scan the data directory, complete the snapshot,
|
|
// and export its metadata, verifying the metadata upload.
|
|
func runBackupPhase(
|
|
ctx context.Context, t *testing.T, fs afero.Fs,
|
|
repos *database.Repositories, mockStorage *MockStorer,
|
|
cfg *config.Config, dataDir, dbPath, agePublicKey string,
|
|
) string {
|
|
t.Helper()
|
|
|
|
// Create snapshot manager
|
|
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
|
Repos: repos,
|
|
Storage: mockStorage,
|
|
Config: cfg,
|
|
})
|
|
sm.SetFilesystem(fs)
|
|
|
|
// Create scanner
|
|
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
|
FS: fs,
|
|
Storage: mockStorage,
|
|
ChunkSize: int64(16 * 1024),
|
|
MaxBlobSize: int64(100 * 1024),
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{agePublicKey},
|
|
Repositories: repos,
|
|
})
|
|
|
|
// Create a snapshot
|
|
snapshotID, err := sm.CreateSnapshotWithName(
|
|
ctx, testHostname, "", "test-version", "test-git")
|
|
require.NoError(t, err)
|
|
t.Logf("Created snapshot: %s", snapshotID)
|
|
|
|
// Run the backup (scan)
|
|
result, err := scanner.Scan(ctx, dataDir, snapshotID)
|
|
require.NoError(t, err)
|
|
t.Logf("Scan complete: %d files, %d blobs",
|
|
result.FilesScanned, result.BlobsCreated)
|
|
|
|
// Complete the snapshot
|
|
err = sm.CompleteSnapshot(ctx, snapshotID)
|
|
require.NoError(t, err)
|
|
|
|
// Export snapshot metadata (this uploads db.zst.age and manifest.json.zst)
|
|
err = sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)
|
|
require.NoError(t, err)
|
|
t.Logf("Exported snapshot metadata")
|
|
|
|
// Verify metadata was uploaded
|
|
keys, err := mockStorage.List(ctx, "metadata/")
|
|
require.NoError(t, err)
|
|
t.Logf("Metadata keys: %v", keys)
|
|
assert.GreaterOrEqual(t, len(keys), 2,
|
|
"Should have at least db.zst.age and manifest.json.zst")
|
|
|
|
return snapshotID
|
|
}
|
|
|
|
func TestBackupAndRestore(t *testing.T) {
|
|
// Initialize logger
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
// Create real temp directory for the database (SQLite needs real filesystem)
|
|
realTempDir := t.TempDir()
|
|
|
|
// Use real OS filesystem for this test
|
|
fs := afero.NewOsFs()
|
|
|
|
// Create test directory structure and files
|
|
dataDir := filepath.Join(realTempDir, "data")
|
|
testFiles := map[string]string{
|
|
filepath.Join(dataDir, "file1.txt"): "This is file 1 content",
|
|
filepath.Join(dataDir, "file2.txt"): "This is file 2 content " +
|
|
"with more data",
|
|
filepath.Join(dataDir, "subdir", "file3.txt"): "This is file 3 " +
|
|
"in a subdirectory",
|
|
}
|
|
|
|
// Create directories and files
|
|
writeTestFileTree(t, fs, testFiles)
|
|
|
|
ctx := context.Background()
|
|
|
|
// Create mock storage
|
|
mockStorage := NewMockStorer()
|
|
|
|
// Test keypair
|
|
agePublicKey := testAgePublicKey
|
|
ageSecretKey := testAgeSecretKey
|
|
|
|
// Create database file
|
|
dbPath := filepath.Join(realTempDir, "test.db")
|
|
db, err := database.New(ctx, dbPath)
|
|
require.NoError(t, err)
|
|
|
|
defer func() { _ = db.Close() }()
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
// Create config for snapshot manager
|
|
cfg := &config.Config{
|
|
AgeSecretKey: ageSecretKey,
|
|
AgeRecipients: []string{agePublicKey},
|
|
CompressionLevel: 3,
|
|
}
|
|
|
|
snapshotID := runBackupPhase(
|
|
ctx, t, fs, repos, mockStorage, cfg, dataDir, dbPath, agePublicKey)
|
|
|
|
// Close the source database
|
|
err = db.Close()
|
|
require.NoError(t, err)
|
|
|
|
// Create Vaultik instance for restore
|
|
vaultikApp := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: mockStorage,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
vaultikApp.SetContext(ctx)
|
|
|
|
// Try to restore - this should work with binary SQLite format
|
|
restoreDir := filepath.Join(realTempDir, "restored")
|
|
err = vaultikApp.Restore(&vaultik.RestoreOptions{
|
|
SnapshotID: snapshotID,
|
|
TargetDir: restoreDir,
|
|
})
|
|
require.NoError(t, err, "Restore should succeed with binary SQLite database format")
|
|
|
|
// Verify restored files match originals
|
|
for origPath, expectedContent := range testFiles {
|
|
restoredPath := filepath.Join(restoreDir, origPath)
|
|
restoredContent, err := afero.ReadFile(fs, restoredPath)
|
|
require.NoError(t, err,
|
|
"Should be able to read restored file: %s", restoredPath)
|
|
assert.Equal(t, expectedContent, string(restoredContent),
|
|
"Restored content should match original for: %s", origPath)
|
|
}
|
|
|
|
t.Log("Backup and restore test completed successfully")
|
|
}
|
|
|
|
// setupE2ESourceTree writes a representative mix of file sizes (empty
|
|
// file, tiny text file, file under the chunk boundary, multi-chunk
|
|
// file, nested subdirectories), a permission-restricted file, an empty
|
|
// directory, and a symlink. It returns the content map keyed by path
|
|
// plus the restricted-file, empty-dir, and symlink paths.
|
|
func setupE2ESourceTree(
|
|
t *testing.T, fs afero.Fs, dataDir string, chunkSize int64,
|
|
) (map[string][]byte, string, string, string) {
|
|
t.Helper()
|
|
|
|
testFiles := map[string][]byte{
|
|
filepath.Join(dataDir, "empty.txt"): {},
|
|
filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"),
|
|
filepath.Join(dataDir, "subdir", "medium.bin"): bytesPattern(
|
|
"medium-", int(chunkSize/2)),
|
|
filepath.Join(dataDir, "subdir", "large.bin"): bytesPattern(
|
|
"large-", int(chunkSize*4)),
|
|
filepath.Join(dataDir, "deep", "nest", "leaf.txt"): []byte("leaf"),
|
|
}
|
|
|
|
for path, content := range testFiles {
|
|
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
|
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
|
}
|
|
|
|
// Create a file with non-default permissions.
|
|
restrictedPath := filepath.Join(dataDir, "restricted.txt")
|
|
require.NoError(t, afero.WriteFile(fs, restrictedPath, []byte("secret"), 0o600))
|
|
|
|
testFiles[restrictedPath] = []byte("secret")
|
|
|
|
// Create an empty directory (should survive round-trip).
|
|
emptyDir := filepath.Join(dataDir, "emptydir")
|
|
require.NoError(t, fs.MkdirAll(emptyDir, 0o755))
|
|
|
|
// Create a symlink.
|
|
symlinkPath := filepath.Join(dataDir, "link-to-small")
|
|
require.NoError(t, os.Symlink("small.txt", symlinkPath))
|
|
|
|
return testFiles, restrictedPath, emptyDir, symlinkPath
|
|
}
|
|
|
|
// TestEndToEndFileStorage exercises the full backup → restore loop against the
|
|
// real `file://` storage backend (FileStorer) on a real OS filesystem. This is
|
|
// the closest local approximation of a production backup: encrypted blobs get
|
|
// written to disk, the metadata SQLite database is exported through the same
|
|
// blobgen pipeline as a real backup, and restoration reads them back through
|
|
// the public Vaultik.Restore entrypoint. It is the canonical end-to-end smoke
|
|
// test for 1.0.
|
|
// runFileStorageBackup performs the backup half of the file-storage
|
|
// end-to-end test against a real on-disk FileStorer, verifies the
|
|
// on-disk layout, and closes the index database so the restore half
|
|
// runs from remote bytes only.
|
|
func runFileStorageBackup(
|
|
ctx context.Context, t *testing.T, fs afero.Fs,
|
|
dataDir, storeDir, dbPath string,
|
|
chunkSize, maxBlobSize int64,
|
|
) (*config.Config, *storage.FileStorer, string) {
|
|
t.Helper()
|
|
|
|
// FileStorer is the real-world local-disk backend.
|
|
storer, err := storage.NewFileStorer(storeDir)
|
|
require.NoError(t, err)
|
|
|
|
cfg := &config.Config{
|
|
AgeRecipients: []string{testAgePublicKey},
|
|
AgeSecretKey: testAgeSecretKey,
|
|
CompressionLevel: 3,
|
|
Hostname: testHostname,
|
|
}
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
require.NoError(t, err)
|
|
|
|
defer func() { _ = db.Close() }()
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
|
Repos: repos,
|
|
Storage: storer,
|
|
Config: cfg,
|
|
})
|
|
sm.SetFilesystem(fs)
|
|
|
|
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
|
FS: fs,
|
|
Storage: storer,
|
|
ChunkSize: chunkSize,
|
|
MaxBlobSize: maxBlobSize,
|
|
CompressionLevel: cfg.CompressionLevel,
|
|
AgeRecipients: cfg.AgeRecipients,
|
|
Repositories: repos,
|
|
})
|
|
|
|
snapshotID, err := sm.CreateSnapshotWithName(
|
|
ctx, cfg.Hostname, "e2e", "test-version", "test-git")
|
|
require.NoError(t, err)
|
|
|
|
scanResult, err := scanner.Scan(ctx, dataDir, snapshotID)
|
|
require.NoError(t, err)
|
|
require.Positive(t, scanResult.FilesScanned)
|
|
require.Positive(t, scanResult.BlobsCreated)
|
|
|
|
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
|
|
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
|
|
|
|
// Verify the backup actually landed on disk under blobs/ and metadata/.
|
|
// The metadata subdirectory uses the hashed remote key, not the human
|
|
// snapshot ID, so the on-disk structure doesn't leak hostname/name/time.
|
|
blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs"))
|
|
require.NoError(t, err)
|
|
require.True(t, blobInfo.IsDir())
|
|
|
|
metaInfo, err := os.Stat(filepath.Join(
|
|
storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID)))
|
|
require.NoError(t, err)
|
|
require.True(t, metaInfo.IsDir())
|
|
|
|
// Tear down the source DB before restore — restore must work using only
|
|
// the remote bytes plus the secret key, with no help from the local index.
|
|
require.NoError(t, db.Close())
|
|
|
|
return cfg, storer, snapshotID
|
|
}
|
|
|
|
func TestEndToEndFileStorage(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
// Real OS filesystem (SQLite + FileStorer both need it).
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
restoreDir := filepath.Join(tempDir, "restored")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
// Write a representative mix of file sizes:
|
|
// - empty file
|
|
// - tiny text file
|
|
// - file just under chunk boundary
|
|
// - file forcing multiple chunks
|
|
// - nested subdirectories
|
|
chunkSize := int64(64 * 1024)
|
|
maxBlobSize := int64(512 * 1024)
|
|
|
|
testFiles, restrictedPath, emptyDir, symlinkPath :=
|
|
setupE2ESourceTree(t, fs, dataDir, chunkSize)
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
restoreVaultik := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
restoreVaultik.SetContext(ctx)
|
|
|
|
require.NoError(t, restoreVaultik.Restore(&vaultik.RestoreOptions{
|
|
SnapshotID: snapshotID,
|
|
TargetDir: restoreDir,
|
|
Verify: true,
|
|
}))
|
|
|
|
verifyE2ERestoredTree(t, fs, restoreDir, testFiles,
|
|
restrictedPath, emptyDir, symlinkPath)
|
|
}
|
|
|
|
// verifyE2ERestoredTree byte-compares every restored file and checks the
|
|
// restricted-permission file, empty directory, and symlink special cases.
|
|
func verifyE2ERestoredTree(
|
|
t *testing.T, fs afero.Fs, restoreDir string,
|
|
testFiles map[string][]byte,
|
|
restrictedPath, emptyDir, symlinkPath string,
|
|
) {
|
|
t.Helper()
|
|
|
|
// Byte-equality compare every original against its restored copy.
|
|
for origPath, expected := range testFiles {
|
|
restoredPath := filepath.Join(restoreDir, origPath)
|
|
got, err := afero.ReadFile(fs, restoredPath)
|
|
require.NoError(t, err, "restored file missing: %s", restoredPath)
|
|
require.Equalf(t, expected, got, "byte-equality failed for %s", origPath)
|
|
}
|
|
|
|
// Verify the restricted file kept its permissions.
|
|
restoredRestricted := filepath.Join(restoreDir, restrictedPath)
|
|
rInfo, err := os.Stat(restoredRestricted)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, os.FileMode(0o600), rInfo.Mode().Perm(),
|
|
"restricted file should preserve 0600 permissions")
|
|
|
|
// Verify the empty directory was restored.
|
|
restoredEmptyDir := filepath.Join(restoreDir, emptyDir)
|
|
dInfo, err := os.Stat(restoredEmptyDir)
|
|
require.NoError(t, err, "empty directory should be restored")
|
|
assert.True(t, dInfo.IsDir(), "emptydir should be a directory")
|
|
|
|
// Verify the symlink was restored with the correct target.
|
|
restoredSymlink := filepath.Join(restoreDir, symlinkPath)
|
|
target, err := os.Readlink(restoredSymlink)
|
|
require.NoError(t, err, "symlink should be restored")
|
|
assert.Equal(t, "small.txt", target, "symlink target should be preserved")
|
|
}
|
|
|
|
// TestDedupOnlySnapshotRestores backs up the same directory twice without
|
|
// touching it between runs, then restores the SECOND (fully-deduplicated)
|
|
// snapshot. The second snapshot uploads no new blobs — every chunk is
|
|
// already in storage from the first run. This test guards against the
|
|
// regression where snapshot_blobs was populated only for blobs uploaded
|
|
// during the snapshot, leaving fully-deduplicated snapshots unrestorable
|
|
// with "chunk X not found in any blob" errors.
|
|
// dedupBackupEnv bundles the moving parts of the dedup round-trip test.
|
|
type dedupBackupEnv struct {
|
|
cfg *config.Config
|
|
storer *storage.FileStorer
|
|
db *database.DB
|
|
repos *database.Repositories
|
|
sm *snapshot.SnapshotManager
|
|
makeScanner func() *snapshot.Scanner
|
|
}
|
|
|
|
// writeDedupSourceFiles writes the two deterministic source files used
|
|
// by the dedup test and returns their expected contents by path.
|
|
func writeDedupSourceFiles(
|
|
t *testing.T, fs afero.Fs, dataDir string, chunkSize int64,
|
|
) map[string][]byte {
|
|
t.Helper()
|
|
|
|
testFiles := map[string][]byte{
|
|
filepath.Join(dataDir, "a.bin"): bytesPattern("a-", int(chunkSize*3)),
|
|
filepath.Join(dataDir, "b.bin"): bytesPattern("b-", int(chunkSize*2)),
|
|
}
|
|
for path, content := range testFiles {
|
|
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
|
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
|
}
|
|
|
|
return testFiles
|
|
}
|
|
|
|
// setupDedupBackupEnv creates the storer, config, database, snapshot
|
|
// manager, and scanner factory for the dedup round-trip test.
|
|
func setupDedupBackupEnv(
|
|
ctx context.Context, t *testing.T, fs afero.Fs,
|
|
storeDir, dbPath string, chunkSize, maxBlobSize int64,
|
|
) *dedupBackupEnv {
|
|
t.Helper()
|
|
|
|
storer, err := storage.NewFileStorer(storeDir)
|
|
require.NoError(t, err)
|
|
|
|
cfg := &config.Config{
|
|
AgeRecipients: []string{testAgePublicKey},
|
|
AgeSecretKey: testAgeSecretKey,
|
|
CompressionLevel: 3,
|
|
Hostname: testHostname,
|
|
}
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
require.NoError(t, err)
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
makeScanner := func() *snapshot.Scanner {
|
|
return snapshot.NewScanner(snapshot.ScannerConfig{
|
|
FS: fs,
|
|
Storage: storer,
|
|
ChunkSize: chunkSize,
|
|
MaxBlobSize: maxBlobSize,
|
|
CompressionLevel: cfg.CompressionLevel,
|
|
AgeRecipients: cfg.AgeRecipients,
|
|
Repositories: repos,
|
|
})
|
|
}
|
|
sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{
|
|
Repos: repos, Storage: storer, Config: cfg,
|
|
})
|
|
sm.SetFilesystem(fs)
|
|
|
|
return &dedupBackupEnv{
|
|
cfg: cfg,
|
|
storer: storer,
|
|
db: db,
|
|
repos: repos,
|
|
sm: sm,
|
|
makeScanner: makeScanner,
|
|
}
|
|
}
|
|
|
|
// runDedupSnapshot creates a "dedup" snapshot, scans dataDir into it,
|
|
// completes it, and exports its metadata, returning the snapshot ID and
|
|
// scan result.
|
|
func runDedupSnapshot(
|
|
ctx context.Context, t *testing.T,
|
|
sm *snapshot.SnapshotManager, scanner *snapshot.Scanner,
|
|
hostname, dataDir, dbPath string,
|
|
) (string, *snapshot.ScanResult) {
|
|
t.Helper()
|
|
|
|
id, err := sm.CreateSnapshotWithName(ctx, hostname, "dedup", "v", "g")
|
|
require.NoError(t, err)
|
|
|
|
result, err := scanner.Scan(ctx, dataDir, id)
|
|
require.NoError(t, err)
|
|
|
|
require.NoError(t, sm.CompleteSnapshot(ctx, id))
|
|
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id))
|
|
|
|
return id, result
|
|
}
|
|
|
|
func TestDedupOnlySnapshotRestores(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
restoreDir := filepath.Join(tempDir, "restored")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(64 * 1024)
|
|
maxBlobSize := int64(512 * 1024)
|
|
|
|
ctx := context.Background()
|
|
|
|
testFiles := writeDedupSourceFiles(t, fs, dataDir, chunkSize)
|
|
|
|
env := setupDedupBackupEnv(
|
|
ctx, t, fs, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
defer func() { _ = env.db.Close() }()
|
|
|
|
cfg, storer, repos, sm := env.cfg, env.storer, env.repos, env.sm
|
|
makeScanner := env.makeScanner
|
|
db := env.db
|
|
|
|
// First snapshot — uploads all blobs.
|
|
_, r1 := runDedupSnapshot(ctx, t, sm, makeScanner(),
|
|
cfg.Hostname, dataDir, dbPath)
|
|
require.Positive(t, r1.BlobsCreated,
|
|
"first snapshot should upload at least one blob")
|
|
|
|
// Second snapshot — same data, every chunk dedups. Sleep past the
|
|
// second-precision timestamp so the snapshot IDs differ.
|
|
time.Sleep(1100 * time.Millisecond)
|
|
|
|
id2, r2 := runDedupSnapshot(ctx, t, sm, makeScanner(),
|
|
cfg.Hostname, dataDir, dbPath)
|
|
require.Equal(t, 0, r2.BlobsCreated,
|
|
"second snapshot should upload zero new blobs (fully dedup'd)")
|
|
|
|
// snapshot_blobs for id2 must be populated despite no uploads.
|
|
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, id2)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, blobHashes, "snapshot_blobs for fully-dedup'd snapshot "+
|
|
"must reference blobs uploaded by prior snapshot")
|
|
|
|
require.NoError(t, db.Close())
|
|
|
|
restoreVaultik := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
restoreVaultik.SetContext(ctx)
|
|
|
|
require.NoError(t, restoreVaultik.Restore(&vaultik.RestoreOptions{
|
|
SnapshotID: id2,
|
|
TargetDir: restoreDir,
|
|
Verify: true,
|
|
}))
|
|
|
|
for origPath, expected := range testFiles {
|
|
restoredPath := filepath.Join(restoreDir, origPath)
|
|
got, err := afero.ReadFile(fs, restoredPath)
|
|
require.NoError(t, err, "restored file missing: %s", restoredPath)
|
|
require.Equalf(t, expected, got, "byte-equality failed for %s", origPath)
|
|
}
|
|
}
|
|
|
|
// bytesPattern returns a deterministic byte slice of length n with a tag prefix,
|
|
// useful for forcing chunker behavior with reproducible content.
|
|
func bytesPattern(tag string, n int) []byte {
|
|
out := make([]byte, n)
|
|
for i := range out {
|
|
out[i] = tag[i%len(tag)] ^ byte(i&0xff)
|
|
}
|
|
|
|
return out
|
|
}
|