Files
vaultik/internal/snapshot/snapshot_test.go
T
sneak 98a36c6a0c
check / check (push) Waiting to run
Take a snapshot's start time from the moment in its ID (closes #482)
CreateSnapshotWithName read the clock once for the ID and again for
started_at. When the second read fell in the next second, the snapshot
shared its started_at with the next run of the same name, and
ListRecent, which orders by started_at, could list the two in either
order. That made TestBackupWithEmptySourcePathDoesNotPrune fail at
random. The start time now comes from the same clock read as the ID, so
successive runs of one name have distinct, increasing start times.

Model: opus-5-5
2026-11-19 12:00:32 +00:00

547 lines
14 KiB
Go

//nolint:testpackage // exercises unexported SnapshotManager internals
package snapshot
import (
"bytes"
"context"
"database/sql"
"io"
"path/filepath"
"slices"
"testing"
"github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/types"
)
const (
// Test age public key for encryption
testAgeRecipient = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
)
// copyFile is a test helper to copy files using afero
func copyFile(fs afero.Fs, src, dst string) error {
sourceFile, err := fs.Open(src)
if err != nil {
return err
}
defer func() { _ = sourceFile.Close() }()
destFile, err := fs.Create(dst)
if err != nil {
return err
}
defer func() { _ = destFile.Close() }()
_, err = io.Copy(destFile, sourceFile)
return err
}
// verifyCleanedDB opens the cleaned database and checks that the kept
// snapshot survived while the orphan file and chunk were removed.
func verifyCleanedDB(
ctx context.Context,
t *testing.T,
tempDBPath, snapshotID string,
file *database.File,
chunk *database.Chunk,
) {
t.Helper()
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID)
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
}
// TestVacuumDatabaseRemovesDeletedData proves the export path uploads a
// compacted database: after rows carrying a recognizable marker are deleted
// and vacuumDatabase runs, no page holding that marker survives in the file
// on disk (the file compressFile later reads for upload).
func TestVacuumDatabaseRemovesDeletedData(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "snapshot.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
// A marker distinctive enough that its presence in the raw file can only
// come from the rows inserted below.
marker := []byte("VACUUM_PROBE_DEADBEEF_DELETED_ROW")
payload := bytes.Repeat(marker, 128) // ~4 KiB per row
_, err = db.Conn().ExecContext(ctx,
"CREATE TABLE vacuum_probe (id INTEGER PRIMARY KEY, payload BLOB)")
if err != nil {
t.Fatalf("failed to create probe table: %v", err)
}
for range 512 {
_, err = db.Conn().ExecContext(ctx,
"INSERT INTO vacuum_probe (payload) VALUES (?)", payload)
if err != nil {
t.Fatalf("failed to insert probe row: %v", err)
}
}
_, err = db.Conn().ExecContext(ctx, "DELETE FROM vacuum_probe")
if err != nil {
t.Fatalf("failed to delete probe rows: %v", err)
}
// Close so the deletes reach the main file, mirroring the state
// prepareExportDB hands to vacuumDatabase.
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
beforeInfo, err := fs.Stat(dbPath)
if err != nil {
t.Fatalf("failed to stat database before vacuum: %v", err)
}
beforeBytes, err := afero.ReadFile(fs, dbPath)
if err != nil {
t.Fatalf("failed to read database before vacuum: %v", err)
}
if !bytes.Contains(beforeBytes, marker) {
t.Fatalf("expected deleted-row data to linger before vacuum")
}
sm := &SnapshotManager{fs: fs}
err = sm.vacuumDatabase(ctx, dbPath)
if err != nil {
t.Fatalf("vacuumDatabase failed: %v", err)
}
afterBytes, err := afero.ReadFile(fs, dbPath)
if err != nil {
t.Fatalf("failed to read database after vacuum: %v", err)
}
if bytes.Contains(afterBytes, marker) {
t.Fatalf("deleted-row data survived vacuum in the uploaded file")
}
afterInfo, err := fs.Stat(dbPath)
if err != nil {
t.Fatalf("failed to stat database after vacuum: %v", err)
}
if afterInfo.Size() >= beforeInfo.Size() {
t.Fatalf("expected vacuum to shrink the file: before=%d after=%d",
beforeInfo.Size(), afterInfo.Size())
}
}
// TestPrepareExportDBKeepsRowsCommittedToOpenIndex exports from an index
// that is still open, as a backup does. A row committed there can still be
// in the index's -wal file, and the export must hold it all the same.
func TestPrepareExportDBKeepsRowsCommittedToOpenIndex(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
dbPath := filepath.Join(t.TempDir(), "index.sqlite")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
snapshot := &database.Snapshot{ID: "open-index-snapshot", Hostname: "test-host"}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
sm := &SnapshotManager{
config: &config.Config{
CompressionLevel: 3,
AgeRecipients: []string{testAgeRecipient},
},
fs: fs,
}
_, tempDBPath, err := sm.prepareExportDB(
ctx, dbPath, snapshot.ID.String(), t.TempDir())
if err != nil {
t.Fatalf("prepareExportDB failed: %v", err)
}
// Only the main database file is compressed and uploaded, so open a
// copy of that file alone.
uploadedPath := filepath.Join(t.TempDir(), "uploaded.db")
err = copyFile(fs, tempDBPath, uploadedPath)
if err != nil {
t.Fatalf("failed to copy exported database: %v", err)
}
exported, err := database.OpenReadOnly(ctx, uploadedPath)
if err != nil {
t.Fatalf("failed to open exported database: %v", err)
}
defer func() { _ = exported.Close() }()
got, err := database.NewRepositories(exported).Snapshots.GetByID(
ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to read snapshot from export: %v", err)
}
if got == nil {
t.Fatal("exported database is missing the snapshot row")
}
}
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
// Create a test database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
repos := database.NewRepositories(db)
// Create an empty snapshot
snapshot := &database.Snapshot{
ID: "empty-snapshot",
Hostname: "test-host",
}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
// Create some files and chunks not associated with any snapshot
file := &database.File{Path: "/orphan/file.txt", Size: 1000}
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
return repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
t.Fatalf("failed to create orphan data: %v", err)
}
// Close the database
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
// Copy database
tempDBPath := filepath.Join(tempDir, "temp.db")
err = copyFile(fs, dbPath, tempDBPath)
if err != nil {
t.Fatalf("failed to copy database: %v", err)
}
// Create a mock config for testing
cfg := &config.Config{
CompressionLevel: 3,
AgeRecipients: []string{testAgeRecipient},
}
// Create SnapshotManager with filesystem
sm := &SnapshotManager{
config: cfg,
fs: fs,
}
_, err = sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to clean snapshot database: %v", err)
}
// Verify the cleaned database
verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk)
}
func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
// Create a test database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
// Close immediately
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
// Copy database
tempDBPath := filepath.Join(tempDir, "temp.db")
err = copyFile(fs, dbPath, tempDBPath)
if err != nil {
t.Fatalf("failed to copy database: %v", err)
}
// Create a mock config for testing
cfg := &config.Config{
CompressionLevel: 3,
AgeRecipients: []string{testAgeRecipient},
}
// Try to clean with non-existent snapshot
sm := &SnapshotManager{config: cfg, fs: fs}
_, err = sm.cleanSnapshotDB(ctx, tempDBPath, "non-existent-snapshot")
// Should not error - it will just delete everything
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
// The exported metadata database keeps only the mount points that the
// exported snapshot recorded. See
// https://git.eeqj.de/sneak/vaultik/issues/395.
func TestCleanSnapshotDBKeepsOnlyTheSnapshotsMountPoints(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "export.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
repos := database.NewRepositories(db)
for _, id := range []types.SnapshotID{"exported", "other"} {
err = repos.Snapshots.Create(ctx, nil, &database.Snapshot{ID: id})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
err = repos.Snapshots.AddMountPoint(ctx, nil, id.String(), "/data/"+id.String())
if err != nil {
t.Fatalf("failed to add mount point: %v", err)
}
}
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
sm := &SnapshotManager{
config: &config.Config{
CompressionLevel: 3,
AgeRecipients: []string{testAgeRecipient},
},
fs: afero.NewOsFs(),
}
_, err = sm.cleanSnapshotDB(ctx, dbPath, "exported")
if err != nil {
t.Fatalf("failed to clean snapshot database: %v", err)
}
cleaned, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() { _ = cleaned.Close() }()
cleanedRepos := database.NewRepositories(cleaned)
for id, want := range map[string][]string{
"exported": {"/data/exported"},
"other": nil,
} {
got, err := cleanedRepos.Snapshots.GetMountPoints(ctx, id)
if err != nil {
t.Fatalf("failed to read mount points: %v", err)
}
if !slices.Equal(got, want) {
t.Errorf("mount points of %s: got %v, want %v", id, got, want)
}
}
}
// Two creates of one snapshot name back to back start within one second,
// the resolution of the timestamp in a snapshot ID. See
// https://git.eeqj.de/sneak/vaultik/issues/270.
func TestCreateSnapshotWithNameTwiceBackToBack(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
sm := &SnapshotManager{repos: database.NewRepositories(db)}
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("second create failed: %v", err)
}
if first == second {
t.Fatalf("both creates returned snapshot ID %s", first)
}
}
// Two creates of one snapshot name back to back list newest first. The
// start time is the moment in the ID, so the two cannot share a start
// time even when the clock crosses a second while the first is created.
// See https://git.eeqj.de/sneak/vaultik/issues/482.
func TestCreateSnapshotWithNameTwiceBackToBackListsNewestFirst(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const created = 2
ctx := context.Background()
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
sm := &SnapshotManager{repos: repos}
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
if err != nil {
t.Fatalf("second create failed: %v", err)
}
snapshots, err := repos.Snapshots.ListRecent(ctx, created)
if err != nil {
t.Fatalf("listing snapshots failed: %v", err)
}
if len(snapshots) != created {
t.Fatalf("listed %d snapshots, want %d", len(snapshots), created)
}
if !snapshots[0].StartedAt.After(snapshots[1].StartedAt) {
t.Fatalf("start times %v and %v are not in order",
snapshots[1].StartedAt, snapshots[0].StartedAt)
}
if snapshots[0].ID.String() != second || snapshots[1].ID.String() != first {
t.Fatalf("listed %s, %s; want %s, %s",
snapshots[0].ID, snapshots[1].ID, second, first)
}
}