check / check (push) Waiting to run
CreateSnapshotWithName read the clock once for the ID and again for started_at. When the second read fell in the next second, the snapshot shared its started_at with the next run of the same name, and ListRecent, which orders by started_at, could list the two in either order. That made TestBackupWithEmptySourcePathDoesNotPrune fail at random. The start time now comes from the same clock read as the ID, so successive runs of one name have distinct, increasing start times. Model: opus-5-5
547 lines
14 KiB
Go
547 lines
14 KiB
Go
//nolint:testpackage // exercises unexported SnapshotManager internals
|
|
package snapshot
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"io"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/spf13/afero"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/database"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/types"
|
|
)
|
|
|
|
const (
|
|
// Test age public key for encryption
|
|
testAgeRecipient = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
|
)
|
|
|
|
// copyFile is a test helper to copy files using afero
|
|
func copyFile(fs afero.Fs, src, dst string) error {
|
|
sourceFile, err := fs.Open(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() { _ = sourceFile.Close() }()
|
|
|
|
destFile, err := fs.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() { _ = destFile.Close() }()
|
|
|
|
_, err = io.Copy(destFile, sourceFile)
|
|
|
|
return err
|
|
}
|
|
|
|
// verifyCleanedDB opens the cleaned database and checks that the kept
|
|
// snapshot survived while the orphan file and chunk were removed.
|
|
func verifyCleanedDB(
|
|
ctx context.Context,
|
|
t *testing.T,
|
|
tempDBPath, snapshotID string,
|
|
file *database.File,
|
|
chunk *database.Chunk,
|
|
) {
|
|
t.Helper()
|
|
|
|
cleanedDB, err := database.New(ctx, tempDBPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to open cleaned database: %v", err)
|
|
}
|
|
|
|
defer func() {
|
|
err := cleanedDB.Close()
|
|
if err != nil {
|
|
t.Errorf("failed to close database: %v", err)
|
|
}
|
|
}()
|
|
|
|
cleanedRepos := database.NewRepositories(cleanedDB)
|
|
|
|
// Verify snapshot exists
|
|
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID)
|
|
if err != nil {
|
|
t.Fatalf("failed to get snapshot: %v", err)
|
|
}
|
|
|
|
if verifySnapshot == nil {
|
|
t.Error("snapshot should exist")
|
|
}
|
|
|
|
// Verify orphan file is gone
|
|
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to check file: %v", err)
|
|
}
|
|
|
|
if f != nil {
|
|
t.Error("orphan file should not exist")
|
|
}
|
|
|
|
// Verify orphan chunk is gone
|
|
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to check chunk: %v", err)
|
|
}
|
|
|
|
if c != nil {
|
|
t.Error("orphan chunk should not exist")
|
|
}
|
|
}
|
|
|
|
// TestVacuumDatabaseRemovesDeletedData proves the export path uploads a
|
|
// compacted database: after rows carrying a recognizable marker are deleted
|
|
// and vacuumDatabase runs, no page holding that marker survives in the file
|
|
// on disk (the file compressFile later reads for upload).
|
|
func TestVacuumDatabaseRemovesDeletedData(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
fs := afero.NewOsFs()
|
|
|
|
tempDir := t.TempDir()
|
|
dbPath := filepath.Join(tempDir, "snapshot.db")
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
// A marker distinctive enough that its presence in the raw file can only
|
|
// come from the rows inserted below.
|
|
marker := []byte("VACUUM_PROBE_DEADBEEF_DELETED_ROW")
|
|
payload := bytes.Repeat(marker, 128) // ~4 KiB per row
|
|
|
|
_, err = db.Conn().ExecContext(ctx,
|
|
"CREATE TABLE vacuum_probe (id INTEGER PRIMARY KEY, payload BLOB)")
|
|
if err != nil {
|
|
t.Fatalf("failed to create probe table: %v", err)
|
|
}
|
|
|
|
for range 512 {
|
|
_, err = db.Conn().ExecContext(ctx,
|
|
"INSERT INTO vacuum_probe (payload) VALUES (?)", payload)
|
|
if err != nil {
|
|
t.Fatalf("failed to insert probe row: %v", err)
|
|
}
|
|
}
|
|
|
|
_, err = db.Conn().ExecContext(ctx, "DELETE FROM vacuum_probe")
|
|
if err != nil {
|
|
t.Fatalf("failed to delete probe rows: %v", err)
|
|
}
|
|
|
|
// Close so the deletes reach the main file, mirroring the state
|
|
// prepareExportDB hands to vacuumDatabase.
|
|
err = db.Close()
|
|
if err != nil {
|
|
t.Fatalf("failed to close database: %v", err)
|
|
}
|
|
|
|
beforeInfo, err := fs.Stat(dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to stat database before vacuum: %v", err)
|
|
}
|
|
|
|
beforeBytes, err := afero.ReadFile(fs, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to read database before vacuum: %v", err)
|
|
}
|
|
|
|
if !bytes.Contains(beforeBytes, marker) {
|
|
t.Fatalf("expected deleted-row data to linger before vacuum")
|
|
}
|
|
|
|
sm := &SnapshotManager{fs: fs}
|
|
|
|
err = sm.vacuumDatabase(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("vacuumDatabase failed: %v", err)
|
|
}
|
|
|
|
afterBytes, err := afero.ReadFile(fs, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to read database after vacuum: %v", err)
|
|
}
|
|
|
|
if bytes.Contains(afterBytes, marker) {
|
|
t.Fatalf("deleted-row data survived vacuum in the uploaded file")
|
|
}
|
|
|
|
afterInfo, err := fs.Stat(dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to stat database after vacuum: %v", err)
|
|
}
|
|
|
|
if afterInfo.Size() >= beforeInfo.Size() {
|
|
t.Fatalf("expected vacuum to shrink the file: before=%d after=%d",
|
|
beforeInfo.Size(), afterInfo.Size())
|
|
}
|
|
}
|
|
|
|
// TestPrepareExportDBKeepsRowsCommittedToOpenIndex exports from an index
|
|
// that is still open, as a backup does. A row committed there can still be
|
|
// in the index's -wal file, and the export must hold it all the same.
|
|
func TestPrepareExportDBKeepsRowsCommittedToOpenIndex(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
fs := afero.NewOsFs()
|
|
|
|
dbPath := filepath.Join(t.TempDir(), "index.sqlite")
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
defer func() { _ = db.Close() }()
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
snapshot := &database.Snapshot{ID: "open-index-snapshot", Hostname: "test-host"}
|
|
|
|
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
sm := &SnapshotManager{
|
|
config: &config.Config{
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{testAgeRecipient},
|
|
},
|
|
fs: fs,
|
|
}
|
|
|
|
_, tempDBPath, err := sm.prepareExportDB(
|
|
ctx, dbPath, snapshot.ID.String(), t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("prepareExportDB failed: %v", err)
|
|
}
|
|
|
|
// Only the main database file is compressed and uploaded, so open a
|
|
// copy of that file alone.
|
|
uploadedPath := filepath.Join(t.TempDir(), "uploaded.db")
|
|
|
|
err = copyFile(fs, tempDBPath, uploadedPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to copy exported database: %v", err)
|
|
}
|
|
|
|
exported, err := database.OpenReadOnly(ctx, uploadedPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to open exported database: %v", err)
|
|
}
|
|
|
|
defer func() { _ = exported.Close() }()
|
|
|
|
got, err := database.NewRepositories(exported).Snapshots.GetByID(
|
|
ctx, snapshot.ID.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to read snapshot from export: %v", err)
|
|
}
|
|
|
|
if got == nil {
|
|
t.Fatal("exported database is missing the snapshot row")
|
|
}
|
|
}
|
|
|
|
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|
// Initialize logger
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
fs := afero.NewOsFs()
|
|
|
|
// Create a test database
|
|
tempDir := t.TempDir()
|
|
dbPath := filepath.Join(tempDir, "test.db")
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
// Create an empty snapshot
|
|
snapshot := &database.Snapshot{
|
|
ID: "empty-snapshot",
|
|
Hostname: "test-host",
|
|
}
|
|
|
|
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
// Create some files and chunks not associated with any snapshot
|
|
file := &database.File{Path: "/orphan/file.txt", Size: 1000}
|
|
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
|
|
|
|
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
|
err := repos.Files.Create(ctx, tx, file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return repos.Chunks.Create(ctx, tx, chunk)
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create orphan data: %v", err)
|
|
}
|
|
|
|
// Close the database
|
|
err = db.Close()
|
|
if err != nil {
|
|
t.Fatalf("failed to close database: %v", err)
|
|
}
|
|
|
|
// Copy database
|
|
tempDBPath := filepath.Join(tempDir, "temp.db")
|
|
|
|
err = copyFile(fs, dbPath, tempDBPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to copy database: %v", err)
|
|
}
|
|
|
|
// Create a mock config for testing
|
|
cfg := &config.Config{
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{testAgeRecipient},
|
|
}
|
|
// Create SnapshotManager with filesystem
|
|
sm := &SnapshotManager{
|
|
config: cfg,
|
|
fs: fs,
|
|
}
|
|
|
|
_, err = sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to clean snapshot database: %v", err)
|
|
}
|
|
|
|
// Verify the cleaned database
|
|
verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk)
|
|
}
|
|
|
|
func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
|
|
// Initialize logger
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
fs := afero.NewOsFs()
|
|
|
|
// Create a test database
|
|
tempDir := t.TempDir()
|
|
dbPath := filepath.Join(tempDir, "test.db")
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
// Close immediately
|
|
err = db.Close()
|
|
if err != nil {
|
|
t.Fatalf("failed to close database: %v", err)
|
|
}
|
|
|
|
// Copy database
|
|
tempDBPath := filepath.Join(tempDir, "temp.db")
|
|
|
|
err = copyFile(fs, dbPath, tempDBPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to copy database: %v", err)
|
|
}
|
|
|
|
// Create a mock config for testing
|
|
cfg := &config.Config{
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{testAgeRecipient},
|
|
}
|
|
// Try to clean with non-existent snapshot
|
|
sm := &SnapshotManager{config: cfg, fs: fs}
|
|
_, err = sm.cleanSnapshotDB(ctx, tempDBPath, "non-existent-snapshot")
|
|
|
|
// Should not error - it will just delete everything
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
// The exported metadata database keeps only the mount points that the
|
|
// exported snapshot recorded. See
|
|
// https://git.eeqj.de/sneak/vaultik/issues/395.
|
|
func TestCleanSnapshotDBKeepsOnlyTheSnapshotsMountPoints(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
dbPath := filepath.Join(t.TempDir(), "export.db")
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
for _, id := range []types.SnapshotID{"exported", "other"} {
|
|
err = repos.Snapshots.Create(ctx, nil, &database.Snapshot{ID: id})
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
err = repos.Snapshots.AddMountPoint(ctx, nil, id.String(), "/data/"+id.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to add mount point: %v", err)
|
|
}
|
|
}
|
|
|
|
err = db.Close()
|
|
if err != nil {
|
|
t.Fatalf("failed to close database: %v", err)
|
|
}
|
|
|
|
sm := &SnapshotManager{
|
|
config: &config.Config{
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{testAgeRecipient},
|
|
},
|
|
fs: afero.NewOsFs(),
|
|
}
|
|
|
|
_, err = sm.cleanSnapshotDB(ctx, dbPath, "exported")
|
|
if err != nil {
|
|
t.Fatalf("failed to clean snapshot database: %v", err)
|
|
}
|
|
|
|
cleaned, err := database.New(ctx, dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to open cleaned database: %v", err)
|
|
}
|
|
|
|
defer func() { _ = cleaned.Close() }()
|
|
|
|
cleanedRepos := database.NewRepositories(cleaned)
|
|
|
|
for id, want := range map[string][]string{
|
|
"exported": {"/data/exported"},
|
|
"other": nil,
|
|
} {
|
|
got, err := cleanedRepos.Snapshots.GetMountPoints(ctx, id)
|
|
if err != nil {
|
|
t.Fatalf("failed to read mount points: %v", err)
|
|
}
|
|
|
|
if !slices.Equal(got, want) {
|
|
t.Errorf("mount points of %s: got %v, want %v", id, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Two creates of one snapshot name back to back start within one second,
|
|
// the resolution of the timestamp in a snapshot ID. See
|
|
// https://git.eeqj.de/sneak/vaultik/issues/270.
|
|
func TestCreateSnapshotWithNameTwiceBackToBack(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
|
|
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
defer func() { _ = db.Close() }()
|
|
|
|
sm := &SnapshotManager{repos: database.NewRepositories(db)}
|
|
|
|
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
|
|
if err != nil {
|
|
t.Fatalf("first create failed: %v", err)
|
|
}
|
|
|
|
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
|
|
if err != nil {
|
|
t.Fatalf("second create failed: %v", err)
|
|
}
|
|
|
|
if first == second {
|
|
t.Fatalf("both creates returned snapshot ID %s", first)
|
|
}
|
|
}
|
|
|
|
// Two creates of one snapshot name back to back list newest first. The
|
|
// start time is the moment in the ID, so the two cannot share a start
|
|
// time even when the clock crosses a second while the first is created.
|
|
// See https://git.eeqj.de/sneak/vaultik/issues/482.
|
|
func TestCreateSnapshotWithNameTwiceBackToBackListsNewestFirst(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
const created = 2
|
|
|
|
ctx := context.Background()
|
|
|
|
db, err := database.New(ctx, filepath.Join(t.TempDir(), "index.sqlite"))
|
|
if err != nil {
|
|
t.Fatalf("failed to create database: %v", err)
|
|
}
|
|
|
|
defer func() { _ = db.Close() }()
|
|
|
|
repos := database.NewRepositories(db)
|
|
sm := &SnapshotManager{repos: repos}
|
|
|
|
first, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
|
|
if err != nil {
|
|
t.Fatalf("first create failed: %v", err)
|
|
}
|
|
|
|
second, err := sm.CreateSnapshotWithName(ctx, "test-host", "data", "v", "g")
|
|
if err != nil {
|
|
t.Fatalf("second create failed: %v", err)
|
|
}
|
|
|
|
snapshots, err := repos.Snapshots.ListRecent(ctx, created)
|
|
if err != nil {
|
|
t.Fatalf("listing snapshots failed: %v", err)
|
|
}
|
|
|
|
if len(snapshots) != created {
|
|
t.Fatalf("listed %d snapshots, want %d", len(snapshots), created)
|
|
}
|
|
|
|
if !snapshots[0].StartedAt.After(snapshots[1].StartedAt) {
|
|
t.Fatalf("start times %v and %v are not in order",
|
|
snapshots[1].StartedAt, snapshots[0].StartedAt)
|
|
}
|
|
|
|
if snapshots[0].ID.String() != second || snapshots[1].ID.String() != first {
|
|
t.Fatalf("listed %s, %s; want %s, %s",
|
|
snapshots[0].ID, snapshots[1].ID, second, first)
|
|
}
|
|
}
|