package vaultik_test import ( "bytes" "context" "encoding/json" "os" "path/filepath" "strings" "testing" "time" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage/faultstore" "sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/vaultik" ) // TestPruneDuringBackupKeepsItsBlobs runs a prune from another host // sharing the destination store at the moment a backup has uploaded its // blobs and its db.zst.age but not yet its manifest, so no manifest lists // those blobs. The prune must leave the blobs and the db.zst.age, and the // backup must then restore. func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() fs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "src") storeDir := t.TempDir() restoreDir := filepath.Join(tempDir, "restored") dbPath := filepath.Join(tempDir, "index.sqlite") ctx := context.Background() cfg := faultTestConfig() testFiles := writeFaultSourceTree(t, fs, dataDir) inner, err := storage.NewFileStorer(storeDir) require.NoError(t, err) otherHostCfg := faultTestConfig() otherHostCfg.PruneGracePeriod = time.Hour otherHost := newReaderVaultik(ctx, otherHostCfg, inner, nil, fs) var ( pruned bool pruneErr error ) store := faultstore.New(inner) store.OnPut = func(key string) faultstore.PutAction { if strings.HasSuffix(key, "/manifest.json.zst") { pruned = true pruneErr = otherHost.PruneBlobs(&vaultik.PruneOptions{Force: true}) } return faultstore.PutNormal } db, err := database.New(ctx, dbPath) require.NoError(t, err) repos := database.NewRepositories(db) id := fullFaultBackup(ctx, t, fs, store, cfg, repos, dataDir, dbPath, "shared") require.NoError(t, db.Close()) require.True(t, pruned, "the prune must run before the manifest upload") require.NoError(t, pruneErr) v := newReaderVaultik(ctx, cfg, inner, nil, fs) require.NoError(t, v.Restore(&vaultik.RestoreOptions{ SnapshotID: id, TargetDir: restoreDir, Verify: true, }), "a prune during the backup must not delete the backup's blobs "+ "or its db.zst.age") assertRestoredTree(t, fs, restoreDir, testFiles) } // TestPruneKeepsUnreferencedBlobWithNoTime checks that a blob no // manifest lists, whose listing gives no modification time (as an S3 // listing entry without LastModified does), is kept and counted with // the blobs too new to delete: its age is unknown. func TestPruneKeepsUnreferencedBlobWithNoTime(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() env := newListEnv(t) // Written into the mock store's map directly rather than through // Put, so its listing gives no time. blobKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] + "/" + testBlobHashA env.store.data[blobKey] = []byte("blob") require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true})) assert.True(t, env.store.hasKey(blobKey), "a blob of unknown age must be kept") var result vaultik.PruneBlobsResult require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result)) assert.Equal(t, 0, result.BlobsDeleted) assert.Equal(t, 1, result.BlobsTooNew) } // TestPruneDeletesUnreferencedBlobOlderThanGracePeriod checks that the // grace period only delays deletion: a blob no manifest lists is deleted // once it is older than the grace period, and kept and counted until // then. func TestPruneDeletesUnreferencedBlobOlderThanGracePeriod(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() storeDir := t.TempDir() store, err := storage.NewFileStorer(storeDir) require.NoError(t, err) oldKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] + "/" + testBlobHashA newKey := "blobs/" + testBlobHashB[:2] + "/" + testBlobHashB[2:4] + "/" + testBlobHashB for _, key := range []string{oldKey, newKey} { require.NoError(t, store.Put(ctx, key, strings.NewReader("blob"))) } gracePeriod := time.Hour beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute) require.NoError(t, os.Chtimes(filepath.Join(storeDir, oldKey), beforeGracePeriod, beforeGracePeriod)) stdout := &bytes.Buffer{} v := &vaultik.Vaultik{ Config: &config.Config{PruneGracePeriod: gracePeriod}, Storage: store, Stdout: stdout, UI: ui.NewWithColor(stdout, false), } v.SetContext(ctx) require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true})) _, err = store.Stat(ctx, oldKey) require.ErrorIs(t, err, storage.ErrNotFound, "a blob older than the grace period must be deleted") _, err = store.Stat(ctx, newKey) require.NoError(t, err, "a blob newer than the grace period must be kept") var result vaultik.PruneBlobsResult require.NoError(t, json.Unmarshal(stdout.Bytes(), &result)) assert.Equal(t, 1, result.BlobsDeleted) assert.Equal(t, 1, result.BlobsTooNew) }