vaultik/internal/vaultik/nuke_remote_test.go

package vaultik_test

import (
	"context"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sneak.berlin/go/vaultik/internal/log"
	"sneak.berlin/go/vaultik/internal/snapshot"
	"sneak.berlin/go/vaultik/internal/storage/faultstore"
)

// TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file
// under a file:// destination, including the `.partial` files uploads
// killed part-way leave next to a snapshot's metadata and next to where a
// blob would have been, a snapshot's database whose manifest was never
// uploaded, and a file under blobs/ whose name is not a blob hash.
func TestNukeRemoteLeavesNoFiles(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	ctx := context.Background()
	storeDir := filepath.Join(t.TempDir(), "store")
	v, repos, out := backUpToFileDestination(ctx, t, storeDir)

	snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
	require.NoError(t, err)
	require.Len(t, snapshots, 1)

	snapshotKey := snapshot.RemoteSnapshotKey(snapshots[0].ID.String())
	hash := testBlobHashA
	leftovers := []string{
		filepath.Join(storeDir, "metadata", snapshotKey,
			"db.zst.age-123456.partial"),
		filepath.Join(storeDir, "blobs", hash[:2], hash[2:4],
			hash+"-123456.partial"),
		filepath.Join(storeDir, "metadata",
			snapshot.RemoteSnapshotKey("no-manifest"), "db.zst.age"),
		filepath.Join(storeDir, "blobs", "0b", "01", ".DS_Store"),
	}

	for _, leftover := range leftovers {
		require.NoError(t, os.MkdirAll(filepath.Dir(leftover), 0o750))
		require.NoError(t, os.WriteFile(leftover, []byte("half an upload"), 0o600))
	}

	require.NoError(t, v.NukeRemote(true))

	var files []string

	err = filepath.WalkDir(storeDir,
		func(path string, entry fs.DirEntry, err error) error {
			if err != nil {
				return err
			}

			if !entry.IsDir() {
				files = append(files, path)
			}

			return nil
		})
	require.NoError(t, err)
	assert.Empty(t, files)

	// Nuke deletes the blobs itself, so advice to run prune is wrong.
	assert.Contains(t, out.String(), "Backup destination store is now empty.")
	assert.NotContains(t, out.String(), "vaultik prune")
}

// TestNukeRemoteFailsWhenADeleteFails checks that remote nuke returns an
// error when a snapshot's metadata or a blob cannot be deleted, does not
// report the store empty, and advises running nuke again: prune deletes
// no snapshot metadata that has its manifest.
func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	for _, refusedPrefix := range []string{"metadata/", "blobs/"} {
		t.Run(refusedPrefix, func(t *testing.T) {
			t.Parallel()

			ctx := context.Background()
			storeDir := filepath.Join(t.TempDir(), "store")
			v, _, out := backUpToFileDestination(ctx, t, storeDir)

			store := faultstore.New(v.Storage)
			store.OnDelete = func(key string) bool {
				return strings.HasPrefix(key, refusedPrefix)
			}
			v.Storage = store

			require.Error(t, v.NukeRemote(true))

			assert.NotContains(t, out.String(), "now empty")
			assert.Contains(t, out.String(), "vaultik remote nuke --force")
			assert.NotContains(t, out.String(), "vaultik prune")
		})
	}
}

// TestNukeRemoteKeepsLocalSnapshotWhoseMetadataIsNotDeleted checks that
// remote nuke keeps a snapshot in the local index while its metadata
// cannot be deleted from the destination store: dropped from the local
// index, it would be left there as a remote-only snapshot without its
// name.
func TestNukeRemoteKeepsLocalSnapshotWhoseMetadataIsNotDeleted(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	ctx := context.Background()
	storeDir := filepath.Join(t.TempDir(), "store")
	v, repos, _ := backUpToFileDestination(ctx, t, storeDir)

	store := faultstore.New(v.Storage)
	store.OnDelete = func(key string) bool {
		return strings.HasPrefix(key, "metadata/")
	}
	v.Storage = store

	require.Error(t, v.NukeRemote(true))

	snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
	require.NoError(t, err)
	assert.Len(t, snapshots, 1)
}

// TestNukeRemoteNamesWhatIsLeft checks that remote nuke, when an object
// it does not recognise cannot be deleted, does not report the store
// empty but names that object and fails.
func TestNukeRemoteNamesWhatIsLeft(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	ctx := context.Background()
	storeDir := filepath.Join(t.TempDir(), "store")
	v, _, out := backUpToFileDestination(ctx, t, storeDir)

	const leftKey = "blobs/0b/01/.DS_Store"

	leftPath := filepath.Join(storeDir, filepath.FromSlash(leftKey))
	require.NoError(t, os.MkdirAll(filepath.Dir(leftPath), 0o750))
	require.NoError(t, os.WriteFile(leftPath, []byte("finder"), 0o600))

	store := faultstore.New(v.Storage)
	store.OnDelete = func(key string) bool { return key == leftKey }
	v.Storage = store

	require.Error(t, v.NukeRemote(true))

	assert.Contains(t, out.String(), leftKey)
	assert.NotContains(t, out.String(), "now empty")
	assert.Contains(t, out.String(), "vaultik remote nuke --force")
}

// TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice checks that
// remote nuke, when the local index belongs to a different destination
// than the configured one, fails without advising to run nuke again:
// that run would fail the same way.
func TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	ctx := context.Background()
	storeDir := filepath.Join(t.TempDir(), "store")
	v, _, out := backUpToFileDestination(ctx, t, storeDir)

	v.Config.StorageURL = "file://" + filepath.Join(t.TempDir(), "elsewhere")

	err := v.NukeRemote(true)

	require.ErrorContains(t, err, "bound to a different backup destination")
	assert.NotContains(t, out.String(), "vaultik remote nuke --force")
}

// TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved checks that a
// snapshot remote nuke cannot remove from the local index fails the
// run, and that nuke then does not report the destination store empty.
func TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	ctx := context.Background()
	storeDir := filepath.Join(t.TempDir(), "store")
	v, _, out := backUpToFileDestination(ctx, t, storeDir)

	_, err := v.DB.Conn().ExecContext(ctx, `
		CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
		BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
	require.NoError(t, err)

	require.Error(t, v.NukeRemote(true))

	assert.NotContains(t, out.String(), "now empty")
	assert.Contains(t, out.String(), "vaultik remote nuke --force")
}