Make prune and remote nuke fail when a delete fails (closes #290 )

prune, and the prune step of snapshot create --prune, counted a blob it could not delete and still exited 0. It now prints its summary or --json document and then returns an error, which replaces the "Failed to delete N blob(s)" line. remote nuke only warned when a snapshot's metadata could not be deleted, advised vaultik prune, which never removes metadata, and then reported the store empty. It now stops with an error, does not report the store empty, and says to run vaultik remote nuke --force again. A successful nuke no longer advises prune for blobs it deletes itself. The faultstore test double gains an OnDelete hook. Judgement call: a snapshot nuke cannot remove from the local index also fails the run, since its metadata is then left on the store. Model: opus-5-5
2026-10-13 02:04:43 +00:00
8 changed files with 38 additions and 255 deletions
@@ -349,11 +349,9 @@ the backup destination store. Blobs are NOT touched — deleting blobs
 requires reading every remaining remote manifest (the destination store
 may hold snapshots this host doesn't know about), which is what
 `vaultik prune` does. On success the command prints the exact `vaultik
-prune` invocation to run as a follow-up. An ID that names no snapshot in
-the local index or on the destination store is an error; with
-`--local-only`, the snapshot must be in the local index. Local row
-cleanup (files, chunks, blobs the snapshot was the last referrer for)
-runs automatically. If the destination store is unreachable, the local-DB
+prune` invocation to run as a follow-up. Local row cleanup (files,
+chunks, blobs the snapshot was the last referrer for) runs
+automatically. If the destination store is unreachable, the local-DB
 removal still completes and a warning is emitted; run `vaultik snapshot
 remove <snapshot-id>` again once the store is reachable to remove the
 snapshot's metadata from it (`vaultik prune` does not). To wipe everything
@@ -33,16 +33,6 @@ the tag exists and is exercised; what is left is merging `next` to
  --force` again, and a successful nuke no longer advises `prune` for
  blobs it deletes itself.

- 2026-10-13: Made `snapshot remove` fail for a snapshot that does not
-  exist ([issue #293](https://git.eeqj.de/sneak/vaultik/issues/293)).
-  The command never checked the ID: it reported the snapshot removed
-  from the local database and from the destination store and exited 0,
-  under `--json`, `--dry-run` and `--local-only` too. An ID that names
-  no snapshot in the local index or on the destination store is now an
-  error; with `--local-only`, the snapshot must be in the local index.
-  The command now reports a removal from either side only when one
-  happened, and `remote_removed` is true only when metadata was deleted.
-
 - 2026-10-12: Bound the local index to the destination an `s3.*` config
  names ([issue #298](https://git.eeqj.de/sneak/vaultik/issues/298)).
  Only `storage_url` was checked, so after `s3.bucket` or `s3.prefix`
@@ -255,12 +255,8 @@ manifest (the destination store may hold snapshots this host doesn't
 know about), which is what 'vaultik prune' does. On success the command
 prints the exact 'vaultik prune' invocation to run as a follow-up.

-An ID that names no snapshot in the local index or on the destination
-store is an error.
-
 Use --local-only to skip the remote half (e.g. when you want to forget a
-snapshot locally without touching the destination store); the snapshot
-must then be in the local index.
+snapshot locally without touching the destination store).

 If the remote is unreachable, the local-database removal still completes
 and a warning is emitted; run 'vaultik snapshot remove <snapshot-id>' again
@@ -100,26 +100,6 @@ func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) {
 	}
 }

-// TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice checks that
-// remote nuke, when the local index belongs to a different destination
-// than the configured one, fails without advising to run nuke again:
-// that run would fail the same way.
-func TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, _, out := backUpToFileDestination(ctx, t, storeDir)
-
-	v.Config.StorageURL = "file://" + filepath.Join(t.TempDir(), "elsewhere")
-
-	err := v.NukeRemote(true)
-
-	require.ErrorContains(t, err, "bound to a different backup destination")
-	assert.NotContains(t, out.String(), "vaultik remote nuke --force")
-}
-
 // TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved checks that a
 // snapshot remote nuke cannot remove from the local index fails the
 // run: nuke then leaves that snapshot's metadata on the destination
@@ -43,14 +43,7 @@ func (v *Vaultik) NukeRemote(force bool) error {
 		return errNukeRequiresForce
 	}

-	// Checked before anything is deleted: running nuke again would fail
-	// the same way, so the advice below must not follow this error.
-	err := v.EnsureStorageBinding()
-	if err != nil {
-		return err
-	}
-
-	err = v.emptyDestinationStore()
+	err := v.emptyDestinationStore()
 	if err != nil {
 		v.UI.Warningf("Run '%s' again to delete what is left on the "+
 			"backup destination store.", nukeCommandHint)
@@ -93,6 +93,6 @@ func TestCreateSnapshotPrune_FailedBlobDeleteReturnsError(t *testing.T) {
 		Prune: true,
 	})

-	require.ErrorContains(t, err, "1 blob(s) could not be deleted")
+	require.Error(t, err, "the prune step must fail the run")
 	assert.FileExists(t, orphan)
 }
@@ -3,21 +3,17 @@ package vaultik_test
 import (
 	"bytes"
 	"context"
-	"database/sql"
 	"io"
 	"strings"
 	"sync"
 	"testing"
-	"time"

 	"github.com/klauspost/compress/zstd"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/types"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )

@@ -218,37 +214,6 @@ func addBlob(t *testing.T, store *testStorer, hash string) {
 	require.NoError(t, err)
 }

-// newLocalIndex returns an in-memory local index holding a completed
-// snapshot row for each of ids.
-func newLocalIndex(t *testing.T, ids ...string) *database.Repositories {
-	t.Helper()
-
-	ctx := context.Background()
-
-	db, err := database.New(ctx, ":memory:")
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	repos := database.NewRepositories(db)
-
-	for _, id := range ids {
-		completedAt := time.Now().UTC()
-		snap := &database.Snapshot{
-			ID:          types.SnapshotID(id),
-			Hostname:    snapHostname,
-			StartedAt:   completedAt,
-			CompletedAt: &completedAt,
-		}
-
-		err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
-			return repos.Snapshots.Create(ctx, tx, snap)
-		})
-		require.NoError(t, err)
-	}
-
-	return repos
-}
-
 // ============================================================================
 // Unit Tests for RemoveSnapshot
 // ============================================================================
@@ -268,7 +233,6 @@ func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) {
 	addBlob(t, store, blobA)

 	tv := vaultik.NewForTesting(store)
-	tv.Repositories = newLocalIndex(t, "snapshot-001")

 	opts := &vaultik.RemoveOptions{Force: true, LocalOnly: true}
 	result, err := tv.RemoveSnapshot("snapshot-001", opts)
@@ -304,7 +268,6 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) {
 	addBlob(t, store, blobShared)

 	tv := vaultik.NewForTesting(store)
-	tv.Repositories = newLocalIndex(t, "snapshot-001", "snapshot-002")

 	opts := &vaultik.RemoveOptions{Force: true}
 	result, err := tv.RemoveSnapshot("snapshot-001", opts)
@@ -353,84 +316,6 @@ func TestRemoveSnapshot_DryRun(t *testing.T) {
 	assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
 }

-// TestRemoveSnapshot_UnknownIDIsError checks that an ID naming no
-// snapshot in the local index or on the destination store is an error
-// under every option, and that nothing is reported as removed.
-func TestRemoveSnapshot_UnknownIDIsError(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	for name, opts := range map[string]vaultik.RemoveOptions{
-		"force":      {Force: true},
-		"json":       {JSON: true},
-		"dry-run":    {Force: true, DryRun: true},
-		"local-only": {Force: true, LocalOnly: true},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			store := newTestStorer()
-			addManifest(t, store, "snapshot-001", nil)
-
-			tv := vaultik.NewForTesting(store)
-			tv.Repositories = newLocalIndex(t, "snapshot-001")
-
-			result, err := tv.RemoveSnapshot("snapshot-999", &opts)
-
-			require.ErrorContains(t, err, "snapshot not found")
-			assert.False(t, result.RemoteRemoved)
-			assert.Empty(t, tv.Stdout.String())
-			assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
-		})
-	}
-}
-
-// TestRemoveSnapshot_NoMetadataOnDestination checks that removing a
-// snapshot whose metadata is not on the destination store does not
-// report metadata removed from it.
-func TestRemoveSnapshot_NoMetadataOnDestination(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	tv := vaultik.NewForTesting(newTestStorer())
-	tv.Repositories = newLocalIndex(t, "snapshot-001")
-
-	result, err := tv.RemoveSnapshot("snapshot-001",
-		&vaultik.RemoveOptions{Force: true})
-
-	require.NoError(t, err)
-	assert.False(t, result.RemoteRemoved)
-
-	out := tv.Stdout.String()
-	assert.Contains(t, out, "Removed snapshot 'snapshot-001' from local database")
-	assert.NotContains(t, out, "Removed snapshot metadata from remote storage")
-}
-
-// TestRemoveSnapshot_NotInLocalIndex checks that removing a snapshot
-// only the destination store holds removes its metadata there without
-// reporting a removal from the local database.
-func TestRemoveSnapshot_NotInLocalIndex(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	store := newTestStorer()
-	addManifest(t, store, "snapshot-001", nil)
-
-	tv := vaultik.NewForTesting(store)
-	tv.Repositories = newLocalIndex(t)
-
-	result, err := tv.RemoveSnapshot("snapshot-001",
-		&vaultik.RemoveOptions{Force: true})
-
-	require.NoError(t, err)
-	assert.True(t, result.RemoteRemoved)
-	assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
-
-	out := tv.Stdout.String()
-	assert.NotContains(t, out, "from local database")
-	assert.Contains(t, out, "Removed snapshot metadata from remote storage")
-}
-
 func TestRemoveAllSnapshots_RequiresForce(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -23,7 +23,6 @@ var (
 	errNoSnapshotsInConfig  = errors.New("no snapshots configured")
 	errSnapshotVerifyFailed = errors.New("verification failed")
 	errRemoveAllNeedsForce  = errors.New("--all requires --force")
-	errSnapshotNotFound     = errors.New("snapshot not found")
 	errSnapshotRemoveFailed = errors.New("snapshot(s) could not be removed")
 	errInvalidTableName     = errors.New("invalid table name")
 	errNoAgeRecipients      = errors.New(
@@ -645,7 +644,7 @@ func (v *Vaultik) confirmAndExecutePurge(
 				"snapshot_id", snapshotID, "error", err)
 		}

-		_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
+		err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
 		if err != nil {
 			return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err)
 		}
@@ -1155,11 +1154,6 @@ func (v *Vaultik) RemoveSnapshot(
 		return result, err
 	}

-	inLocalIndex, err := v.findSnapshotToRemove(snapshotID, opts)
-	if err != nil {
-		return result, err
-	}
-
 	if opts.DryRun {
 		return v.removeSnapshotDryRun(result, snapshotID, opts)
 	}
@@ -1168,13 +1162,11 @@ func (v *Vaultik) RemoveSnapshot(
 		return result, nil
 	}

-	if inLocalIndex {
-		log.Info("Removing snapshot from local database", "snapshot_id", snapshotID)
+	log.Info("Removing snapshot from local database", "snapshot_id", snapshotID)

-		err = v.deleteSnapshotFromLocalDB(snapshotID)
-		if err != nil {
-			return result, fmt.Errorf("removing from local database: %w", err)
-		}
+	err = v.deleteSnapshotFromLocalDB(snapshotID)
+	if err != nil {
+		return result, fmt.Errorf("removing from local database: %w", err)
 	}

 	if !opts.LocalOnly {
@@ -1193,11 +1185,9 @@ func (v *Vaultik) RemoveSnapshot(
 		return result, v.outputRemoveJSON(result)
 	}

-	if inLocalIndex {
-		v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID)
-	}
+	v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID)

-	if result.RemoteRemoved {
+	if !opts.LocalOnly && result.RemoteRemoved {
 		v.printlnStdout("Removed snapshot metadata from remote storage")
 		v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. "+
 			"Run '%s' to delete any blobs no longer referenced by any "+
@@ -1207,45 +1197,6 @@ func (v *Vaultik) RemoveSnapshot(
 	return result, nil
 }

-// findSnapshotToRemove reports whether the local index has a row for
-// snapshotID, and returns errSnapshotNotFound when there is nothing to
-// remove: no row and, unless LocalOnly is set, no metadata on the
-// destination store either. The destination store is consulted only
-// when the local index has no row, so a snapshot in the local index can
-// still be removed from it while the destination store is unreachable.
-func (v *Vaultik) findSnapshotToRemove(
-	snapshotID string, opts *RemoveOptions,
-) (bool, error) {
-	if v.Repositories != nil {
-		snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID)
-		if err != nil {
-			return false, fmt.Errorf("looking up snapshot in local index: %w", err)
-		}
-
-		if snap != nil {
-			return true, nil
-		}
-	}
-
-	if opts.LocalOnly {
-		return false, fmt.Errorf("%w in the local index: %s",
-			errSnapshotNotFound, snapshotID)
-	}
-
-	objects, err := v.remoteSnapshotObjects(snapshot.RemoteSnapshotKey(snapshotID))
-	if err != nil {
-		return false, fmt.Errorf("looking up snapshot on destination store: %w", err)
-	}
-
-	if len(objects) == 0 {
-		return false, fmt.Errorf(
-			"%w in the local index or on the destination store: %s",
-			errSnapshotNotFound, snapshotID)
-	}
-
-	return false, nil
-}
-
 // removeSnapshotDryRun reports what RemoveSnapshot would do without
 // making changes.
 func (v *Vaultik) removeSnapshotDryRun(
@@ -1305,14 +1256,14 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
 // didn't finish and to run `vaultik snapshot remove` for the snapshot
 // again once the destination store is reachable (`vaultik prune` never
 // removes snapshot metadata). Returns true when the remote removal
-// succeeded and deleted something.
+// succeeded.
 func (v *Vaultik) removeSnapshotRemote(snapshotID string, opts *RemoveOptions) bool {
 	log.Info("Removing snapshot metadata from remote storage",
 		"snapshot_id", snapshotID)

 	remoteKey := snapshot.RemoteSnapshotKey(snapshotID)

-	deleted, err := v.deleteRemoteSnapshotByKey(remoteKey)
+	err := v.deleteRemoteSnapshotByKey(remoteKey)
 	if err != nil {
 		log.Warn("Could not remove snapshot metadata from remote storage; "+
 			"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
@@ -1330,7 +1281,7 @@ func (v *Vaultik) removeSnapshotRemote(snapshotID string, opts *RemoveOptions) b
 		return false
 	}

-	return deleted > 0
+	return true
 }

 // RemoveAllSnapshots removes every snapshot known to the local
@@ -1619,7 +1570,7 @@ func (v *Vaultik) removeAllLocalSnapshots(
 		}

 		if !opts.LocalOnly {
-			_, err = v.deleteRemoteSnapshotByKey(
+			err = v.deleteRemoteSnapshotByKey(
 				snapshot.RemoteSnapshotKey(snapshotID))
 			if err != nil {
 				log.Warn("Failed to remove snapshot metadata from remote",
@@ -1645,7 +1596,7 @@ func (v *Vaultik) removeAllOrphanRemotes(
 	for _, key := range orphanRemoteKeys {
 		log.Info("Removing orphan remote snapshot", "remote_key", key)

-		_, err := v.deleteRemoteSnapshotByKey(key)
+		err := v.deleteRemoteSnapshotByKey(key)
 		if err != nil {
 			log.Warn("Failed to remove orphan from remote",
 				"remote_key", key, "error", err)
@@ -1692,44 +1643,34 @@ func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error {
 }

 // deleteRemoteSnapshotByKey removes everything under
-// metadata/<remoteKey>/ on the destination store and returns how many
-// objects it deleted. The argument is a remote key (double-SHA256
-// derivation), not a human snapshot ID; callers that have a human ID
-// must hash via snapshot.RemoteSnapshotKey first.
-func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) (int, error) {
-	objectsToDelete, err := v.remoteSnapshotObjects(remoteKey)
-	if err != nil {
-		return 0, err
+// metadata/<remoteKey>/ on the destination store. The argument is a
+// remote key (double-SHA256 derivation), not a human snapshot ID;
+// callers that have a human ID must hash via snapshot.RemoteSnapshotKey
+// first.
+func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) error {
+	prefix := fmt.Sprintf("metadata/%s/", remoteKey)
+	objectCh := v.Storage.ListStream(v.ctx, prefix)
+
+	var objectsToDelete []string
+
+	for object := range objectCh {
+		if object.Err != nil {
+			return fmt.Errorf("listing objects: %w", object.Err)
+		}
+
+		objectsToDelete = append(objectsToDelete, object.Key)
 	}

-	for i, key := range objectsToDelete {
-		err = v.Storage.Delete(v.ctx, key)
+	for _, key := range objectsToDelete {
+		err := v.Storage.Delete(v.ctx, key)
 		if err != nil {
-			return i, fmt.Errorf("removing %s: %w", key, err)
+			return fmt.Errorf("removing %s: %w", key, err)
 		}

 		log.Debug("Deleted remote object", "key", key)
 	}

-	return len(objectsToDelete), nil
-}
-
-// remoteSnapshotObjects lists the keys of every object under
-// metadata/<remoteKey>/ on the destination store.
-func (v *Vaultik) remoteSnapshotObjects(remoteKey string) ([]string, error) {
-	prefix := fmt.Sprintf("metadata/%s/", remoteKey)
-
-	var keys []string
-
-	for object := range v.Storage.ListStream(v.ctx, prefix) {
-		if object.Err != nil {
-			return nil, fmt.Errorf("listing objects: %w", object.Err)
-		}
-
-		keys = append(keys, object.Key)
-	}
-
-	return keys, nil
+	return nil
 }

 // outputRemoveJSON outputs the removal result as JSON