diff --git a/README.md b/README.md index 784afca..387cdd3 100644 --- a/README.md +++ b/README.md @@ -413,11 +413,12 @@ under `blobs/` whose name is not a blob hash is not counted as a blob, since `prune` never deletes it. * `--json`: Output as JSON -**`remote nuke`**: Delete every snapshot's metadata and every blob from the -backup destination store, leaving the bucket prefix empty. Destructive and -irreversible. This is the single supported way to wipe the entire -destination store. If anything cannot be deleted, the command exits -non-zero; running it again deletes what is left. +**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on +the backup destination store, whatever its name, leaving the bucket prefix +empty. Destructive and irreversible. This is the single supported way to +wipe the entire destination store. If a listing afterwards still finds +anything there, the command names each object left and exits non-zero; +running it again deletes what is left. * `--force`: Required to confirm destruction. **`database delete`**: Delete the local SQLite state database file diff --git a/internal/vaultik/nuke_remote_test.go b/internal/vaultik/nuke_remote_test.go index 701b0a5..04addda 100644 --- a/internal/vaultik/nuke_remote_test.go +++ b/internal/vaultik/nuke_remote_test.go @@ -18,7 +18,8 @@ import ( // TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file // under a file:// destination, including the `.partial` files uploads // killed part-way leave next to a snapshot's metadata and next to where a -// blob would have been. +// blob would have been, a snapshot's database whose manifest was never +// uploaded, and a file under blobs/ whose name is not a blob hash. func TestNukeRemoteLeavesNoFiles(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() @@ -38,6 +39,9 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) { "db.zst.age-123456.partial"), filepath.Join(storeDir, "blobs", hash[:2], hash[2:4], hash+"-123456.partial"), + filepath.Join(storeDir, "metadata", + snapshot.RemoteSnapshotKey("no-manifest"), "db.zst.age"), + filepath.Join(storeDir, "blobs", "0b", "01", ".DS_Store"), } for _, leftover := range leftovers { @@ -100,6 +104,34 @@ func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) { } } +// TestNukeRemoteNamesWhatIsLeft checks that remote nuke, when an object +// it does not recognise cannot be deleted, does not report the store +// empty but names that object and fails. +func TestNukeRemoteNamesWhatIsLeft(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + ctx := context.Background() + storeDir := filepath.Join(t.TempDir(), "store") + v, _, out := backUpToFileDestination(ctx, t, storeDir) + + const leftKey = "blobs/0b/01/.DS_Store" + + leftPath := filepath.Join(storeDir, filepath.FromSlash(leftKey)) + require.NoError(t, os.MkdirAll(filepath.Dir(leftPath), 0o750)) + require.NoError(t, os.WriteFile(leftPath, []byte("finder"), 0o600)) + + store := faultstore.New(v.Storage) + store.OnDelete = func(key string) bool { return key == leftKey } + v.Storage = store + + require.Error(t, v.NukeRemote(true)) + + assert.Contains(t, out.String(), leftKey) + assert.NotContains(t, out.String(), "now empty") + assert.Contains(t, out.String(), "vaultik remote nuke --force") +} + // TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice checks that // remote nuke, when the local index belongs to a different destination // than the configured one, fails without advising to run nuke again: diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 7df20e1..d32f68c 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -22,6 +22,11 @@ var errNukeRequiresForce = errors.New( // errBlobDeleteFailed reports unreferenced blobs prune could not delete. var errBlobDeleteFailed = errors.New("blob(s) could not be deleted") +// errObjectsLeft reports objects remote nuke still finds on the backup +// destination store after deleting. +var errObjectsLeft = errors.New( + "object(s) still on the backup destination store") + // nukeCommandHint is the command suggested when a remote nuke could not // delete everything: running it again deletes what is left, which // `vaultik prune` does not do for snapshot metadata. @@ -35,10 +40,11 @@ const metadataDirName = "metadata" // (blobs///). const blobKeyParts = 4 -// NukeRemote deletes every snapshot's metadata and every blob from remote -// storage, along with any object an upload cut off part-way left under a -// temporary `.partial` name. After this returns successfully the bucket -// prefix is empty and the next backup starts from scratch. +// NukeRemote deletes every object under metadata/ and blobs/ in remote +// storage, whatever its name, along with any object an upload cut off +// part-way left under a temporary `.partial` name. It returns +// successfully only when a listing afterwards finds nothing left there, +// and the next backup then starts from scratch. // // Refuses to run unless force is true. The caller is responsible for // confirming with the user. @@ -68,7 +74,8 @@ func (v *Vaultik) NukeRemote(force bool) error { } // emptyDestinationStore does the deleting for NukeRemote and stops at -// the first step that fails. +// the first step that fails. It also fails when objects are still +// listed at the end, naming each one. func (v *Vaultik) emptyDestinationStore() error { v.UI.Beginf("Removing all snapshot metadata from backup destination store.") @@ -77,25 +84,62 @@ func (v *Vaultik) emptyDestinationStore() error { return fmt.Errorf("removing all snapshots: %w", err) } - v.UI.Beginf("Removing any blobs still present in backup destination store.") + v.UI.Beginf("Removing everything still present in backup destination store.") - err = v.PruneBlobs(&PruneOptions{Force: true}) - if err != nil { - return fmt.Errorf("pruning blobs: %w", err) + var left []string + + for _, prefix := range []string{"metadata/", "blobs/"} { + keys, err := v.deleteEverythingUnder(prefix) + if err != nil { + return err + } + + left = append(left, keys...) } - // The file and rclone listings skip `.partial` objects, so the two - // steps above never delete them. - for _, prefix := range []string{"metadata/", "blobs/"} { - err = v.Storage.DeletePartialUploads(v.ctx, prefix) - if err != nil { - return fmt.Errorf("deleting partial uploads: %w", err) - } + for _, key := range left { + v.UI.Warningf("Still on the backup destination store: %s", key) + } + + if len(left) > 0 { + return fmt.Errorf("%d %w", len(left), errObjectsLeft) } return nil } +// deleteEverythingUnder deletes every object under prefix, whatever its +// name, and returns the keys a listing afterwards still finds. A delete +// that fails is logged and the others still run; that object then +// shows up in the listing afterwards. +func (v *Vaultik) deleteEverythingUnder(prefix string) ([]string, error) { + keys, err := v.Storage.List(v.ctx, prefix) + if err != nil { + return nil, fmt.Errorf("listing %s: %w", prefix, err) + } + + for _, key := range keys { + err = v.Storage.Delete(v.ctx, key) + if err != nil { + log.Error("Failed to delete object", "key", key, "error", err) + } + } + + // The file and rclone listings skip `.partial` objects, so the + // deletes above never reach them. + err = v.Storage.DeletePartialUploads(v.ctx, prefix) + if err != nil { + return nil, fmt.Errorf("deleting partial uploads: %w", err) + } + + left, err := v.Storage.List(v.ctx, prefix) + if err != nil { + return nil, fmt.Errorf("listing %s: %w", prefix, err) + } + + return left, nil +} + // PruneBlobsResult contains the result of a blob prune operation // //nolint:tagliatelle // snake_case is the established JSON output format