From ac7194617081737427e266d31b8c8ff7f7978455 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 21:26:39 +0000 Subject: [PATCH] Delete metadata directories a backup left without a manifest (closes #390) A backup that stopped between uploading db.zst.age and its manifest left that directory under metadata/ on the destination for good: prune deleted nothing there. The blob step of prune, which snapshot create --prune also runs, now deletes each directory under metadata/ that has no manifest.json.zst, that no local snapshot record claims, and all of whose objects are older than prune_grace_period; an object listed with no time keeps its directory, and only names that are remote keys are deleted. It shares prune's one listing of metadata/, follows the --force, prompt and --json rules of unreferenced blobs, reports the count (metadata_dirs_deleted, metadata_dirs_failed), and fails the run when a delete fails. Model: opus-5-5 --- README.md | 20 +- internal/cli/prune.go | 4 + internal/vaultik/prune.go | 281 ++++++++++++++++---- internal/vaultik/prune_metadata_dir_test.go | 181 +++++++++++++ internal/vaultik/snapshot.go | 5 +- 5 files changed, 430 insertions(+), 61 deletions(-) create mode 100644 internal/vaultik/prune_metadata_dir_test.go diff --git a/README.md b/README.md index 14a7bb1..db66b68 100644 --- a/README.md +++ b/README.md @@ -453,22 +453,29 @@ may belong to a backup still running, on this host or another sharing the destination store, that has not yet uploaded its manifest. The grace period does not protect a blob such a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so it can -be deleted. Step (3) reads all remote manifests — network cost scales with -the number of snapshots. +be deleted. Step (3) also deletes each directory under `metadata/` that has +no `manifest.json.zst`, as a backup leaves when it stops after uploading +`db.zst.age` but before its manifest, once every object in it is older than +`prune_grace_period` and its snapshot is not in the local index. An object the +listing gives no time for keeps its directory, and a directory whose name is +not a remote key is left alone. Step (3) reads all remote manifests — network +cost scales with the number of snapshots. `snapshot create --prune` runs the same cleanup automatically; this is the manual entry point for the same work. If the destination store cannot be listed, or the local index holds snapshots and the destination has no `vaultik-destination.txt` (an unplugged volume's empty mount point), prune fails and deletes nothing. `snapshot purge` does the same. If a -blob cannot be deleted, the command (and `snapshot create --prune`) -exits non-zero. +blob or such a directory cannot be deleted, the command (and +`snapshot create --prune`) exits non-zero. * `--force`: Skip confirmation prompt * `--json`: Output stats as JSON. Also skips the confirmation prompt, as `--force` does. A blob that cannot be deleted is counted in `blobs_failed`, and the document is printed before the non-zero exit. A blob kept for being newer than `prune_grace_period`, or because the listing of the destination gives no time for it, is counted in - `blobs_too_new` and not in `blobs_found`. + `blobs_too_new` and not in `blobs_found`. A directory under `metadata/` + with no manifest is counted in `metadata_dirs_deleted` once deleted, or + in `metadata_dirs_failed` if it cannot be. **`info`**: Display system configuration, storage settings, encryption recipients, and local database statistics. @@ -639,6 +646,9 @@ format does and does not protect. 3. List all blobs in storage 4. Delete any blob not in the referenced set that is older than `prune_grace_period` +5. Delete each directory under `metadata/` with no manifest whose objects + are all older than `prune_grace_period`, unless its snapshot is in the + local index ### chunking and deduplication diff --git a/internal/cli/prune.go b/internal/cli/prune.go index 77f38b6..66f2feb 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -26,6 +26,10 @@ another sharing the destination store, that has not yet uploaded its manifest. The grace period does not protect a blob such a backup reuses from an earlier run that stopped before its manifest: that blob keeps the time of its first upload, so it can be deleted. +Remote cleanup also deletes each directory under metadata/ with no +manifest, as a backup that stopped before its manifest leaves, once +all its objects are older than prune_grace_period and its snapshot is +not in the local index. Snapshot create --prune runs the same cleanup automatically; this command is the manual entry point for the same work (e.g. after a diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 56619d7..c21bf84 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -4,10 +4,12 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "time" "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/storage" ) @@ -24,6 +26,11 @@ var errNukeRequiresForce = errors.New( // errBlobDeleteFailed reports unreferenced blobs prune could not delete. var errBlobDeleteFailed = errors.New("blob(s) could not be deleted") +// errMetadataDirDeleteFailed reports directories under metadata/ with no +// manifest that prune could not delete. +var errMetadataDirDeleteFailed = errors.New( + "metadata director(ies) with no manifest could not be deleted") + // errObjectsLeft reports objects remote nuke still finds on the backup // destination store after deleting. var errObjectsLeft = errors.New( @@ -31,7 +38,8 @@ var errObjectsLeft = errors.New( // nukeCommandHint is the command suggested when a remote nuke could not // delete everything: running it again deletes what is left, which -// `vaultik prune` does not do for snapshot metadata. +// `vaultik prune` does not do for snapshot metadata that has its +// manifest. const nukeCommandHint = "vaultik remote nuke --force" // metadataDirName is the top-level remote directory holding snapshot @@ -169,6 +177,11 @@ type PruneBlobsResult struct { // newer than the prune grace period, or of unknown age. BlobsFound // does not include them. BlobsTooNew int `json:"blobs_too_new"` + // MetadataDirsDeleted counts the directories under metadata/ with no + // manifest that prune deleted, and MetadataDirsFailed those it could + // not delete. + MetadataDirsDeleted int `json:"metadata_dirs_deleted"` + MetadataDirsFailed int `json:"metadata_dirs_failed,omitempty"` } // Prune removes orphaned data from the local index database AND @@ -200,11 +213,17 @@ func (v *Vaultik) Prune(opts *PruneOptions) error { return v.PruneBlobs(opts) } -// PruneBlobs removes unreferenced blobs from storage +// PruneBlobs removes unreferenced blobs from storage, and each directory +// under metadata/ that a backup left with no manifest. func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { log.Info("Starting prune operation") - allBlobsReferenced, err := v.collectReferencedBlobs() + metadataObjects, err := v.listMetadataObjects() + if err != nil { + return err + } + + allBlobsReferenced, err := v.collectReferencedBlobs(metadataObjects) if err != nil { return err } @@ -217,6 +236,11 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs( allBlobs, allBlobsReferenced) + dirsWithoutManifest, err := v.findMetadataDirsWithoutManifest(metadataObjects) + if err != nil { + return err + } + result := &PruneBlobsResult{ BlobsFound: len(unreferencedBlobs), BlobsTooNew: tooNew, @@ -224,7 +248,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { v.reportBlobsTooNew(tooNew, opts) - if len(unreferencedBlobs) == 0 { + if len(unreferencedBlobs) == 0 && len(dirsWithoutManifest) == 0 { log.Info("No unreferenced blobs found") if opts.JSON { @@ -237,38 +261,16 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { } log.Info("Found unreferenced blobs", - "count", len(unreferencedBlobs), "total_size", ubytes(totalSize)) + "count", len(unreferencedBlobs), "total_size", ubytes(totalSize), + "metadata_dirs_without_manifest", len(dirsWithoutManifest)) - if opts.Force && !opts.JSON { - v.UI.Infof("Found %d unreferenced blob(s) totaling %s.", - len(unreferencedBlobs), ubytes(totalSize)) - } - - if !opts.Force && !opts.JSON { - // The count, the prompt and its answer bypass the UI: they are an - // interactive exchange the operator must see, so --quiet does not - // silence them. - v.stdoutf("Found %d unreferenced blob(s) totaling %s.\n", - len(unreferencedBlobs), ubytes(totalSize)) - v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) - - var confirm string - - _, err = v.scanStdin(&confirm) - if err != nil { - v.printlnStdout("Cancelled") - - return nil //nolint:nilerr // read failure means no confirmation - } - - if strings.ToLower(confirm) != "y" { - v.printlnStdout("Cancelled") - - return nil - } + if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize, + len(dirsWithoutManifest)) { + return nil } v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result) + v.deleteMetadataDirs(dirsWithoutManifest, result) if opts.JSON { err = v.outputPruneBlobsJSON(result) @@ -276,33 +278,132 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return err } } else { + v.reportPruneDeletions(result, len(dirsWithoutManifest)) + } + + var failures []error + + if result.BlobsFailed > 0 { + failures = append(failures, + fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed)) + } + + if result.MetadataDirsFailed > 0 { + failures = append(failures, fmt.Errorf("%d %w", + result.MetadataDirsFailed, errMetadataDirDeleteFailed)) + } + + return errors.Join(failures...) +} + +// confirmPrune states what prune found to delete and, unless --force or +// --json is given, asks the operator to confirm. It reports whether to +// go ahead. +func (v *Vaultik) confirmPrune( + opts *PruneOptions, blobCount int, blobsSize int64, dirCount int, +) bool { + if opts.JSON { + return true + } + + var found []string + + if blobCount > 0 { + found = append(found, fmt.Sprintf( + "Found %d unreferenced blob(s) totaling %s.", + blobCount, ubytes(blobsSize))) + } + + if dirCount > 0 { + found = append(found, fmt.Sprintf( + "Found %d metadata director(ies) left by backups that "+ + "stopped before their manifest.", dirCount)) + } + + if opts.Force { + for _, line := range found { + v.UI.Infof("%s", line) + } + + return true + } + + // The counts, the prompt and its answer bypass the UI: they are an + // interactive exchange the operator must see, so --quiet does not + // silence them. + for _, line := range found { + v.stdoutf("%s\n", line) + } + + switch { + case dirCount == 0: + v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", blobCount) + case blobCount == 0: + v.stdoutf("\nDelete %d metadata director(ies)? [y/N] ", dirCount) + default: + v.stdoutf("\nDelete %d unreferenced blob(s) and %d metadata "+ + "director(ies)? [y/N] ", blobCount, dirCount) + } + + var confirm string + + // A failure to read the answer means no confirmation. + _, err := v.scanStdin(&confirm) + if err != nil || strings.ToLower(confirm) != "y" { + v.printlnStdout("Cancelled") + + return false + } + + return true +} + +// reportPruneDeletions says how many blobs and metadata directories +// prune deleted, naming each kind only when it found some to delete. +func (v *Vaultik) reportPruneDeletions(result *PruneBlobsResult, dirCount int) { + if result.BlobsFound > 0 { v.UI.Infof("Deleted %d blob(s) totaling %s.", result.BlobsDeleted, ubytes(result.BytesFreed)) } - if result.BlobsFailed > 0 { - return fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed) + if dirCount > 0 { + v.UI.Infof("Deleted %d metadata director(ies) with no manifest.", + result.MetadataDirsDeleted) } - - return nil } -// collectReferencedBlobs downloads all manifests and returns the set of -// referenced blob hashes. +// listMetadataObjects lists every object under metadata/ on the +// destination store. +func (v *Vaultik) listMetadataObjects() ([]storage.ObjectInfo, error) { + log.Info("Listing remote snapshots") + + var objects []storage.ObjectInfo + + for object := range v.Storage.ListStream(v.ctx, "metadata/") { + if object.Err != nil { + return nil, fmt.Errorf("listing metadata objects: %w", object.Err) + } + + objects = append(objects, object) + } + + return objects, nil +} + +// collectReferencedBlobs downloads every manifest among the listed +// metadata objects and returns the set of referenced blob hashes. // // Every manifest must be read successfully. A manifest that cannot be // downloaded or decoded means its snapshot's blobs are unknown, so // treating them as unreferenced would let prune delete data a snapshot // still needs. Rather than risk that silent loss, any failure returns an // error naming the remote key and prune deletes nothing. -func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { - log.Info("Listing remote snapshots") - // IDs returned by listUniqueSnapshotIDs are remote keys (hashed +func (v *Vaultik) collectReferencedBlobs( + metadataObjects []storage.ObjectInfo, +) (map[string]bool, error) { + // IDs returned by uniqueSnapshotIDs are remote keys (hashed // subdirectories under metadata/), not human snapshot IDs. - remoteKeys, err := v.listUniqueSnapshotIDs() - if err != nil { - return nil, fmt.Errorf("listing snapshot keys: %w", err) - } + remoteKeys := uniqueSnapshotIDs(metadataObjects) log.Info("Found manifests in remote storage", "count", len(remoteKeys)) @@ -327,18 +428,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { return allBlobsReferenced, nil } -// listUniqueSnapshotIDs returns deduplicated snapshot IDs from remote metadata -func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { - objectCh := v.Storage.ListStream(v.ctx, "metadata/") +// uniqueSnapshotIDs returns deduplicated snapshot IDs from the listed +// metadata objects. +func uniqueSnapshotIDs(metadataObjects []storage.ObjectInfo) []string { seen := make(map[string]bool) var snapshotIDs []string - for object := range objectCh { - if object.Err != nil { - return nil, fmt.Errorf("listing metadata objects: %w", object.Err) - } - + for _, object := range metadataObjects { parts := strings.Split(object.Key, "/") if len(parts) >= minSnapshotIDParts && parts[0] == metadataDirName && parts[1] != "" { @@ -353,7 +450,7 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { } } - return snapshotIDs, nil + return snapshotIDs } // listAllRemoteBlobs returns a map of all blob hashes in remote storage @@ -427,6 +524,82 @@ func (v *Vaultik) findUnreferencedBlobs( return unreferenced, totalSize, tooNew } +// findMetadataDirsWithoutManifest returns the remote key of each +// directory under metadata/ that has no manifest.json.zst, such as one a +// backup leaves when its manifest upload fails after db.zst.age was +// uploaded. It leaves out a directory a local snapshot record claims, +// and one holding an object newer than the prune grace period, or of +// unknown age because the listing gave no time for it: a backup still +// running may be writing there. A name that is not a remote key is +// never returned. +func (v *Vaultik) findMetadataDirsWithoutManifest( + metadataObjects []storage.ObjectInfo, +) ([]string, error) { + localIDs, err := v.localSnapshotIDs() + if err != nil { + return nil, fmt.Errorf("listing local snapshots: %w", err) + } + + keep := make(map[string]bool, len(localIDs)) + for _, id := range localIDs { + keep[snapshot.RemoteSnapshotKey(id)] = true + } + + cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod) + listed := make(map[string]bool) + + for _, object := range metadataObjects { + parts := strings.Split(object.Key, "/") + if len(parts) < metadataKeyParts || !isBlobHash(parts[1]) { + continue + } + + remoteKey := parts[1] + listed[remoteKey] = true + + if parts[2] == "manifest.json.zst" || + object.ModTime.IsZero() || object.ModTime.After(cutoff) { + keep[remoteKey] = true + } + } + + var remoteKeys []string + + for remoteKey := range listed { + if !keep[remoteKey] { + remoteKeys = append(remoteKeys, remoteKey) + } + } + + sort.Strings(remoteKeys) + + return remoteKeys, nil +} + +// deleteMetadataDirs deletes everything in the given directories under +// metadata/ and counts in the result the directories deleted and those +// it could not delete. +func (v *Vaultik) deleteMetadataDirs( + remoteKeys []string, result *PruneBlobsResult, +) { + for _, remoteKey := range remoteKeys { + _, err := v.deleteRemoteSnapshotByKey(remoteKey) + if err != nil { + log.Error("Failed to delete metadata directory with no manifest", + "remote_key", remoteKey, "error", err) + + result.MetadataDirsFailed++ + + continue + } + + log.Info("Deleted metadata directory with no manifest", + "remote_key", remoteKey) + + result.MetadataDirsDeleted++ + } +} + // reportBlobsTooNew says how many unreferenced blobs prune kept for being // newer than the grace period or of unknown age. Under --json the // document carries the count instead. diff --git a/internal/vaultik/prune_metadata_dir_test.go b/internal/vaultik/prune_metadata_dir_test.go new file mode 100644 index 0000000..24d7f24 --- /dev/null +++ b/internal/vaultik/prune_metadata_dir_test.go @@ -0,0 +1,181 @@ +package vaultik_test + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/config" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/snapshot" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/storage/faultstore" + "sneak.berlin/go/vaultik/internal/ui" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// metadataObjectKey returns the key of the named file in the metadata +// directory of the snapshot with the given human ID. +func metadataObjectKey(snapshotID, name string) string { + return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/" + name +} + +// TestPruneDeletesMetadataDirWithoutManifestOlderThanGracePeriod checks +// that prune deletes a directory under metadata/ holding only +// db.zst.age, as a backup leaves when its manifest upload fails, once it +// is older than the grace period, and keeps one newer than that. A +// directory with a manifest is never deleted, however old. +func TestPruneDeletesMetadataDirWithoutManifestOlderThanGracePeriod( + t *testing.T, +) { + log.Initialize(log.Config{}) + t.Parallel() + + ctx := context.Background() + storeDir := t.TempDir() + + store, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + const ( + oldID = "testhost_home_2026-01-01T00:00:00Z" + newID = "testhost_home_2026-01-02T00:00:00Z" + completeID = "testhost_home_2026-01-03T00:00:00Z" + ) + + oldDB := metadataObjectKey(oldID, "db.zst.age") + newDB := metadataObjectKey(newID, "db.zst.age") + completeDB := metadataObjectKey(completeID, "db.zst.age") + completeManifest := metadataObjectKey(completeID, "manifest.json.zst") + + manifest, err := snapshot.EncodeManifest(&snapshot.Manifest{ + SnapshotID: snapshot.RemoteSnapshotKey(completeID), + }, 3) + require.NoError(t, err) + + for _, key := range []string{oldDB, newDB, completeDB} { + require.NoError(t, store.Put(ctx, key, strings.NewReader("db"))) + } + + require.NoError(t, store.Put(ctx, completeManifest, + bytes.NewReader(manifest))) + + gracePeriod := time.Hour + beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute) + + for _, key := range []string{oldDB, completeDB, completeManifest} { + require.NoError(t, os.Chtimes(filepath.Join(storeDir, key), + beforeGracePeriod, beforeGracePeriod)) + } + + stdout := &bytes.Buffer{} + v := &vaultik.Vaultik{ + Config: &config.Config{PruneGracePeriod: gracePeriod}, + Storage: store, + Stdout: stdout, + UI: ui.NewWithColor(stdout, false), + } + v.SetContext(ctx) + + require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true})) + + _, err = store.Stat(ctx, oldDB) + require.ErrorIs(t, err, storage.ErrNotFound, + "a directory with no manifest older than the grace period "+ + "must be deleted") + + for _, key := range []string{newDB, completeDB, completeManifest} { + _, err = store.Stat(ctx, key) + require.NoError(t, err, "%s must be kept", key) + } + + var result vaultik.PruneBlobsResult + + require.NoError(t, json.Unmarshal(stdout.Bytes(), &result)) + assert.Equal(t, 1, result.MetadataDirsDeleted) +} + +// TestPruneKeepsMetadataDirClaimedOrOfUnknownAge checks that prune keeps +// a directory under metadata/ with no manifest when a local snapshot +// record claims it, or when the listing gives no time for an object in +// it. A third such directory is deleted and reported, although there are +// no unreferenced blobs to delete. +func TestPruneKeepsMetadataDirClaimedOrOfUnknownAge(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + ctx := context.Background() + env := newListEnv(t) + + const claimedID = "testhost_home_2026-01-01T00:00:00Z" + + env.addLocal(t, claimedID, time.Now().Add(-time.Hour)) + + claimedDB := metadataObjectKey(claimedID, "db.zst.age") + unknownAgeDB := metadataObjectKey("testhost_home_2026-01-02T00:00:00Z", + "db.zst.age") + leftDB := metadataObjectKey("testhost_home_2026-01-03T00:00:00Z", + "db.zst.age") + + for _, key := range []string{claimedDB, leftDB} { + require.NoError(t, env.store.Put(ctx, key, strings.NewReader("db"))) + } + + // Written into the mock store's map directly rather than through + // Put, so its listing gives no time. + env.store.data[unknownAgeDB] = []byte("db") + + require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{Force: true})) + + assert.True(t, env.store.hasKey(claimedDB), + "a directory a local snapshot record claims must be kept") + assert.True(t, env.store.hasKey(unknownAgeDB), + "a directory holding an object of unknown age must be kept") + assert.False(t, env.store.hasKey(leftDB), + "a directory with no manifest, unclaimed and old, must be deleted") + assert.Contains(t, env.stdout.String(), + "Deleted 1 metadata director(ies) with no manifest.") + assert.NotContains(t, env.stdout.String(), + "No unreferenced blobs to remove.") +} + +// TestPruneFailedMetadataDirDeleteReturnsError checks that prune --json +// counts a directory under metadata/ with no manifest that it cannot +// delete, and then returns an error, so the command exits non-zero. +func TestPruneFailedMetadataDirDeleteReturnsError(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + env := newListEnv(t) + + leftDB := metadataObjectKey("testhost_home_2026-01-01T00:00:00Z", + "db.zst.age") + require.NoError(t, env.store.Put(context.Background(), leftDB, + strings.NewReader("db"))) + + store := faultstore.New(env.store) + store.OnDelete = func(key string) bool { + return strings.HasPrefix(key, "metadata/") + } + env.v.Storage = store + + err := env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true}) + + require.ErrorContains(t, err, + "1 metadata director(ies) with no manifest could not be deleted") + assert.True(t, env.store.hasKey(leftDB)) + + var result vaultik.PruneBlobsResult + + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result), + "stdout must hold the JSON document, got:\n%s", env.stdout.String()) + assert.Equal(t, 1, result.MetadataDirsFailed) + assert.Equal(t, 0, result.MetadataDirsDeleted) +} diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 806495f..ba6ff4b 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -1301,7 +1301,7 @@ const pruneCommandHint = "vaultik prune" // snapshotRemoveCommandHint is the command suggested, with the // snapshot's ID, when a remove could not reach the destination store: // running it again removes the snapshot's metadata there, which -// `vaultik prune` never does. +// `vaultik prune` never does while the manifest is there. const snapshotRemoveCommandHint = "vaultik snapshot remove" // RemoveSnapshot removes a snapshot from the local index database and, @@ -1504,7 +1504,8 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) // the local index, its removal there has already happened, so a failure // is only a warning telling the user to run `vaultik snapshot remove` // for the snapshot again once the destination store is reachable -// (`vaultik prune` never removes snapshot metadata). +// (`vaultik prune` never removes snapshot metadata that has its +// manifest). func (v *Vaultik) removeSnapshotRemote( snapshotID string, inLocalIndex bool, markerErr error, opts *RemoveOptions,