Delete metadata directories a backup left without a manifest (closes #390)

A backup that stopped between uploading db.zst.age and its manifest
left that directory under metadata/ on the destination for good:
prune deleted nothing there.

The blob step of prune, which snapshot create --prune also runs, now
deletes each directory under metadata/ that has no manifest.json.zst,
that no local snapshot record claims, and all of whose objects are
older than prune_grace_period; an object listed with no time keeps its
directory, and only names that are remote keys are deleted. The prompt
and the --force output name each such directory, the --json result
counts them (metadata_dirs_deleted, metadata_dirs_failed), and a failed
delete fails the run. Every description of the grace period now
covers these directories.

Model: opus-5-5

internal/vaultik/prune.go

@@ -4,10 +4,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/storage"
 )
 
@@ -24,6 +26,11 @@ var errNukeRequiresForce = errors.New(
 // errBlobDeleteFailed reports unreferenced blobs prune could not delete.
 var errBlobDeleteFailed = errors.New("blob(s) could not be deleted")
 
+// errMetadataDirDeleteFailed reports directories under metadata/ with no
+// manifest that prune could not delete.
+var errMetadataDirDeleteFailed = errors.New(
+	"metadata director(ies) with no manifest could not be deleted")
+
 // errObjectsLeft reports objects remote nuke still finds on the backup
 // destination store after deleting.
 var errObjectsLeft = errors.New(
@@ -31,7 +38,8 @@ var errObjectsLeft = errors.New(
 
 // nukeCommandHint is the command suggested when a remote nuke could not
 // delete everything: running it again deletes what is left, which
-// `vaultik prune` does not do for snapshot metadata.
+// `vaultik prune` does not do for snapshot metadata that has its
+// manifest.
 const nukeCommandHint = "vaultik remote nuke --force"
 
 // metadataDirName is the top-level remote directory holding snapshot
@@ -169,14 +177,20 @@ type PruneBlobsResult struct {
 	// newer than the prune grace period, or of unknown age. BlobsFound
 	// does not include them.
 	BlobsTooNew int `json:"blobs_too_new"`
+	// MetadataDirsDeleted counts the directories under metadata/ with no
+	// manifest that prune deleted, and MetadataDirsFailed those it could
+	// not delete.
+	MetadataDirsDeleted int `json:"metadata_dirs_deleted"`
+	MetadataDirsFailed  int `json:"metadata_dirs_failed,omitempty"`
 }
 
-// Prune removes orphaned data from the local index database AND
-// unreferenced blobs from the backup destination store. This is the
-// single user-facing prune entry point — the split between local and
-// remote cleanup is an implementation detail. Calling code should
-// prefer this method over PruneDatabase or PruneBlobs individually
-// unless it specifically wants one half.
+// Prune removes orphaned data from the local index database AND, from the
+// backup destination store, unreferenced blobs and the directories under
+// metadata/ with no manifest. This is the single user-facing prune entry
+// point — the split between local and remote cleanup is an
+// implementation detail. Calling code should prefer this method over
+// PruneDatabase or PruneBlobs individually unless it specifically wants
+// one half.
 func (v *Vaultik) Prune(opts *PruneOptions) error {
 	err := v.EnsureStorageBinding()
 	if err != nil {
@@ -200,11 +214,17 @@ func (v *Vaultik) Prune(opts *PruneOptions) error {
 	return v.PruneBlobs(opts)
 }
 
-// PruneBlobs removes unreferenced blobs from storage
+// PruneBlobs removes unreferenced blobs from storage, and each directory
+// under metadata/ that a backup left with no manifest.
 func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	log.Info("Starting prune operation")
 
-	allBlobsReferenced, err := v.collectReferencedBlobs()
+	metadataObjects, err := v.listMetadataObjects()
+	if err != nil {
+		return err
+	}
+
+	allBlobsReferenced, err := v.collectReferencedBlobs(metadataObjects)
 	if err != nil {
 		return err
 	}
@@ -217,6 +237,11 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs(
 		allBlobs, allBlobsReferenced)
 
+	dirsWithoutManifest, err := v.findMetadataDirsWithoutManifest(metadataObjects)
+	if err != nil {
+		return err
+	}
+
 	result := &PruneBlobsResult{
 		BlobsFound:  len(unreferencedBlobs),
 		BlobsTooNew: tooNew,
@@ -224,7 +249,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 
 	v.reportBlobsTooNew(tooNew, opts)
 
-	if len(unreferencedBlobs) == 0 {
+	if len(unreferencedBlobs) == 0 && len(dirsWithoutManifest) == 0 {
 		log.Info("No unreferenced blobs found")
 
 		if opts.JSON {
@@ -237,38 +262,16 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	}
 
 	log.Info("Found unreferenced blobs",
-		"count", len(unreferencedBlobs), "total_size", ubytes(totalSize))
+		"count", len(unreferencedBlobs), "total_size", ubytes(totalSize),
+		"metadata_dirs_without_manifest", len(dirsWithoutManifest))
 
-	if opts.Force && !opts.JSON {
-		v.UI.Infof("Found %d unreferenced blob(s) totaling %s.",
-			len(unreferencedBlobs), ubytes(totalSize))
-	}
-
-	if !opts.Force && !opts.JSON {
-		// The count, the prompt and its answer bypass the UI: they are an
-		// interactive exchange the operator must see, so --quiet does not
-		// silence them.
-		v.stdoutf("Found %d unreferenced blob(s) totaling %s.\n",
-			len(unreferencedBlobs), ubytes(totalSize))
-		v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
-
-		var confirm string
-
-		_, err = v.scanStdin(&confirm)
-		if err != nil {
-			v.printlnStdout("Cancelled")
-
-			return nil //nolint:nilerr // read failure means no confirmation
-		}
-
-		if strings.ToLower(confirm) != "y" {
-			v.printlnStdout("Cancelled")
-
-			return nil
-		}
+	if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize,
+		dirsWithoutManifest) {
+		return nil
 	}
 
 	v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
+	v.deleteMetadataDirs(dirsWithoutManifest, result)
 
 	if opts.JSON {
 		err = v.outputPruneBlobsJSON(result)
@@ -276,33 +279,142 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 			return err
 		}
 	} else {
+		v.reportPruneDeletions(result, len(dirsWithoutManifest))
+	}
+
+	var failures []error
+
+	if result.BlobsFailed > 0 {
+		failures = append(failures,
+			fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed))
+	}
+
+	if result.MetadataDirsFailed > 0 {
+		failures = append(failures, fmt.Errorf("%d %w",
+			result.MetadataDirsFailed, errMetadataDirDeleteFailed))
+	}
+
+	return errors.Join(failures...)
+}
+
+// confirmPrune states what prune found to delete, unreferenced blobs and
+// directories under metadata/ with no manifest, naming each directory by
+// its remote key, and, unless --force is given, asks the operator to
+// confirm. Under --json it prints nothing and does not ask. It reports
+// whether to go ahead.
+func (v *Vaultik) confirmPrune(
+	opts *PruneOptions, blobCount int, blobsSize int64, dirs []string,
+) bool {
+	if opts.JSON {
+		return true
+	}
+
+	dirCount := len(dirs)
+	blobsFound := fmt.Sprintf("Found %d unreferenced blob(s) totaling %s.",
+		blobCount, ubytes(blobsSize))
+	dirsFound := fmt.Sprintf("Found %d metadata director(ies) left by "+
+		"backups that stopped before their manifest:", dirCount)
+
+	if opts.Force {
+		if blobCount > 0 {
+			v.UI.Infof("%s", blobsFound)
+		}
+
+		if dirCount > 0 {
+			v.UI.Infof("%s", dirsFound)
+
+			for _, remoteKey := range dirs {
+				v.UI.Detailf("metadata/%s/", remoteKey)
+			}
+		}
+
+		return true
+	}
+
+	// The counts, the names, the prompt and its answer bypass the UI:
+	// they are an interactive exchange the operator must see, so --quiet
+	// does not silence them.
+	if blobCount > 0 {
+		v.stdoutf("%s\n", blobsFound)
+	}
+
+	if dirCount > 0 {
+		v.stdoutf("%s\n", dirsFound)
+
+		for _, remoteKey := range dirs {
+			v.stdoutf("  metadata/%s/\n", remoteKey)
+		}
+	}
+
+	switch {
+	case dirCount == 0:
+		v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", blobCount)
+	case blobCount == 0:
+		v.stdoutf("\nDelete %d metadata director(ies)? [y/N] ", dirCount)
+	default:
+		v.stdoutf("\nDelete %d unreferenced blob(s) and %d metadata "+
+			"director(ies)? [y/N] ", blobCount, dirCount)
+	}
+
+	var confirm string
+
+	// A failure to read the answer means no confirmation.
+	_, err := v.scanStdin(&confirm)
+	if err != nil || strings.ToLower(confirm) != "y" {
+		v.printlnStdout("Cancelled")
+
+		return false
+	}
+
+	return true
+}
+
+// reportPruneDeletions says how many blobs and metadata directories
+// prune deleted, naming each kind only when it found some to delete.
+func (v *Vaultik) reportPruneDeletions(result *PruneBlobsResult, dirCount int) {
+	if result.BlobsFound > 0 {
 		v.UI.Infof("Deleted %d blob(s) totaling %s.",
 			result.BlobsDeleted, ubytes(result.BytesFreed))
 	}
 
-	if result.BlobsFailed > 0 {
-		return fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed)
+	if dirCount > 0 {
+		v.UI.Infof("Deleted %d metadata director(ies) with no manifest.",
+			result.MetadataDirsDeleted)
 	}
-
-	return nil
 }
 
-// collectReferencedBlobs downloads all manifests and returns the set of
-// referenced blob hashes.
+// listMetadataObjects lists every object under metadata/ on the
+// destination store.
+func (v *Vaultik) listMetadataObjects() ([]storage.ObjectInfo, error) {
+	log.Info("Listing remote snapshots")
+
+	var objects []storage.ObjectInfo
+
+	for object := range v.Storage.ListStream(v.ctx, "metadata/") {
+		if object.Err != nil {
+			return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
+		}
+
+		objects = append(objects, object)
+	}
+
+	return objects, nil
+}
+
+// collectReferencedBlobs downloads every manifest among the listed
+// metadata objects and returns the set of referenced blob hashes.
 //
 // Every manifest must be read successfully. A manifest that cannot be
 // downloaded or decoded means its snapshot's blobs are unknown, so
 // treating them as unreferenced would let prune delete data a snapshot
 // still needs. Rather than risk that silent loss, any failure returns an
 // error naming the remote key and prune deletes nothing.
-func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
-	log.Info("Listing remote snapshots")
-	// IDs returned by listUniqueSnapshotIDs are remote keys (hashed
+func (v *Vaultik) collectReferencedBlobs(
+	metadataObjects []storage.ObjectInfo,
+) (map[string]bool, error) {
+	// IDs returned by uniqueSnapshotIDs are remote keys (hashed
 	// subdirectories under metadata/), not human snapshot IDs.
-	remoteKeys, err := v.listUniqueSnapshotIDs()
-	if err != nil {
-		return nil, fmt.Errorf("listing snapshot keys: %w", err)
-	}
+	remoteKeys := uniqueSnapshotIDs(metadataObjects)
 
 	log.Info("Found manifests in remote storage", "count", len(remoteKeys))
 
@@ -327,18 +439,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
 	return allBlobsReferenced, nil
 }
 
-// listUniqueSnapshotIDs returns deduplicated snapshot IDs from remote metadata
-func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
-	objectCh := v.Storage.ListStream(v.ctx, "metadata/")
+// uniqueSnapshotIDs returns deduplicated snapshot IDs from the listed
+// metadata objects.
+func uniqueSnapshotIDs(metadataObjects []storage.ObjectInfo) []string {
 	seen := make(map[string]bool)
 
 	var snapshotIDs []string
 
-	for object := range objectCh {
-		if object.Err != nil {
-			return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
-		}
-
+	for _, object := range metadataObjects {
 		parts := strings.Split(object.Key, "/")
 		if len(parts) >= minSnapshotIDParts &&
 			parts[0] == metadataDirName && parts[1] != "" {
@@ -353,7 +461,7 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
 		}
 	}
 
-	return snapshotIDs, nil
+	return snapshotIDs
 }
 
 // listAllRemoteBlobs returns a map of all blob hashes in remote storage
@@ -427,6 +535,82 @@ func (v *Vaultik) findUnreferencedBlobs(
 	return unreferenced, totalSize, tooNew
 }
 
+// findMetadataDirsWithoutManifest returns the remote key of each
+// directory under metadata/ that has no manifest.json.zst, such as one a
+// backup leaves when its manifest upload fails after db.zst.age was
+// uploaded. It leaves out a directory a local snapshot record claims,
+// and one holding an object newer than the prune grace period, or of
+// unknown age because the listing gave no time for it: a backup still
+// running may be writing there. A name that is not a remote key is
+// never returned.
+func (v *Vaultik) findMetadataDirsWithoutManifest(
+	metadataObjects []storage.ObjectInfo,
+) ([]string, error) {
+	localIDs, err := v.localSnapshotIDs()
+	if err != nil {
+		return nil, fmt.Errorf("listing local snapshots: %w", err)
+	}
+
+	keep := make(map[string]bool, len(localIDs))
+	for _, id := range localIDs {
+		keep[snapshot.RemoteSnapshotKey(id)] = true
+	}
+
+	cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod)
+	listed := make(map[string]bool)
+
+	for _, object := range metadataObjects {
+		parts := strings.Split(object.Key, "/")
+		if len(parts) < metadataKeyParts || !isBlobHash(parts[1]) {
+			continue
+		}
+
+		remoteKey := parts[1]
+		listed[remoteKey] = true
+
+		if parts[2] == "manifest.json.zst" ||
+			object.ModTime.IsZero() || object.ModTime.After(cutoff) {
+			keep[remoteKey] = true
+		}
+	}
+
+	var remoteKeys []string
+
+	for remoteKey := range listed {
+		if !keep[remoteKey] {
+			remoteKeys = append(remoteKeys, remoteKey)
+		}
+	}
+
+	sort.Strings(remoteKeys)
+
+	return remoteKeys, nil
+}
+
+// deleteMetadataDirs deletes everything in the given directories under
+// metadata/ and counts in the result the directories deleted and those
+// it could not delete.
+func (v *Vaultik) deleteMetadataDirs(
+	remoteKeys []string, result *PruneBlobsResult,
+) {
+	for _, remoteKey := range remoteKeys {
+		_, err := v.deleteRemoteSnapshotByKey(remoteKey)
+		if err != nil {
+			log.Error("Failed to delete metadata directory with no manifest",
+				"remote_key", remoteKey, "error", err)
+
+			result.MetadataDirsFailed++
+
+			continue
+		}
+
+		log.Info("Deleted metadata directory with no manifest",
+			"remote_key", remoteKey)
+
+		result.MetadataDirsDeleted++
+	}
+}
+
 // reportBlobsTooNew says how many unreferenced blobs prune kept for being
 // newer than the grace period or of unknown age. Under --json the
 // document carries the count instead.