Keep unreferenced blobs newer than prune_grace_period (closes #387)

A backup uploads its blobs before the manifest that lists them, and the
process lock covers one user on one machine, so a prune from another host
or user sharing the destination deleted a running backup's blobs.

Prune now deletes an unreferenced blob only once its listed time is older
than the new config key prune_grace_period (default 24h; a negative value
is refused). A newer blob, or one listed with no time, is kept and counted
(blobs_too_new). The rclone backend takes each object's time from the
listing (use_server_modtime).

Decision: a blob a backup reuses from an earlier run that stopped before
its manifest keeps its old time and is not protected; the docs and config
comments say so.

Model: opus-5-5

internal/vaultik/prune.go

@@ -5,8 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
 )
 
 // PruneOptions contains options for the prune command
@@ -163,6 +165,10 @@ type PruneBlobsResult struct {
 	BlobsDeleted int   `json:"blobs_deleted"`
 	BlobsFailed  int   `json:"blobs_failed,omitempty"`
 	BytesFreed   int64 `json:"bytes_freed"`
+	// BlobsTooNew counts the unreferenced blobs kept because they are
+	// newer than the prune grace period, or of unknown age. BlobsFound
+	// does not include them.
+	BlobsTooNew int `json:"blobs_too_new"`
 }
 
 // Prune removes orphaned data from the local index database AND
@@ -208,9 +214,15 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 		return err
 	}
 
-	unreferencedBlobs, totalSize := v.findUnreferencedBlobs(allBlobs, allBlobsReferenced)
+	unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs(
+		allBlobs, allBlobsReferenced)
 
-	result := &PruneBlobsResult{BlobsFound: len(unreferencedBlobs)}
+	result := &PruneBlobsResult{
+		BlobsFound:  len(unreferencedBlobs),
+		BlobsTooNew: tooNew,
+	}
+
+	v.reportBlobsTooNew(tooNew, opts)
 
 	if len(unreferencedBlobs) == 0 {
 		log.Info("No unreferenced blobs found")
@@ -344,11 +356,12 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
 	return snapshotIDs, nil
 }
 
-// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage
-func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
+// listAllRemoteBlobs returns a map of all blob hashes in remote storage
+// to their listing entries.
+func (v *Vaultik) listAllRemoteBlobs() (map[string]storage.ObjectInfo, error) {
 	log.Info("Listing all blobs in storage")
 
-	allBlobs := make(map[string]int64)
+	allBlobs := make(map[string]storage.ObjectInfo)
 	blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
 
 	for object := range blobObjectCh {
@@ -372,7 +385,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
 			continue
 		}
 
-		allBlobs[parts[3]] = object.Size
+		allBlobs[parts[3]] = object
 	}
 
 	log.Info("Found blobs in storage", "count", len(allBlobs))
@@ -380,30 +393,59 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
 	return allBlobs, nil
 }
 
-// findUnreferencedBlobs returns blob hashes not referenced by any
-// manifest and their total size.
+// findUnreferencedBlobs returns the hashes of the blobs no manifest
+// references that are older than the prune grace period, and their total
+// size. It also returns how many unreferenced blobs it left out for
+// being newer than that, or of unknown age because the listing gave no
+// time for them.
 func (v *Vaultik) findUnreferencedBlobs(
-	allBlobs map[string]int64, referenced map[string]bool,
-) ([]string, int64) {
+	allBlobs map[string]storage.ObjectInfo, referenced map[string]bool,
+) ([]string, int64, int) {
 	var (
 		unreferenced []string
 		totalSize    int64
+		tooNew       int
 	)
 
-	for hash, size := range allBlobs {
-		if !referenced[hash] {
-			unreferenced = append(unreferenced, hash)
-			totalSize += size
+	cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod)
+
+	for hash, blob := range allBlobs {
+		if referenced[hash] {
+			continue
 		}
+
+		if blob.ModTime.IsZero() || blob.ModTime.After(cutoff) {
+			tooNew++
+
+			continue
+		}
+
+		unreferenced = append(unreferenced, hash)
+		totalSize += blob.Size
 	}
 
-	return unreferenced, totalSize
+	return unreferenced, totalSize, tooNew
+}
+
+// reportBlobsTooNew says how many unreferenced blobs prune kept for being
+// newer than the grace period or of unknown age. Under --json the
+// document carries the count instead.
+func (v *Vaultik) reportBlobsTooNew(tooNew int, opts *PruneOptions) {
+	if tooNew == 0 || opts.JSON {
+		return
+	}
+
+	v.UI.Infof("Kept %d unreferenced blob(s) newer than "+
+		"prune_grace_period (%s), or of unknown age: a backup still "+
+		"running may need them.",
+		tooNew, v.Config.PruneGracePeriod)
 }
 
 // deleteUnreferencedBlobs deletes the given blobs from storage and
 // populates the result.
 func (v *Vaultik) deleteUnreferencedBlobs(
-	unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult,
+	unreferencedBlobs []string, allBlobs map[string]storage.ObjectInfo,
+	result *PruneBlobsResult,
 ) {
 	log.Info("Deleting unreferenced blobs")
 
@@ -418,7 +460,7 @@ func (v *Vaultik) deleteUnreferencedBlobs(
 		}
 
 		result.BlobsDeleted++
-		result.BytesFreed += allBlobs[hash]
+		result.BytesFreed += allBlobs[hash].Size
 
 		if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 {
 			log.Info("Deletion progress",