Keep unreferenced blobs newer than prune_grace_period (closes #387)

A backup uploads its blobs before the manifest that lists them, and the
process lock is per user per machine, so a prune from another host or
user sharing the destination deleted a running backup's blobs; that
backup still reported success and could not be restored.

Prune now deletes an unreferenced blob only once it is older than the new
config key prune_grace_period (default 24h), and reports how many it kept
(blobs_too_new under --json). The storage listing now carries each
object's modification time. remote nuke is unchanged. The README says what
the lock covers and that a shared destination relies on the grace period
or needs one prefix per host.

Model: opus-5-5

internal/cli/config.go

@@ -239,6 +239,13 @@ storage_url: ""
 # Default: 3
 # compression_level: 3
 
+# How old a blob that no snapshot lists must be before prune deletes it.
+# A backup uploads its blobs before the manifest that lists them, so on a
+# destination shared between hosts or users this must be longer than the
+# longest backup run. Accepts: 36h, 90m, etc. 0s deletes them at once.
+# Default: 24h
+# prune_grace_period: 24h
+
 # Hostname used in snapshot IDs. Default: system hostname.
 # hostname: myserver
 

internal/cli/entry_prune_quiet_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -14,8 +15,8 @@ import (
 const flagCron = "--cron"
 
 // plantUnreferencedBlob writes a blob that no snapshot references to the
-// destination store at storeDir, so that a prune deletes it. Returns its
-// path.
+// destination store at storeDir, dated long before any prune grace period,
+// so that a prune deletes it. Returns its path.
 func plantUnreferencedBlob(t *testing.T, storeDir string) string {
 	t.Helper()
 
@@ -25,6 +26,9 @@ func plantUnreferencedBlob(t *testing.T, storeDir string) string {
 	require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750))
 	require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600))
 
+	longAgo := time.Unix(0, 0)
+	require.NoError(t, os.Chtimes(path, longAgo, longAgo))
+
 	return path
 }
 

internal/cli/prune.go

@@ -20,7 +20,10 @@ Local cleanup drops incomplete snapshots and any files, chunks, or
 blobs no longer referenced by a completed snapshot. Remote cleanup
 scans every snapshot manifest in the destination store, builds the
 set of still-referenced blob hashes, and deletes any blob not in that
-set.
+set that is older than prune_grace_period (default 24h). A newer one
+is kept: it may belong to a backup still running, on this host or
+another sharing the destination store, that has not yet uploaded its
+manifest.
 
 Snapshot create --prune runs the same cleanup automatically; this
 command is the manual entry point for the same work (e.g. after a

internal/cli/snapshot.go

@@ -132,8 +132,8 @@ on Linux), then /etc/vaultik/config.yml.`,
 		"Run in cron mode (silent unless warning or error)")
 	cmd.Flags().BoolVar(&opts.Prune, "prune", false,
 		"After backup, drop older snapshots of the same name and remove "+
-			"orphaned blobs (not after a backup that skipped unreadable "+
-			"files)")
+			"orphaned blobs older than prune_grace_period (not after a "+
+			"backup that skipped unreadable files)")
 	cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
 		"With --prune: keep snapshots newer than this duration "+
 			"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+

internal/config/config.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"filippo.io/age"
 	"git.eeqj.de/sneak/smartconfig"
@@ -38,6 +39,7 @@ const (
 	defaultChunkSize        = Size(10 * 1000 * 1000)        // 10MB
 	defaultS3PartSize       = Size(5 * 1024 * 1024)         // 5MiB
 	defaultCompressionLevel = 3
+	defaultPruneGracePeriod = 24 * time.Hour
 	minChunkSize            = 1000 * 1000 // 1MB
 	minCompressionLevel     = 1
 	maxCompressionLevel     = 19
@@ -187,6 +189,13 @@ type Config struct {
 	Snapshots        map[string]SnapshotConfig `yaml:"snapshots"`
 	CompressionLevel int                       `yaml:"compression_level"`
 
+	// PruneGracePeriod is how old a blob no manifest lists must be before
+	// a prune deletes it. A backup lists its blobs in a manifest only
+	// once it has uploaded them all, so a younger one may belong to a
+	// backup still running, on this host or another sharing the
+	// destination store.
+	PruneGracePeriod time.Duration `yaml:"prune_grace_period"`
+
 	// StorageURL specifies the storage backend using a URL format.
 	// Takes precedence over S3Config if set.
 	// Supported formats:
@@ -288,6 +297,7 @@ func Parse(data []byte) (*Config, error) {
 		ChunkSize:        defaultChunkSize,
 		IndexPath:        filepath.Join(xdg.DataHome, appName, "index.sqlite"),
 		CompressionLevel: defaultCompressionLevel,
+		PruneGracePeriod: defaultPruneGracePeriod,
 		S3:               S3Config{PartSize: defaultS3PartSize},
 	}
 

internal/s3/client.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"strings"
 	"sync/atomic"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -302,7 +303,10 @@ func IsNotFound(err error) bool {
 type ObjectInfo struct {
 	Key  string
 	Size int64
-	Err  error
+	// ModTime is the object's LastModified time. Only ListObjectsStream
+	// sets it.
+	ModTime time.Time
+	Err     error
 }
 
 // ListObjectsStream lists objects with the given prefix and returns a channel.
@@ -343,8 +347,9 @@ func (c *Client) ListObjectsStream(
 					}
 
 					ch <- ObjectInfo{
-						Key:  key,
-						Size: *obj.Size,
+						Key:     key,
+						Size:    *obj.Size,
+						ModTime: aws.ToTime(obj.LastModified),
 					}
 				}
 			}

internal/storage/conformance_test.go

@@ -8,6 +8,7 @@ import (
 	"reflect"
 	"sort"
 	"testing"
+	"time"
 
 	"sneak.berlin/go/vaultik/internal/storage"
 )
@@ -19,15 +20,17 @@ import (
 // constructor, so the contract is defined once.
 //
 // It exercises the public Storer interface: round-trip, stat, list with
-// prefix filtering, overwrite, delete, delete-of-missing, and not-found on
-// Get and Stat. Each section takes its own fresh backend instance, so the
-// order of sections never matters and no section sees another's objects.
+// prefix filtering, the modification time ListStream reports, overwrite,
+// delete, delete-of-missing, and not-found on Get and Stat. Each section
+// takes its own fresh backend instance, so the order of sections never
+// matters and no section sees another's objects.
 func runStorerConformance(t *testing.T, newStorer func(*testing.T) storage.Storer) {
 	t.Helper()
 
 	conformanceRoundTrip(t, newStorer(t))
 	conformanceOverwrite(t, newStorer(t))
 	conformanceList(t, newStorer(t))
+	conformanceListStreamModTime(t, newStorer(t))
 	conformanceDelete(t, newStorer(t))
 	conformanceNotFound(t, newStorer(t))
 }
@@ -118,6 +121,35 @@ func conformanceList(t *testing.T, s storage.Storer) {
 	}
 }
 
+// conformanceListStreamModTime checks that ListStream gives an object
+// just written a modification time of about now, which prune compares
+// with its grace period.
+func conformanceListStreamModTime(t *testing.T, s storage.Storer) {
+	t.Helper()
+
+	ctx := context.Background()
+
+	err := s.Put(ctx, "blobs/dd/written-now", bytes.NewReader([]byte("data")))
+	if err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	var listed []storage.ObjectInfo
+
+	for info := range s.ListStream(ctx, "blobs/") {
+		listed = append(listed, info)
+	}
+
+	if len(listed) != 1 || listed[0].Err != nil {
+		t.Fatalf("ListStream(\"blobs/\") = %v, want the one object", listed)
+	}
+
+	if time.Since(listed[0].ModTime).Abs() > time.Minute {
+		t.Errorf("ListStream ModTime = %v, want within a minute of now",
+			listed[0].ModTime)
+	}
+}
+
 // conformanceDelete checks that Delete removes an object and that deleting
 // a missing key is not an error.
 func conformanceDelete(t *testing.T, s storage.Storer) {

internal/storage/file.go

@@ -235,8 +235,9 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 				// Normalize path separators
 				relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
 				ch <- ObjectInfo{
-					Key:  relPath,
-					Size: info.Size(),
+					Key:     relPath,
+					Size:    info.Size(),
+					ModTime: info.ModTime(),
 				}
 			}
 

internal/storage/rclone.go

@@ -211,8 +211,9 @@ func (r *RcloneStorer) ListStream(
 
 			if prefix == "" || strings.HasPrefix(key, prefix) {
 				ch <- ObjectInfo{
-					Key:  key,
-					Size: obj.Size(),
+					Key:     key,
+					Size:    obj.Size(),
+					ModTime: obj.ModTime(ctx),
 				}
 			}
 		})

internal/storage/s3.go

@@ -89,9 +89,10 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
 
 		for info := range s.client.ListObjectsStream(ctx, prefix, false) {
 			ch <- ObjectInfo{
-				Key:  info.Key,
-				Size: info.Size,
-				Err:  info.Err,
+				Key:     info.Key,
+				Size:    info.Size,
+				ModTime: info.ModTime,
+				Err:     info.Err,
 			}
 		}
 	}()

internal/storage/storer.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	"errors"
 	"io"
+	"time"
 )
 
 // ErrNotFound is returned when an object does not exist.
@@ -27,7 +28,10 @@ type ProgressCallback func(bytesTransferred int64) error
 type ObjectInfo struct {
 	Key  string // Object key/path
 	Size int64  // Size in bytes
-	Err  error  // Error for streaming results (nil on success)
+	// ModTime is when the object was written, as the store reports it.
+	// Only ListStream sets it; Stat leaves it zero.
+	ModTime time.Time
+	Err     error // Error for streaming results (nil on success)
 }
 
 // Info provides human-readable storage configuration.
@@ -67,7 +71,8 @@ type Storer interface {
 	// For large result sets, prefer ListStream.
 	List(ctx context.Context, prefix string) ([]string, error)
 
-	// ListStream returns a channel of ObjectInfo for large result sets.
+	// ListStream returns a channel of ObjectInfo, with each object's key,
+	// size and modification time, for large result sets.
 	// The channel is closed when listing completes.
 	// If an error occurs during listing, the final item will have Err set.
 	ListStream(ctx context.Context, prefix string) <-chan ObjectInfo

internal/vaultik/info.go

@@ -457,9 +457,9 @@ func (v *Vaultik) scanRemoteBlobStorage(
 		return err
 	}
 
-	for _, size := range allBlobs {
+	for _, blob := range allBlobs {
 		result.TotalBlobCount++
-		result.TotalBlobSize += size
+		result.TotalBlobSize += blob.Size
 	}
 
 	// A blob named only by a manifest that could not be read, or by one
@@ -474,10 +474,10 @@ func (v *Vaultik) scanRemoteBlobStorage(
 		orphanedSize  int64
 	)
 
-	for hash, size := range allBlobs {
+	for hash, blob := range allBlobs {
 		if _, referenced := referencedBlobs[hash]; !referenced {
 			orphanedCount++
-			orphanedSize += size
+			orphanedSize += blob.Size
 		}
 	}
 

internal/vaultik/prune.go

@@ -5,8 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
 )
 
 // PruneOptions contains options for the prune command
@@ -163,6 +165,9 @@ type PruneBlobsResult struct {
 	BlobsDeleted int   `json:"blobs_deleted"`
 	BlobsFailed  int   `json:"blobs_failed,omitempty"`
 	BytesFreed   int64 `json:"bytes_freed"`
+	// BlobsTooNew counts the unreferenced blobs kept because they are
+	// newer than the prune grace period. BlobsFound does not include them.
+	BlobsTooNew int `json:"blobs_too_new"`
 }
 
 // Prune removes orphaned data from the local index database AND
@@ -208,9 +213,15 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 		return err
 	}
 
-	unreferencedBlobs, totalSize := v.findUnreferencedBlobs(allBlobs, allBlobsReferenced)
+	unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs(
+		allBlobs, allBlobsReferenced)
 
-	result := &PruneBlobsResult{BlobsFound: len(unreferencedBlobs)}
+	result := &PruneBlobsResult{
+		BlobsFound:  len(unreferencedBlobs),
+		BlobsTooNew: tooNew,
+	}
+
+	v.reportBlobsTooNew(tooNew, opts)
 
 	if len(unreferencedBlobs) == 0 {
 		log.Info("No unreferenced blobs found")
@@ -344,11 +355,12 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
 	return snapshotIDs, nil
 }
 
-// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage
-func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
+// listAllRemoteBlobs returns a map of all blob hashes in remote storage
+// to their listing entries.
+func (v *Vaultik) listAllRemoteBlobs() (map[string]storage.ObjectInfo, error) {
 	log.Info("Listing all blobs in storage")
 
-	allBlobs := make(map[string]int64)
+	allBlobs := make(map[string]storage.ObjectInfo)
 	blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
 
 	for object := range blobObjectCh {
@@ -372,7 +384,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
 			continue
 		}
 
-		allBlobs[parts[3]] = object.Size
+		allBlobs[parts[3]] = object
 	}
 
 	log.Info("Found blobs in storage", "count", len(allBlobs))
@@ -380,30 +392,57 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
 	return allBlobs, nil
 }
 
-// findUnreferencedBlobs returns blob hashes not referenced by any
-// manifest and their total size.
+// findUnreferencedBlobs returns the hashes of the blobs no manifest
+// references that are older than the prune grace period, and their total
+// size. It also returns how many unreferenced blobs it left out for
+// being newer than that.
 func (v *Vaultik) findUnreferencedBlobs(
-	allBlobs map[string]int64, referenced map[string]bool,
-) ([]string, int64) {
+	allBlobs map[string]storage.ObjectInfo, referenced map[string]bool,
+) ([]string, int64, int) {
 	var (
 		unreferenced []string
 		totalSize    int64
+		tooNew       int
 	)
 
-	for hash, size := range allBlobs {
-		if !referenced[hash] {
-			unreferenced = append(unreferenced, hash)
-			totalSize += size
+	cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod)
+
+	for hash, blob := range allBlobs {
+		if referenced[hash] {
+			continue
 		}
+
+		if blob.ModTime.After(cutoff) {
+			tooNew++
+
+			continue
+		}
+
+		unreferenced = append(unreferenced, hash)
+		totalSize += blob.Size
 	}
 
-	return unreferenced, totalSize
+	return unreferenced, totalSize, tooNew
+}
+
+// reportBlobsTooNew says how many unreferenced blobs prune kept for being
+// newer than the grace period. Under --json the document carries the
+// count instead.
+func (v *Vaultik) reportBlobsTooNew(tooNew int, opts *PruneOptions) {
+	if tooNew == 0 || opts.JSON {
+		return
+	}
+
+	v.UI.Infof("Kept %d unreferenced blob(s) newer than "+
+		"prune_grace_period (%s): a backup still running may need them.",
+		tooNew, v.Config.PruneGracePeriod)
 }
 
 // deleteUnreferencedBlobs deletes the given blobs from storage and
 // populates the result.
 func (v *Vaultik) deleteUnreferencedBlobs(
-	unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult,
+	unreferencedBlobs []string, allBlobs map[string]storage.ObjectInfo,
+	result *PruneBlobsResult,
 ) {
 	log.Info("Deleting unreferenced blobs")
 
@@ -418,7 +457,7 @@ func (v *Vaultik) deleteUnreferencedBlobs(
 		}
 
 		result.BlobsDeleted++
-		result.BytesFreed += allBlobs[hash]
+		result.BytesFreed += allBlobs[hash].Size
 
 		if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 {
 			log.Info("Deletion progress",

internal/vaultik/prune_grace_period_test.go

@@ -0,0 +1,138 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/storage/faultstore"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestPruneDuringBackupKeepsItsBlobs runs a prune from another host
+// sharing the destination store at the moment a backup has uploaded its
+// blobs but not yet its manifest, so no manifest lists those blobs. The
+// prune must leave them, and the backup must then restore.
+func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	dataDir := filepath.Join(tempDir, "src")
+	storeDir := t.TempDir()
+	restoreDir := filepath.Join(tempDir, "restored")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	ctx := context.Background()
+	cfg := faultTestConfig()
+	testFiles := writeFaultSourceTree(t, fs, dataDir)
+
+	inner, err := storage.NewFileStorer(storeDir)
+	require.NoError(t, err)
+
+	otherHostCfg := faultTestConfig()
+	otherHostCfg.PruneGracePeriod = time.Hour
+	otherHost := newReaderVaultik(ctx, otherHostCfg, inner, nil, fs)
+
+	var (
+		pruned   bool
+		pruneErr error
+	)
+
+	store := faultstore.New(inner)
+	store.OnPut = func(key string) faultstore.PutAction {
+		if strings.HasSuffix(key, "/manifest.json.zst") {
+			pruned = true
+			pruneErr = otherHost.PruneBlobs(&vaultik.PruneOptions{Force: true})
+		}
+
+		return faultstore.PutNormal
+	}
+
+	db, err := database.New(ctx, dbPath)
+	require.NoError(t, err)
+
+	repos := database.NewRepositories(db)
+
+	id := fullFaultBackup(ctx, t, fs, store, cfg, repos, dataDir, dbPath, "shared")
+	require.NoError(t, db.Close())
+
+	require.True(t, pruned, "the prune must run before the manifest upload")
+	require.NoError(t, pruneErr)
+
+	v := newReaderVaultik(ctx, cfg, inner, nil, fs)
+	require.NoError(t, v.Restore(&vaultik.RestoreOptions{
+		SnapshotID: id,
+		TargetDir:  restoreDir,
+		Verify:     true,
+	}), "a prune during the backup must not delete the backup's blobs")
+
+	assertRestoredTree(t, fs, restoreDir, testFiles)
+}
+
+// TestPruneDeletesUnreferencedBlobOlderThanGracePeriod checks that the
+// grace period only delays deletion: a blob no manifest lists is deleted
+// once it is older than the grace period, and kept and counted until
+// then.
+func TestPruneDeletesUnreferencedBlobOlderThanGracePeriod(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := t.TempDir()
+
+	store, err := storage.NewFileStorer(storeDir)
+	require.NoError(t, err)
+
+	oldKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] +
+		"/" + testBlobHashA
+	newKey := "blobs/" + testBlobHashB[:2] + "/" + testBlobHashB[2:4] +
+		"/" + testBlobHashB
+
+	for _, key := range []string{oldKey, newKey} {
+		require.NoError(t, store.Put(ctx, key, strings.NewReader("blob")))
+	}
+
+	gracePeriod := time.Hour
+	beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute)
+	require.NoError(t, os.Chtimes(filepath.Join(storeDir, oldKey),
+		beforeGracePeriod, beforeGracePeriod))
+
+	stdout := &bytes.Buffer{}
+	v := &vaultik.Vaultik{
+		Config:  &config.Config{PruneGracePeriod: gracePeriod},
+		Storage: store,
+		Stdout:  stdout,
+		UI:      ui.NewWithColor(stdout, false),
+	}
+	v.SetContext(ctx)
+
+	require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true}))
+
+	_, err = store.Stat(ctx, oldKey)
+	require.ErrorIs(t, err, storage.ErrNotFound,
+		"a blob older than the grace period must be deleted")
+
+	_, err = store.Stat(ctx, newKey)
+	require.NoError(t, err, "a blob newer than the grace period must be kept")
+
+	var result vaultik.PruneBlobsResult
+
+	require.NoError(t, json.Unmarshal(stdout.Bytes(), &result))
+	assert.Equal(t, 1, result.BlobsDeleted)
+	assert.Equal(t, 1, result.BlobsTooNew)
+}