Refuse credentials in a storage URL's endpoint or rclone remote name (closes #322)

storage_url refused only the outer URL's userinfo and unknown query
parameters. Userinfo inside the s3 endpoint= value, and rclone's
remote,param=value parameters in the remote name, were accepted and
then printed by info, remote info and snapshot create. Both are now
refused without echoing the value: a host name never contains "@", and
an rclone remote name never contains a comma. The comma is checked
before url.Parse, because url.Parse refuses a name such as
":s3,secret_access_key=..." or "x,secret=a:b", and its error prints
the name from the colon on. The rclone case has its own error pointing
at rclone config, since the existing credentials error names the s3.*
keys.

Model: opus-5-5

README.md

@@ -386,10 +386,12 @@ manifest on the destination store to compute the still-referenced blob
 set and delete any blob not in that set. Step (3) reads all remote
 manifests — network cost scales with the number of snapshots. `snapshot
 create --prune` runs the same cleanup automatically; this is the
-manual entry point for the same work.
+manual entry point for the same work. If a blob cannot be deleted, the
+command (and `snapshot create --prune`) exits non-zero.
 * `--force`: Skip confirmation prompt
 * `--json`: Output stats as JSON. Also skips the confirmation prompt, as
-  `--force` does.
+  `--force` does. A blob that cannot be deleted is counted in
+  `blobs_failed`, and the document is printed before the non-zero exit.
 
 **`info`**: Display system configuration, storage settings, encryption
 recipients, and local database statistics.
@@ -411,7 +413,8 @@ since `prune` never deletes it.
 **`remote nuke`**: Delete every snapshot's metadata and every blob from the
 backup destination store, leaving the bucket prefix empty. Destructive and
 irreversible. This is the single supported way to wipe the entire
-destination store.
+destination store. If anything cannot be deleted, the command exits
+non-zero; running it again deletes what is left.
 * `--force`: Required to confirm destruction.
 
 **`database delete`**: Delete the local SQLite state database file

internal/s3/client.go

@@ -18,6 +18,13 @@ import (
 	"github.com/aws/smithy-go/logging"
 )
 
+// ErrNoContinuationToken is returned by ListObjects and ListObjectsStream
+// when the server marks a page of a listing truncated but sends no
+// continuation token to fetch the rest with, as a server answering V2
+// listings V1-style does.
+var ErrNoContinuationToken = errors.New(
+	"server marked the listing truncated but sent no continuation token")
+
 // Client wraps the AWS S3 client for vaultik operations.
 // It provides a simplified interface for S3 operations with automatic
 // prefix handling and connection management. All operations are performed
@@ -214,7 +221,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
 	})
 
 	for paginator.HasMorePages() {
-		page, err := paginator.NextPage(ctx)
+		page, err := nextListPage(ctx, paginator)
 		if err != nil {
 			return nil, err
 		}
@@ -235,6 +242,26 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
 	return keys, nil
 }
 
+// nextListPage fetches the next page of a listing. The SDK paginator takes
+// a page marked truncated without a continuation token for the last page
+// and stops without an error, so such a page is ErrNoContinuationToken
+// here instead.
+func nextListPage(
+	ctx context.Context, paginator *s3.ListObjectsV2Paginator,
+) (*s3.ListObjectsV2Output, error) {
+	page, err := paginator.NextPage(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	truncated := aws.ToBool(page.IsTruncated)
+	if truncated && aws.ToString(page.NextContinuationToken) == "" {
+		return nil, ErrNoContinuationToken
+	}
+
+	return page, nil
+}
+
 // HeadObject checks if an object exists in S3.
 // Returns true if the object exists, false otherwise.
 // The key is automatically prefixed with the configured prefix.
@@ -300,7 +327,7 @@ func (c *Client) ListObjectsStream(
 		})
 
 		for paginator.HasMorePages() {
-			page, err := paginator.NextPage(ctx)
+			page, err := nextListPage(ctx, paginator)
 			if err != nil {
 				ch <- ObjectInfo{Err: err}
 

internal/storage/faultstore/faultstore.go

@@ -1,15 +1,16 @@
 // Package faultstore provides a storage.Storer wrapper that injects
 // faults on demand, so tests can reproduce the failure modes a real
 // backend exhibits: an upload that fails partway, a backend that reports
-// success while storing nothing, and reads that return corrupt or
-// truncated bytes. It is the seam called for by the fault-injection
-// tests (sneak/vaultik issue 72) and is meant to be reused by future
-// tests rather than re-implemented per case.
+// success while storing nothing, reads that return corrupt or truncated
+// bytes, and a delete the backend refuses. It is the seam called for by
+// the fault-injection tests (sneak/vaultik issue 72) and is meant to be
+// reused by future tests rather than re-implemented per case.
 //
-// The wrapper delegates every method to the inner Storer. Two hooks
-// change that: OnPut decides the fate of each write, and OnGet decides
-// how each read's bytes are returned. Both are keyed by the object key,
-// so a test can fault only blobs, only metadata, or a single object.
+// The wrapper delegates every method to the inner Storer. Three hooks
+// change that: OnPut decides the fate of each write, OnGet decides how
+// each read's bytes are returned, and OnDelete decides whether each
+// delete fails. All are keyed by the object key, so a test can fault
+// only blobs, only metadata, or a single object.
 package faultstore
 
 import (
@@ -25,6 +26,10 @@ import (
 // ErrInjectedUpload is returned by a Put the OnPut hook chose to fail.
 var ErrInjectedUpload = errors.New("faultstore: injected upload failure")
 
+// ErrInjectedDelete is returned by a Delete the OnDelete hook chose to
+// fail.
+var ErrInjectedDelete = errors.New("faultstore: injected delete failure")
+
 // PutAction is the disposition OnPut assigns to a write.
 type PutAction int
 
@@ -66,6 +71,11 @@ type Storer struct {
 	// OnGet, when set, is consulted for every Get with the object key
 	// and damages the returned bytes accordingly.
 	OnGet func(key string) GetFault
+
+	// OnDelete, when set, is consulted before every Delete with the
+	// object key. Returning true fails the delete and leaves the object
+	// in place.
+	OnDelete func(key string) bool
 }
 
 // New wraps inner. inner must be non-nil.
@@ -154,8 +164,12 @@ func (f *Storer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, err
 	return f.inner.Stat(ctx, key)
 }
 
-// Delete delegates unchanged.
+// Delete removes the object unless OnDelete fails the delete.
 func (f *Storer) Delete(ctx context.Context, key string) error {
+	if f.OnDelete != nil && f.OnDelete(key) {
+		return fmt.Errorf("%w for %q", ErrInjectedDelete, key)
+	}
+
 	return f.inner.Delete(ctx, key)
 }
 

internal/storage/s3_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"slices"
@@ -22,13 +23,16 @@ import (
 // s3TestBucket is the bucket created for each in-process S3 server.
 const s3TestBucket = "test-bucket"
 
-// Credentials for the tests that build a storer from a config.Config. The
-// in-process S3 server accepts any.
+// Credentials for the tests' S3 clients. The test servers accept any.
 const (
 	s3TestAccessKeyID     = "key"
 	s3TestSecretAccessKey = "secret"
 )
 
+// s3TestRegion is the region of the S3 clients built here with
+// s3.NewClient. The test servers ignore it.
+const s3TestRegion = "us-east-1"
+
 // newS3Storer builds an s3:// backend backed by a fresh in-process
 // S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
 // over httptest) that internal/s3 and the not-found regression test use,
@@ -55,7 +59,7 @@ func newS3Storer(t *testing.T) storage.Storer {
 		Bucket:          s3TestBucket,
 		AccessKeyID:     "test",
 		SecretAccessKey: "test",
-		Region:          "us-east-1",
+		Region:          s3TestRegion,
 	})
 	if err != nil {
 		t.Fatalf("new client: %v", err)
@@ -93,6 +97,68 @@ func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
 	}
 }
 
+// TestS3ListingCutShortIsAnError checks that List and ListStream fail on a
+// listing page marked truncated that carries no continuation token, as a
+// server answering V2 listings V1-style sends it. Taken as the last page,
+// it would hide every later key, and prune would treat the blobs of the
+// snapshots listed there as unreferenced.
+func TestS3ListingCutShortIsAnError(t *testing.T) {
+	t.Parallel()
+
+	const (
+		prefix        = "metadata/"
+		truncatedPage = `<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>test-bucket</Name>
+  <Prefix>metadata/</Prefix>
+  <KeyCount>1</KeyCount>
+  <MaxKeys>1</MaxKeys>
+  <IsTruncated>true</IsTruncated>
+  <Contents>
+    <Key>metadata/snap/manifest.json.zst</Key>
+    <Size>8</Size>
+  </Contents>
+</ListBucketResult>`
+	)
+
+	srv := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, _ *http.Request) {
+			_, _ = io.WriteString(w, truncatedPage)
+		}))
+	t.Cleanup(srv.Close)
+
+	client, err := s3.NewClient(context.Background(), s3.Config{
+		Endpoint:        srv.URL,
+		Bucket:          s3TestBucket,
+		AccessKeyID:     s3TestAccessKeyID,
+		SecretAccessKey: s3TestSecretAccessKey,
+		Region:          s3TestRegion,
+	})
+	if err != nil {
+		t.Fatalf("new client: %v", err)
+	}
+
+	storer := storage.NewS3Storer(client)
+	ctx := context.Background()
+
+	keys, err := storer.List(ctx, prefix)
+	if !errors.Is(err, s3.ErrNoContinuationToken) {
+		t.Errorf("List = %q, %v; want ErrNoContinuationToken", keys, err)
+	}
+
+	var streamErr error
+
+	for obj := range storer.ListStream(ctx, prefix) {
+		if obj.Err != nil {
+			streamErr = obj.Err
+		}
+	}
+
+	if !errors.Is(streamErr, s3.ErrNoContinuationToken) {
+		t.Errorf("ListStream error = %v, want ErrNoContinuationToken", streamErr)
+	}
+}
+
 // TestS3URLPrefixKeyLayout pins the bucket keys an s3:// URL reads and
 // writes: the README's remote storage layout, with the prefix joined to
 // each key by one "/". s3://b/p and s3://b/p/ must be the same

internal/vaultik/nuke_remote_test.go

@@ -5,12 +5,14 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
+	"sneak.berlin/go/vaultik/internal/storage/faultstore"
 )
 
 // TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file
@@ -23,7 +25,7 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 
 	ctx := context.Background()
 	storeDir := filepath.Join(t.TempDir(), "store")
-	v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
+	v, repos, out := backUpToFileDestination(ctx, t, storeDir)
 
 	snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
 	require.NoError(t, err)
@@ -61,4 +63,82 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 		})
 	require.NoError(t, err)
 	assert.Empty(t, files)
+
+	// Nuke deletes the blobs itself, so advice to run prune is wrong.
+	assert.Contains(t, out.String(), "Backup destination store is now empty.")
+	assert.NotContains(t, out.String(), "vaultik prune")
+}
+
+// TestNukeRemoteFailsWhenADeleteFails checks that remote nuke returns an
+// error when a snapshot's metadata or a blob cannot be deleted, does not
+// report the store empty, and advises running nuke again: prune deletes
+// no metadata.
+func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	for _, refusedPrefix := range []string{"metadata/", "blobs/"} {
+		t.Run(refusedPrefix, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			storeDir := filepath.Join(t.TempDir(), "store")
+			v, _, out := backUpToFileDestination(ctx, t, storeDir)
+
+			store := faultstore.New(v.Storage)
+			store.OnDelete = func(key string) bool {
+				return strings.HasPrefix(key, refusedPrefix)
+			}
+			v.Storage = store
+
+			require.Error(t, v.NukeRemote(true))
+
+			assert.NotContains(t, out.String(), "now empty")
+			assert.Contains(t, out.String(), "vaultik remote nuke --force")
+			assert.NotContains(t, out.String(), "vaultik prune")
+		})
+	}
+}
+
+// TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice checks that
+// remote nuke, when the local index belongs to a different destination
+// than the configured one, fails without advising to run nuke again:
+// that run would fail the same way.
+func TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := filepath.Join(t.TempDir(), "store")
+	v, _, out := backUpToFileDestination(ctx, t, storeDir)
+
+	v.Config.StorageURL = "file://" + filepath.Join(t.TempDir(), "elsewhere")
+
+	err := v.NukeRemote(true)
+
+	require.ErrorContains(t, err, "bound to a different backup destination")
+	assert.NotContains(t, out.String(), "vaultik remote nuke --force")
+}
+
+// TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved checks that a
+// snapshot remote nuke cannot remove from the local index fails the
+// run: nuke then leaves that snapshot's metadata on the destination
+// store.
+func TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := filepath.Join(t.TempDir(), "store")
+	v, _, out := backUpToFileDestination(ctx, t, storeDir)
+
+	_, err := v.DB.Conn().ExecContext(ctx, `
+		CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
+		BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
+	require.NoError(t, err)
+
+	require.Error(t, v.NukeRemote(true))
+
+	assert.NotContains(t, out.String(), "now empty")
+	assert.Contains(t, out.String(), "vaultik remote nuke --force")
 }

internal/vaultik/prune.go

@@ -19,6 +19,14 @@ type PruneOptions struct {
 var errNukeRequiresForce = errors.New(
 	"nuke requires --force (this deletes ALL remote snapshots and blobs)")
 
+// errBlobDeleteFailed reports unreferenced blobs prune could not delete.
+var errBlobDeleteFailed = errors.New("blob(s) could not be deleted")
+
+// nukeCommandHint is the command suggested when a remote nuke could not
+// delete everything: running it again deletes what is left, which
+// `vaultik prune` does not do for snapshot metadata.
+const nukeCommandHint = "vaultik remote nuke --force"
+
 // metadataDirName is the top-level remote directory holding snapshot
 // metadata.
 const metadataDirName = "metadata"
@@ -39,6 +47,29 @@ func (v *Vaultik) NukeRemote(force bool) error {
 		return errNukeRequiresForce
 	}
 
+	// Checked before anything is deleted: running nuke again would fail
+	// the same way, so the advice below must not follow this error.
+	err := v.EnsureStorageBinding()
+	if err != nil {
+		return err
+	}
+
+	err = v.emptyDestinationStore()
+	if err != nil {
+		v.UI.Warningf("Run '%s' again to delete what is left on the "+
+			"backup destination store.", nukeCommandHint)
+
+		return err
+	}
+
+	v.UI.Completef("Backup destination store is now empty.")
+
+	return nil
+}
+
+// emptyDestinationStore does the deleting for NukeRemote and stops at
+// the first step that fails.
+func (v *Vaultik) emptyDestinationStore() error {
 	v.UI.Beginf("Removing all snapshot metadata from backup destination store.")
 
 	_, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true})
@@ -62,8 +93,6 @@ func (v *Vaultik) NukeRemote(force bool) error {
 		}
 	}
 
-	v.UI.Completef("Backup destination store is now empty.")
-
 	return nil
 }
 
@@ -166,14 +195,17 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
 
 	if opts.JSON {
-		return v.outputPruneBlobsJSON(result)
+		err = v.outputPruneBlobsJSON(result)
+		if err != nil {
+			return err
+		}
+	} else {
+		v.stdoutf("\nDeleted %d blob(s) totaling %s\n",
+			result.BlobsDeleted, ubytes(result.BytesFreed))
 	}
 
-	v.stdoutf("\nDeleted %d blob(s) totaling %s\n",
-		result.BlobsDeleted, ubytes(result.BytesFreed))
-
 	if result.BlobsFailed > 0 {
-		v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed)
+		return fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed)
 	}
 
 	return nil

internal/vaultik/prune_delete_failure_test.go

@@ -0,0 +1,98 @@
+package vaultik_test
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage/faultstore"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// refuseBlobDeletes is an OnDelete hook that fails every blob delete.
+func refuseBlobDeletes(key string) bool {
+	return strings.HasPrefix(key, "blobs/")
+}
+
+// TestPruneBlobs_FailedDeleteReturnsError checks that prune returns an
+// error, so the command exits non-zero, when an unreferenced blob cannot
+// be deleted.
+func TestPruneBlobs_FailedDeleteReturnsError(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newListEnv(t)
+
+	// The store holds no manifest, so nothing references this blob.
+	addBlob(t, env.store.testStorer, testBlobHashA)
+
+	store := faultstore.New(env.store)
+	store.OnDelete = refuseBlobDeletes
+	env.v.Storage = store
+
+	err := env.v.PruneBlobs(&vaultik.PruneOptions{Force: true})
+
+	require.ErrorContains(t, err, "1 blob(s) could not be deleted")
+}
+
+// TestPruneBlobs_JSONFailedDeleteReturnsError checks that prune --json
+// prints its document and then returns an error when an unreferenced
+// blob cannot be deleted.
+func TestPruneBlobs_JSONFailedDeleteReturnsError(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newListEnv(t)
+	addBlob(t, env.store.testStorer, testBlobHashA)
+
+	store := faultstore.New(env.store)
+	store.OnDelete = refuseBlobDeletes
+	env.v.Storage = store
+
+	err := env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true})
+
+	require.Error(t, err, "prune must fail when a blob cannot be deleted")
+
+	var result vaultik.PruneBlobsResult
+
+	require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result),
+		"stdout must hold the JSON document, got:\n%s", env.stdout.String())
+	assert.Equal(t, 1, result.BlobsFailed)
+	assert.Equal(t, 0, result.BlobsDeleted)
+}
+
+// TestCreateSnapshotPrune_FailedBlobDeleteReturnsError checks that the
+// prune step of snapshot create --prune fails the run when an
+// unreferenced blob cannot be deleted.
+func TestCreateSnapshotPrune_FailedBlobDeleteReturnsError(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := filepath.Join(t.TempDir(), "store")
+	v, _, _ := backUpToFileDestination(ctx, t, storeDir)
+
+	// No snapshot references this blob, so the prune step deletes it.
+	orphan := filepath.Join(storeDir, "blobs", testBlobHashA[:2],
+		testBlobHashA[2:4], testBlobHashA)
+	require.NoError(t, os.MkdirAll(filepath.Dir(orphan), 0o750))
+	require.NoError(t, os.WriteFile(orphan, []byte("orphan"), 0o600))
+
+	store := faultstore.New(v.Storage)
+	store.OnDelete = refuseBlobDeletes
+	v.Storage = store
+
+	err := v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+		Cron:  true,
+		Prune: true,
+	})
+
+	require.ErrorContains(t, err, "1 blob(s) could not be deleted")
+	assert.FileExists(t, orphan)
+}

internal/vaultik/remove_snapshot_test.go

@@ -476,7 +476,8 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) {
 	out := tv.Stdout.String()
 	assert.Contains(t, out, "Removed 2 snapshot(s)")
 	assert.Contains(t, out, "Removed snapshot metadata from remote storage")
-	assert.Contains(t, out, "vaultik prune")
+	// Remote nuke, the only caller, deletes the blobs next itself.
+	assert.NotContains(t, out, "vaultik prune")
 }
 
 func TestRemoveAllSnapshots_DryRun(t *testing.T) {

internal/vaultik/restore.go

@@ -702,6 +702,8 @@ func (v *Vaultik) downloadSnapshotDB(
 		return nil, "", fmt.Errorf("downloading %s: %w", dbKey, err)
 	}
 
+	reader = newCloseOnCancelReader(v.ctx, reader)
+
 	defer func() { _ = reader.Close() }()
 
 	// Decrypt and decompress straight from the storage stream, then stream

internal/vaultik/restore_interrupt_test.go

@@ -207,30 +207,37 @@ func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
 		"the cancel was reported as a failed file")
 }
 
-// stallingBlobStorer wraps a Storer so that every blob download hands
-// over the first half of the blob and then stalls, ignoring the context,
-// as a file:// blob on a hung USB stick or network share does. Only
-// closing the returned reader ends the stall.
-type stallingBlobStorer struct {
+// stallingStorer wraps a Storer so that each download of a key chosen by
+// stalls hands over the first half of the object and then stalls,
+// ignoring the context, as a file:// read on a hung USB stick or network
+// share does. Only closing the returned reader ends the stall.
+type stallingStorer struct {
 	storage.Storer
 
+	stalls  func(key string) bool
 	once    sync.Once
 	stalled chan struct{}
 }
 
-func newStallingBlobStorer(inner storage.Storer) *stallingBlobStorer {
-	return &stallingBlobStorer{Storer: inner, stalled: make(chan struct{})}
+func newStallingStorer(
+	inner storage.Storer, stalls func(key string) bool,
+) *stallingStorer {
+	return &stallingStorer{
+		Storer:  inner,
+		stalls:  stalls,
+		stalled: make(chan struct{}),
+	}
 }
 
-func (s *stallingBlobStorer) Get(
+func (s *stallingStorer) Get(
 	ctx context.Context, key string,
 ) (io.ReadCloser, error) {
 	rc, err := s.Storer.Get(ctx, key)
-	if err != nil || !strings.HasPrefix(key, "blobs/") {
+	if err != nil || !s.stalls(key) {
 		return rc, err
 	}
 
-	blob, err := io.ReadAll(rc)
+	object, err := io.ReadAll(rc)
 	_ = rc.Close()
 
 	if err != nil {
@@ -242,7 +249,7 @@ func (s *stallingBlobStorer) Get(
 	pr, pw := io.Pipe()
 
 	go func() {
-		_, _ = pw.Write(blob[:len(blob)/2])
+		_, _ = pw.Write(object[:len(object)/2])
 
 		s.once.Do(func() { close(s.stalled) })
 	}()
@@ -250,17 +257,26 @@ func (s *stallingBlobStorer) Get(
 	return pr, nil
 }
 
-// TestInterruptDuringStalledBlobReadCleansTempDir stops a restore and a
-// deep verify while a blob read is stalled on a source that ignores the
-// context. Stopping must end that read, so the operation unwinds and
-// removes its decrypted scratch files well within the shutdown timeout
-// (issue #325).
+// TestInterruptDuringStalledReadCleansTempDir stops a restore and a deep
+// verify while a read of a blob or of the snapshot database is stalled
+// on a source that ignores the context. Stopping must end that read, so
+// the operation unwinds and removes its decrypted scratch files well
+// within the shutdown timeout (issues #325 and #356).
 //
 // Not parallel: it points TMPDIR at a private directory (via t.Setenv)
 // so it can assert on exactly the scratch files the operation created.
-func TestInterruptDuringStalledBlobReadCleansTempDir(t *testing.T) {
+func TestInterruptDuringStalledReadCleansTempDir(t *testing.T) {
 	log.Initialize(log.Config{})
 
+	stalledReads := map[string]func(key string) bool{
+		"blob": func(key string) bool {
+			return strings.HasPrefix(key, "blobs/")
+		},
+		"snapshot database": func(key string) bool {
+			return strings.HasSuffix(key, "/db.zst.age")
+		},
+	}
+
 	operations := map[string]func(v *Vaultik, snapshotID, targetDir string) error{
 		"restore": func(v *Vaultik, snapshotID, targetDir string) error {
 			return v.Restore(&RestoreOptions{
@@ -273,52 +289,58 @@ func TestInterruptDuringStalledBlobReadCleansTempDir(t *testing.T) {
 		},
 	}
 
-	for name, op := range operations {
-		t.Run(name, func(t *testing.T) {
-			fs := afero.NewOsFs()
-			root := t.TempDir()
+	for readName, stalls := range stalledReads {
+		for opName, op := range operations {
+			t.Run(readName+" read during "+opName, func(t *testing.T) {
+				fs := afero.NewOsFs()
+				root := t.TempDir()
 
-			// Random content, so the blob stays large enough to stall
-			// part-way through its compressed, encrypted body.
-			cfg, storer, snapshotID, _ := backupOneFile(context.Background(),
-				t, fs, root, "a.bin", randomBytes(t, 256*1024), 0o644)
+				// Random content, so the blob stays large enough to stall
+				// part-way through its compressed, encrypted body.
+				cfg, storer, snapshotID, _ := backupOneFile(context.Background(),
+					t, fs, root, "a.bin", randomBytes(t, 256*1024), 0o644)
 
-			scratch := filepath.Join(root, "scratch")
-			require.NoError(t, fs.MkdirAll(scratch, 0o755))
-			t.Setenv("TMPDIR", scratch)
+				scratch := filepath.Join(root, "scratch")
+				require.NoError(t, fs.MkdirAll(scratch, 0o755))
+				t.Setenv("TMPDIR", scratch)
 
-			stalling := newStallingBlobStorer(storer)
+				stalling := newStallingStorer(storer, stalls)
 
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
 
-			v := newRestoreVaultik(ctx, cfg, stalling, fs)
-			v.cancel = cancel
+				v := newRestoreVaultik(ctx, cfg, stalling, fs)
+				v.cancel = cancel
 
-			var opErr error
+				var opErr error
 
-			stop := v.StartOperation(func() {
-				opErr = op(v, snapshotID, filepath.Join(root, "restored"))
+				stop := v.StartOperation(func() {
+					opErr = op(v, snapshotID, filepath.Join(root, "restored"))
+				})
+
+				select {
+				case <-stalling.stalled:
+				case <-time.After(30 * time.Second):
+					t.Fatal("the operation never reached the stalled read")
+				}
+
+				// The snapshot database's temp directory is created just
+				// after its read starts, so it can appear after the stall.
+				require.Eventually(t, func() bool {
+					return len(scratchEntries(t, scratch)) > 0
+				}, 10*time.Second, 10*time.Millisecond,
+					"expected decrypted scratch files to exist mid-read")
+
+				stopCtx, stopCancel := context.WithTimeout(
+					context.Background(), 10*time.Second)
+				defer stopCancel()
+
+				require.True(t, stop(stopCtx),
+					"stop timed out; the stalled read was not interrupted")
+				require.ErrorIs(t, opErr, context.Canceled)
+				require.Empty(t, scratchEntries(t, scratch),
+					"decrypted scratch files remained after the interrupt")
 			})
-
-			select {
-			case <-stalling.stalled:
-			case <-time.After(30 * time.Second):
-				t.Fatal("the operation never reached a blob read")
-			}
-
-			require.NotEmpty(t, scratchEntries(t, scratch),
-				"expected decrypted scratch files to exist mid-read")
-
-			stopCtx, stopCancel := context.WithTimeout(
-				context.Background(), 10*time.Second)
-			defer stopCancel()
-
-			require.True(t, stop(stopCtx),
-				"stop timed out; the stalled blob read was not interrupted")
-			require.ErrorIs(t, opErr, context.Canceled)
-			require.Empty(t, scratchEntries(t, scratch),
-				"decrypted scratch files remained after the interrupt")
-		})
+		}
 	}
 }

internal/vaultik/snapshot.go

@@ -25,6 +25,7 @@ var (
 	errSnapshotVerifyFailed = errors.New("verification failed")
 	errRemoveAllNeedsForce  = errors.New("--all requires --force")
 	errSnapshotNotFound     = errors.New("snapshot not found")
+	errSnapshotRemoveFailed = errors.New("snapshot(s) could not be removed")
 	errInvalidTableName     = errors.New("invalid table name")
 	errNoAgeRecipients      = errors.New(
 		"creating a snapshot needs at least one public key in " +
@@ -1591,8 +1592,8 @@ func (v *Vaultik) handleRemoveAllDryRun(
 
 // executeRemoveAll deletes every local snapshot and, unless LocalOnly
 // is set, every corresponding remote metadata directory plus any
-// orphan remote keys. Blobs are not touched; the printed prune-command
-// hint is the next step.
+// orphan remote keys. Blobs are not touched. It returns an error when
+// any snapshot could not be removed.
 func (v *Vaultik) executeRemoveAll(
 	localSnaps, orphanRemoteKeys []string, opts *RemoveOptions,
 ) (*RemoveResult, error) {
@@ -1606,18 +1607,11 @@ func (v *Vaultik) executeRemoveAll(
 
 	result := &RemoveResult{}
 
-	remoteErrors := v.removeAllLocalSnapshots(localSnaps, opts, result)
+	failures := v.removeAllLocalSnapshots(localSnaps, opts, result)
 
 	if !opts.LocalOnly {
-		remoteErrors += v.removeAllOrphanRemotes(orphanRemoteKeys, result)
-
-		if remoteErrors == 0 {
-			result.RemoteRemoved = true
-		} else if v.UI != nil {
-			v.UI.Warningf("Some remote metadata deletions failed. Run '%s' "+
-				"once the remote is healthy to clean up unreferenced blobs.",
-				pruneCommandHint)
-		}
+		failures += v.removeAllOrphanRemotes(orphanRemoteKeys, result)
+		result.RemoteRemoved = failures == 0
 	}
 
 	if v.SnapshotManager != nil {
@@ -1627,6 +1621,10 @@ func (v *Vaultik) executeRemoveAll(
 		}
 	}
 
+	if failures > 0 {
+		return result, fmt.Errorf("%d %w", failures, errSnapshotRemoveFailed)
+	}
+
 	if opts.JSON {
 		return result, v.outputRemoveJSON(result)
 	}
@@ -1635,9 +1633,6 @@ func (v *Vaultik) executeRemoveAll(
 
 	if !opts.LocalOnly && result.RemoteRemoved {
 		v.printlnStdout("Removed snapshot metadata from remote storage")
-		v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. "+
-			"Run '%s' to delete any blobs no longer referenced by any "+
-			"remaining remote snapshot.\n", pruneCommandHint)
 	}
 
 	return result, nil
@@ -1680,12 +1675,13 @@ func (v *Vaultik) deleteIncompleteSnapshot(snapshotIDStr string, id any) bool {
 }
 
 // removeAllLocalSnapshots removes each known local snapshot (and, unless
-// --local-only, its remote metadata), returning the number of remote
-// deletion failures.
+// --local-only, its remote metadata), returning the number of snapshots
+// it could not remove. A snapshot left in the local index keeps its
+// remote metadata too, since it is not an orphan remote key.
 func (v *Vaultik) removeAllLocalSnapshots(
 	localSnaps []string, opts *RemoveOptions, result *RemoveResult,
 ) int {
-	remoteErrors := 0
+	failures := 0
 
 	for _, snapshotID := range localSnaps {
 		log.Info("Removing snapshot", "snapshot_id", snapshotID)
@@ -1695,6 +1691,8 @@ func (v *Vaultik) removeAllLocalSnapshots(
 			log.Error("Failed to remove from local database",
 				"snapshot_id", snapshotID, "error", err)
 
+			failures++
+
 			continue
 		}
 
@@ -1705,14 +1703,14 @@ func (v *Vaultik) removeAllLocalSnapshots(
 				log.Warn("Failed to remove snapshot metadata from remote",
 					"snapshot_id", snapshotID, "error", err)
 
-				remoteErrors++
+				failures++
 			}
 		}
 
 		result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID)
 	}
 
-	return remoteErrors
+	return failures
 }
 
 // removeAllOrphanRemotes deletes remote snapshot metadata that has no

internal/vaultik/verify.go

@@ -260,6 +260,8 @@ func (v *Vaultik) downloadVerifiedSnapshotDB(
 			fmt.Errorf("failed to download database: %w", err))
 	}
 
+	dbReader = newCloseOnCancelReader(v.ctx, dbReader)
+
 	defer func() { _ = dbReader.Close() }()
 
 	tdb, err := v.decryptAndLoadDatabase(dbReader, identities)