Report blobs shallow verify could not check apart from missing ones (closes #318)

Shallow snapshot verify counted every error from checking a blob or the
encrypted database as "missing" and printed the error nowhere, so a
permission error or an S3 403 read as data loss. Now only a not-found
answer from storage counts as missing. Any other error is reported as
could not be checked, with the error storage returned, in the text
report and under --json (the new unchecked list and
database_check_error field), and the run fails. Under --quiet, which
drops the report, each of those errors is logged as a warning on
stderr.

The database check moved into its own small function because the verify
function was at the linter's length limit.

Model: opus-5-5

ARCHITECTURE.md

@@ -279,8 +279,8 @@ Key methods:
 - `Scan(ctx, path, snapshotID)` → Main scan operation
 
 Scan phases:
-1. **Phase 1**: Walk directory, identify files needing processing
+1. **Phase 0**: Detect deleted files from previous snapshots
-2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists, the walk did not find, and are gone from disk; a file that is excluded, or inside an excluded directory, is not counted
+2. **Phase 1**: Walk directory, identify files needing processing
 3. **Phase 2**: Process files (chunk → pack → upload)
 
 #### SnapshotManager
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
     │       │
     │       ├─► scanner.Scan(ctx, path, snapshotID)
     │       │       │
+    │       │       ├─► Phase 0: detectDeletedFiles()
+    │       │       │
     │       │       ├─► Phase 1: scanPhase()
     │       │       │       Walk directory
     │       │       │       Check file metadata changes
     │       │       │       Build list of files to process
     │       │       │
-    │       │       ├─► Phase 1b: detectDeletedFiles()
-    │       │       │
     │       │       └─► Phase 2: processPhase()
     │       │               For each file:
     │       │                   chunker.ChunkReaderStreaming()

Dockerfile

@@ -5,16 +5,7 @@
 FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
 WORKDIR /src
 COPY go.mod go.sum ./
-# Without the time limit a stalled connection to the module proxy holds
+RUN go mod download
-# this step, and every build on the host sharing it, until the kernel
-# drops the connection, which can take an hour. The limit is well above
-# the three minutes a working download can take on a busy build host.
-RUN for attempt in 1 2 3; do \
-        timeout 300 go mod download && exit 0; \
-        echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
-    done; \
-    echo "module download failed: giving up after 3 attempts" >&2; \
-    exit 1
 COPY . .
 # `golangci-lint run` silently ignores an unknown top-level key in
 # .golangci.yml, such as a misspelt `linters:`; `config verify` fails on it.
@@ -27,13 +18,7 @@ RUN golangci-lint run --config .golangci.yml ./...
 FROM golang:1.26.1@sha256:cd78d88e00afadbedd272f977d375a6247455f3a4b1178f8ae8bbcb201743a8a AS test
 WORKDIR /src
 COPY go.mod go.sum ./
-# Limited and retried as in the lint phase above.
+RUN go mod download
-RUN for attempt in 1 2 3; do \
-        timeout 300 go mod download && exit 0; \
-        echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
-    done; \
-    echo "module download failed: giving up after 3 attempts" >&2; \
-    exit 1
 COPY . .
 RUN go test -timeout 90s -race -cover ./... || \
     { echo "--- Rerunning with -v for details ---"; \
@@ -51,13 +36,7 @@ RUN apk add --no-cache git
 RUN git config --system --add safe.directory /src
 WORKDIR /src
 COPY go.mod go.sum ./
-# Limited and retried as in the lint phase above.
+RUN go mod download
-RUN for attempt in 1 2 3; do \
-        timeout 300 go mod download && exit 0; \
-        echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
-    done; \
-    echo "module download failed: giving up after 3 attempts" >&2; \
-    exit 1
 COPY . .
 
 # The VERSION build arg when one is given, otherwise

README.md

@@ -271,17 +271,8 @@ existing file. Created with mode `0600` since it will contain credentials.
 **`config set`**: Set a scalar config value by dotted YAML path
 (e.g. `vaultik config set compression_level 9`,
 `vaultik config set storage_url "file:///mnt/backups"`). Comments and
-formatting in the file are preserved. Missing keys along the path are
+formatting in the file are preserved; intermediate maps are created as
-created: as a list when the next path element is a number (so
+needed.
-`vaultik config set age_recipients.0 age1...` works on a file without
-`age_recipients`), otherwise as a map. If the config loads, a change
-after which it would not, such as `vaultik config set compression_level 99`,
-is refused and the file is left unchanged. A config that does not load,
-such as the one `config init` writes before `storage_url` is set or one
-with a `${ENV:...}` reference to a variable not set in this shell, is
-checked one setting at a time: only a change after which the changed
-setting cannot be read, such as one giving it the wrong type (a single
-value where a list belongs or a word where a number belongs), is refused.
 
 **`snapshot create`**: Perform incremental backup of configured snapshots.
 * Optional snapshot names argument to create specific snapshots (default: all)
@@ -303,8 +294,7 @@ value where a list belongs or a word where a number belongs), is refused.
 of the local index and the backup destination store — with timestamps
 and three sizes per snapshot (compressed remote size; total
 uncompressed chunk size; size of the chunks that snapshot stored that
-were not stored before). Both chunk sizes are recorded when the
+were not stored before, recorded when its backup ran).
-snapshot's backup runs.
 
 Listing the destination store does **not** require the age secret key,
 so it works in vaultik's intended configuration, where the backed-up
@@ -323,11 +313,7 @@ read from the manifest.
 
 Snapshots in the local index with no counterpart on the destination
 store are reported below the table as drift, with the `vaultik prune`
-invocation that reconciles them. If the local index is bound to a
+invocation that reconciles them.
-different destination than the configured one (after a change to
-`storage_url`, or `s3.endpoint`, `s3.bucket` or `s3.prefix`), the
-command says so instead and gives the same two ways out that `prune`
-gives: revert the configuration, or run `vaultik database delete`.
 
 If the destination store cannot be listed (unmounted volume,
 permission denied, network down), the command warns, falls back to the
@@ -336,11 +322,9 @@ local index alone, and still exits zero.
   (whether the snapshot is in the local index), `remote_key` (the full
   64-character storage key), and `remote_present` (whether it was seen
   on the destination store, or `null` if the destination could not be
-  listed). Warnings about an unlistable destination, a local index
+  listed). Warnings about an unlistable destination, unreadable
-  bound to a different destination (with both destinations and the
+  manifests, and a truncated listing all go to stderr through the
-  same two ways out), unreadable manifests, and a truncated listing all
+  logger, so stdout stays a single parseable document.
-  go to stderr through the logger, so stdout stays a single parseable
-  document.
 
 **`snapshot verify`**: Verify snapshot integrity.
 * Default (shallow): checks that every blob the manifest lists is present in
@@ -366,20 +350,16 @@ snapshot from the local index and strips the snapshot's metadata from
 the backup destination store. Blobs are NOT touched — deleting blobs
 requires reading every remaining remote manifest (the destination store
 may hold snapshots this host doesn't know about), which is what
-`vaultik prune` does. The command prints the `vaultik prune` invocation
+`vaultik prune` does. On success the command prints the exact `vaultik
-to run as a follow-up only when it removed the snapshot's metadata from
+prune` invocation to run as a follow-up. An ID that names no snapshot in
-the destination store, and never under `--json`. An ID that names no
+the local index or on the destination store is an error; with
-snapshot in the local index or on the destination store is an error;
+`--local-only`, the snapshot must be in the local index. Local row
-with `--local-only`, the snapshot must be in the local index. Local row
 cleanup (files, chunks, blobs the snapshot was the last referrer for)
-runs automatically. If removing the snapshot's metadata from the
+runs automatically. If the destination store is unreachable, the local-DB
-destination store fails (for example because the store is unreachable),
+removal still completes and a warning is emitted; run `vaultik snapshot
-the local-DB removal still completes and a warning is emitted; run
+remove <snapshot-id>` again once the store is reachable to remove the
-`vaultik snapshot remove <snapshot-id>` again once the store is reachable
+snapshot's metadata from it (`vaultik prune` does not). To wipe everything
-to remove the snapshot's metadata from it (`vaultik prune` does not). For
+on the destination in one go, use `vaultik remote nuke --force`.
-a snapshot that is only on the destination store, that failure is an
-error. To wipe everything on the destination in one go, use `vaultik
-remote nuke --force`.
 * `--local-only`: Skip remote cleanup; only touch the local index
 * `--dry-run`: Show what would be deleted without deleting
 * `--force`: Skip confirmation prompt
@@ -392,14 +372,10 @@ remote nuke --force`.
   store — its remote key (or an unambiguous leading part of it) as shown
   by `snapshot list`. See
   [restoring on another machine](#restoring-on-another-machine).
-* Optional path arguments to restore specific files/directories (default: all).
+* Optional path arguments to restore specific files/directories (default: all)
-  A path that matches nothing in the snapshot is named in a warning, and the
-  command exits non-zero after restoring the paths that match.
 * Preserves file permissions, timestamps, ownership (ownership requires root),
   symlinks, and empty directories
-* `--verify`: After restoring, verify every restored file's chunk hashes
+* `--verify`: After restoring, verify every file's chunk hashes match
-  match. A file skipped under `--skip-errors` is reported as failed to
-  restore and is not verified.
 
 **`prune`**: Tidy up everything that isn't needed. Runs three passes:
 (1) reconcile the local index against the destination store — any
@@ -410,12 +386,10 @@ manifest on the destination store to compute the still-referenced blob
 set and delete any blob not in that set. Step (3) reads all remote
 manifests — network cost scales with the number of snapshots. `snapshot
 create --prune` runs the same cleanup automatically; this is the
-manual entry point for the same work. If a blob cannot be deleted, the
+manual entry point for the same work.
-command (and `snapshot create --prune`) exits non-zero.
 * `--force`: Skip confirmation prompt
 * `--json`: Output stats as JSON. Also skips the confirmation prompt, as
-  `--force` does. A blob that cannot be deleted is counted in
+  `--force` does.
-  `blobs_failed`, and the document is printed before the non-zero exit.
 
 **`info`**: Display system configuration, storage settings, encryption
 recipients, and local database statistics.
@@ -434,12 +408,10 @@ under `blobs/` whose name is not a blob hash is not counted as a blob,
 since `prune` never deletes it.
 * `--json`: Output as JSON
 
-**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on
+**`remote nuke`**: Delete every snapshot's metadata and every blob from the
-the backup destination store, whatever its name, leaving the bucket prefix
+backup destination store, leaving the bucket prefix empty. Destructive and
-empty. Destructive and irreversible. This is the single supported way to
+irreversible. This is the single supported way to wipe the entire
-wipe the entire destination store. If a listing afterwards still finds
+destination store.
-anything there, the command names each object left and exits non-zero;
-running it again deletes what is left.
 * `--force`: Required to confirm destruction.
 
 **`database delete`**: Delete the local SQLite state database file
@@ -722,10 +694,8 @@ the `NO_COLOR` environment variable is unset (https://no-color.org/).
   reports and tables (`version`, `info`, `remote info`, `snapshot
   verify`, `snapshot list`) but never the machine-consumed `config get`
   value or the `--json` documents, which a script depends on. The
-  confirmation prompts of `database delete`, `prune`, `snapshot purge`
+  `database delete` confirmation prompt is also written this way and
-  and `snapshot remove`, each with what it asks to delete, are also
+  always shown: it is an interactive exchange the operator must see.
-  written this way and always shown: they are an interactive exchange
-  the operator must see.
 
 `internal/ui` writes to stdout; it is the output the user asked for. The
 exceptions are the startup banner and the error a failed command ends

cmd/vaultik/dockermoddownload_test.go

@@ -1,45 +0,0 @@
-package main_test
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestDockerfileBoundsEveryModuleDownload fails unless every
-// `go mod download` in the Dockerfile runs under a time limit, is tried
-// a fixed number of times, and then fails the build with a message that
-// names the module download (issue #351). Like the version tests, this
-// parses the Dockerfile; that a stalled download really is cut off and
-// retried was checked by hand.
-func TestDockerfileBoundsEveryModuleDownload(t *testing.T) {
-	t.Parallel()
-
-	var downloads []string
-
-	for _, instruction := range instructions(t, productDockerfile) {
-		if strings.Contains(instruction, "go mod download") {
-			downloads = append(downloads, instruction)
-		}
-	}
-
-	require.NotEmpty(t, downloads, "%s must download modules",
-		productDockerfile)
-
-	for _, download := range downloads {
-		assert.Contains(t, download, "timeout 300 go mod download",
-			"%s: a module download must have a time limit: %s",
-			productDockerfile, download)
-		assert.Contains(t, download, "for attempt in 1 2 3",
-			"%s: a module download must be retried: %s",
-			productDockerfile, download)
-		assert.Contains(t, download, "module download failed",
-			"%s: a module download that never succeeds must say so: %s",
-			productDockerfile, download)
-		assert.True(t, strings.HasSuffix(download, "exit 1"),
-			"%s: a module download that never succeeds must fail the"+
-				" build: %s", productDockerfile, download)
-	}
-}

config.example.yml

@@ -302,8 +302,7 @@ storage_url: "rclone://myremote/path/to/backups"
 
 # Average chunk size for content-defined chunking
 # Smaller chunks = better deduplication but more metadata
-# Minimum 1MB
+# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
-# Supports: 10MB, 5M, 1GB, 64MiB, etc.
 # Default: 10MB
 #chunk_size: 10MB
 

docs/DATAMODEL.md

@@ -90,7 +90,7 @@ Stores information about packed, compressed, and encrypted blob files.
 - `created_ts` (INTEGER NOT NULL) - Creation timestamp
 - `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress)
 - `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before compression
-- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption. Once uploaded, the size of the object in storage: when the blob was already stored, under another compression level or other recipients, this can differ from the size of the local copy
+- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption
 - `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded)
 
 **Purpose:** Tracks blob lifecycle from creation through upload. The UUID primary key allows immediate association of chunks with blobs.
@@ -122,7 +122,6 @@ Tracks backup snapshots.
 - `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE
 - `blob_count` (INTEGER) - Number of blobs this snapshot created
 - `total_size` (INTEGER) - Total size of all files
-- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the distinct chunks the snapshot's files reference, recorded when its backup ran; `snapshot list` shows it as UNCOMPRESSED SIZE
 - `blob_size` (INTEGER) - Total compressed size of all referenced blobs
 - `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
 - `compression_ratio` (REAL) - Compression ratio achieved
@@ -185,7 +184,7 @@ Tracks blob upload metrics.
    - `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after packing
 
 4. **Upload**
-   - `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as uploaded, with the size of the object in storage (the upload is skipped when the blob is already stored)
+   - `UPDATE blobs SET uploaded_ts = ?` - Mark blob as uploaded
    - `INSERT INTO uploads` - Record upload metrics with snapshot_id
    - `INSERT INTO snapshot_blobs` - Associate blob with snapshot
 

go.mod

@@ -22,7 +22,6 @@ require (
 	github.com/spf13/cobra v1.10.1
 	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1
-	go.uber.org/dig v1.19.0
 	go.uber.org/fx v1.24.0
 	golang.org/x/sync v0.18.0
 	golang.org/x/sys v0.38.0
@@ -257,6 +256,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
 	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
+	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect

internal/cli/app.go

@@ -8,12 +8,12 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/adrg/xdg"
 	"github.com/spf13/cobra"
-	"go.uber.org/dig"
 	"go.uber.org/fx"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
@@ -146,12 +146,16 @@ func (e *startupError) Error() string {
 //	could not build arguments for function "Y" (file:line): failed to build U:
 //	received non-nil error from function "Z" (file:line): <real error>
 //
-// Users care about the real error, not the DI plumbing. dig.RootCause,
+// Users care about the real error, not the DI plumbing. We strip everything
-// from the library fx is built on, unwraps those layers and returns the
+// up through the last "): " (which is always the close-paren of an fx
-// real error whole. Cutting the text at "): " instead would also cut a
+// function-location annotation followed by the wrapped error).
-// real error that contains it, such as the blob_size_limit one.
 func cleanStartupError(err error) error {
-	return &startupError{msg: dig.RootCause(err).Error()}
+	msg := err.Error()
+	if idx := strings.LastIndex(msg, "): "); idx >= 0 {
+		msg = msg[idx+3:]
+	}
+
+	return &startupError{msg: msg}
 }
 
 // RunApp starts the fx application, blocks until it is asked to stop, and

internal/cli/app_test.go

@@ -2,65 +2,58 @@ package cli //nolint:testpackage // needs access to unexported cleanStartupError
 
 import (
 	"errors"
-	"os"
 	"path/filepath"
 	"testing"
 
-	"go.uber.org/fx"
-	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/pidlock"
 )
 
-// TestCleanStartupError starts a real fx app whose config has a
-// blob_size_limit below the minimum. fx puts annotations naming the
-// functions it was building in front of the config error; only those
-// may be removed, although the config error itself contains "): ".
 func TestCleanStartupError(t *testing.T) {
 	t.Parallel()
 
-	path := filepath.Join(t.TempDir(), "config.yml")
+	tests := []struct {
-
+		name string
-	err := os.WriteFile(path, []byte("snapshots:\n"+
+		in   string
-		"  test:\n"+
+		want string
-		"    paths: [/tmp/vaultik-test-source]\n"+
+	}{
-		"storage_url: file:///tmp/vaultik-test-storage\n"+
+		{
-		"chunk_size: 10MB\n"+
+			name: "real fx error chain",
-		"blob_size_limit: 30MB\n"), 0o600)
+			in: `could not build arguments for function ` +
-	if err != nil {
+				`"sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 ` +
-		t.Fatalf("write config: %v", err)
+				`(/Users/user/dev/vaultik/internal/cli/snapshot.go:71): ` +
+				`failed to build *vaultik.Vaultik: ` +
+				`could not build arguments for function ` +
+				`"sneak.berlin/go/vaultik/internal/vaultik".New ` +
+				`(/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): ` +
+				`failed to build storage.Storer: ` +
+				`received non-nil error from function ` +
+				`"sneak.berlin/go/vaultik/internal/storage".NewStorer ` +
+				`(/Users/user/dev/vaultik/internal/storage/module.go:23): ` +
+				`creating base path: mkdir /Volumes/BACKUPS: permission denied`,
+			want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`,
+		},
+		{
+			name: "no fx wrapping",
+			in:   "plain error",
+			want: "plain error",
+		},
+		{
+			name: "single fx wrapping",
+			in:   `received non-nil error from function "foo" (file.go:1): underlying problem`,
+			want: "underlying problem",
+		},
 	}
 
-	app := fx.New(
+	for _, tt := range tests {
-		fx.NopLogger,
+		t.Run(tt.name, func(t *testing.T) {
-		fx.Supply(config.Path(path)),
+			t.Parallel()
-		config.Module,
-		fx.Invoke(func(*config.Config) {}),
-	)
 
-	err = app.Start(t.Context())
+			//nolint:err113 // test constructs errors from table input
-	if err == nil {
+			got := cleanStartupError(errors.New(tt.in)).Error()
-		t.Fatal("app started; want a blob_size_limit error")
+			if got != tt.want {
-	}
+				t.Errorf("got %q, want %q", got, tt.want)
-
+			}
-	want := "failed to load config: invalid config: blob_size_limit must " +
+		})
-		"be at least the largest chunk the chunker can emit (chunk_size " +
-		"times the FastCDC size spread): need at least 40000000 bytes"
-
-	got := cleanStartupError(err).Error()
-	if got != want {
-		t.Errorf("got %q, want %q", got, want)
-	}
-}
-
-// TestCleanStartupErrorNotFromFx checks that an error fx did not wrap is
-// shown unchanged.
-func TestCleanStartupErrorNotFromFx(t *testing.T) {
-	t.Parallel()
-
-	//nolint:err113 // a one-off error standing in for a startup failure
-	got := cleanStartupError(errors.New("plain error")).Error()
-	if got != "plain error" {
-		t.Errorf("got %q, want %q", got, "plain error")
 	}
 }
 

internal/cli/config.go

@@ -8,7 +8,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"reflect"
-	"slices"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -45,9 +44,6 @@ var (
 	errNeedNumericIndex = errors.New("key is a list; use a numeric index")
 	errIndexOutOfRange  = errors.New("index out of range")
 	errNotMapOrList     = errors.New("key is not a map or list")
-	errWouldNotLoad     = errors.New(
-		"vaultik would not load the config after this change, " +
-			"so the file was not changed")
 )
 
 const defaultConfigTemplate = `# vaultik configuration
@@ -224,7 +220,7 @@ storage_url: ""
 
 # Average chunk size for content-defined chunking (FastCDC).
 # Smaller = better deduplication but more metadata overhead.
-# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
+# Accepts: 1MB, 10M, 64KB, etc.
 # Default: 10MB
 # chunk_size: 10MB
 
@@ -333,15 +329,8 @@ func newConfigEditCommand() *cobra.Command {
 				editor = "vi"
 			}
 
-			// Run $EDITOR through the shell, as git does, so a value
-			// with arguments such as "code --wait" works. The path
-			// goes in as an argument ("$@"), not into the command
-			// text, so a path with spaces or quotes arrives whole.
-			// sh -c takes the word after the command as $0, which
-			// is why editor appears twice.
 			//nolint:gosec // G204: launching the operator's own $EDITOR is the point
-			ed := exec.CommandContext(cmd.Context(),
+			ed := exec.CommandContext(cmd.Context(), editor, path)
-				"sh", "-c", editor+` "$@"`, editor, path)
 			ed.Stdin = os.Stdin
 			ed.Stdout = os.Stdout
 			ed.Stderr = os.Stderr
@@ -402,17 +391,8 @@ func newConfigSetCommand() *cobra.Command {
 		Use:   "set <key> <value>",
 		Short: "Set a config value by dotted path (e.g. compression_level 5)",
 		Long: `Sets a scalar config value addressed by dotted YAML path and writes
-the file back, preserving comments and formatting. Missing keys along
+the file back, preserving comments and formatting. Intermediate maps
-the path are created: as a list when the next path element is a
+are created as needed.
-number, otherwise as a map.
-
-If the config loads, a change after which it would not, such as
-compression_level 99, is refused and the file is left unchanged. A
-config that does not load, such as the one config init writes before
-storage_url is set or one with a ${ENV:...} reference to a variable
-not set here, is checked one setting at a time: only a change after
-which the changed setting cannot be read, such as one giving it the
-wrong type, is refused.
 
 Examples:
   vaultik config set storage_url "file:///mnt/backups"
@@ -442,14 +422,7 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 		return err
 	}
 
-	oldData, err := marshalConfigYAML(root)
+	err = yamlPathSet(root, strings.Split(key, "."), value)
-	if err != nil {
-		return fmt.Errorf("marshaling config: %w", err)
-	}
-
-	keys := strings.Split(key, ".")
-
-	err = yamlPathSet(root, keys, value)
 	if err != nil {
 		return err
 	}
@@ -459,21 +432,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 		return fmt.Errorf("marshaling config: %w", err)
 	}
 
-	setting, err := yamlPathOnly(root, keys)
-	if err != nil {
-		return err
-	}
-
-	settingData, err := marshalConfigYAML(setting)
-	if err != nil {
-		return fmt.Errorf("marshaling config: %w", err)
-	}
-
-	err = checkConfigChange(oldData, data, settingData)
-	if err != nil {
-		return fmt.Errorf("%w: %w", errWouldNotLoad, err)
-	}
-
 	err = os.WriteFile(path, data, configFileMode)
 	if err != nil {
 		return fmt.Errorf("writing config file: %w", err)
@@ -495,58 +453,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 	return nil
 }
 
-// checkConfigChange returns the reason vaultik would not load newData, the
-// config after a change, if the change brings that failure in. If oldData,
-// the config before it, loads, the new one must load too. A config that
-// does not load, such as one still being filled in or one with a ${...}
-// reference that cannot be filled in here, is checked one setting at a
-// time: settingData, the changed setting on its own, must decode into the
-// config type.
-func checkConfigChange(oldData, newData, settingData []byte) error {
-	oldCfg, err := config.Parse(oldData)
-	if err == nil && oldCfg.Validate() == nil {
-		newCfg, err := config.Parse(newData)
-		if err != nil {
-			return err
-		}
-
-		return newCfg.Validate()
-	}
-
-	_, err = config.Parse(settingData)
-
-	return err
-}
-
-// yamlPathOnly returns a document holding only the value at the dotted key
-// path in root, inside the maps and lists that lead to it there.
-func yamlPathOnly(root *yaml.Node, keys []string) (*yaml.Node, error) {
-	node, err := yamlPathGet(root, keys)
-	if err != nil {
-		return nil, err
-	}
-
-	for i, key := range slices.Backward(keys) {
-		parent, err := yamlPathGet(root, keys[:i])
-		if err != nil {
-			return nil, err
-		}
-
-		if parent.Kind == yaml.SequenceNode {
-			node = &yaml.Node{
-				Kind: yaml.SequenceNode, Content: []*yaml.Node{node},
-			}
-		} else {
-			keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
-			node = &yaml.Node{
-				Kind: yaml.MappingNode, Content: []*yaml.Node{keyNode, node},
-			}
-		}
-	}
-
-	return &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{node}}, nil
-}
-
 // marshalConfigYAML renders a config document tree with 2-space indentation,
 // matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
 // would reindent the whole file on the first `config set` despite the promise
@@ -653,8 +559,8 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
 	return node, nil
 }
 
-// yamlPathSet navigates a dotted key path, creating missing keys as needed,
+// yamlPathSet navigates a dotted key path, creating intermediate maps as
-// and sets the final key to the given scalar value. Numeric path
+// needed, and sets the final key to the given scalar value. Numeric path
 // components index into sequences; an index equal to the sequence length
 // appends a new element (e.g. "age_recipients.1" on a 1-element list).
 func yamlPathSet(root *yaml.Node, keys []string, value string) error {
@@ -667,12 +573,12 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
 		node = node.Content[0]
 	}
 
-	for i := range keys {
+	for i, key := range keys {
 		last := i == len(keys)-1
 
 		switch node.Kind {
 		case yaml.MappingNode:
-			node = yamlSetInMapping(node, keys, i, value, last)
+			node = yamlSetInMapping(node, key, value, last)
 		case yaml.SequenceNode:
 			next, err := yamlSetInSequence(node, keys, i, value, last)
 			if err != nil {
@@ -741,16 +647,14 @@ func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
 	return reflect.StructField{}, false
 }
 
-// yamlSetInMapping resolves (creating if needed) the value node for the
+// yamlSetInMapping resolves (creating if needed) the value node for key
-// path element keys[i] within a mapping node, setting it to value when it
+// within a mapping node, setting it to value when it is the final path
-// is the final path element, and returns the node to descend into.
+// element, and returns the node to descend into.
-func yamlSetInMapping(
+func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
-	node *yaml.Node, keys []string, i int, value string, last bool,
-) *yaml.Node {
 	var valueNode *yaml.Node
 
 	for j := 0; j+1 < len(node.Content); j += 2 {
-		if node.Content[j].Value == keys[i] {
+		if node.Content[j].Value == key {
 			valueNode = node.Content[j+1]
 
 			break
@@ -758,8 +662,13 @@ func yamlSetInMapping(
 	}
 
 	if valueNode == nil {
-		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: keys[i]}
+		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
-		valueNode = newPathNode(keys, i, value)
+
+		valueNode = &yaml.Node{Kind: yaml.MappingNode}
+		if last {
+			valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
+		}
+
 		node.Content = append(node.Content, keyNode, valueNode)
 	} else if last {
 		setScalar(valueNode, value)
@@ -787,7 +696,12 @@ func yamlSetInSequence(
 	}
 
 	if idx == len(node.Content) {
-		node.Content = append(node.Content, newPathNode(keys, i, value))
+		newNode := &yaml.Node{Kind: yaml.MappingNode}
+		if last {
+			newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
+		}
+
+		node.Content = append(node.Content, newNode)
 	} else if last {
 		setScalar(node.Content[idx], value)
 	}
@@ -795,22 +709,6 @@ func yamlSetInSequence(
 	return node.Content[idx], nil
 }
 
-// newPathNode returns the node to create for the missing path element
-// keys[i]: the value when it is the final path element, otherwise an empty
-// list when the next path element is a number and an empty map when not.
-func newPathNode(keys []string, i int, value string) *yaml.Node {
-	if i == len(keys)-1 {
-		return &yaml.Node{Kind: yaml.ScalarNode, Value: value}
-	}
-
-	_, err := strconv.Atoi(keys[i+1])
-	if err == nil {
-		return &yaml.Node{Kind: yaml.SequenceNode}
-	}
-
-	return &yaml.Node{Kind: yaml.MappingNode}
-}
-
 // setScalar overwrites a node in place with a plain scalar value.
 func setScalar(n *yaml.Node, value string) {
 	n.Kind = yaml.ScalarNode

internal/cli/config_test.go

@@ -58,17 +58,14 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
 	}
 }
 
-// testRecipient is a valid age public key.
-const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
-
 // TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
 // file `config init` writes, `config set age_recipients.0` and
 // `config set storage_url` give a config that loads with that recipient.
-// Neither is refused, although the config does not load until both are set;
-// once it loads, a recipient that is not a valid key is.
 func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 	t.Parallel()
 
+	const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
+
 	path := filepath.Join(t.TempDir(), "config.yml")
 
 	err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
@@ -78,7 +75,7 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 
 	out := ui.NewWithColor(&bytes.Buffer{}, false)
 
-	err = writeConfigSet(out, path, "age_recipients.0", testRecipient)
+	err = writeConfigSet(out, path, "age_recipients.0", recipient)
 	if err != nil {
 		t.Fatalf("config set age_recipients.0: %v", err)
 	}
@@ -93,211 +90,8 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 		t.Fatalf("config.Load: %v", err)
 	}
 
-	if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != testRecipient {
+	if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
-		t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, testRecipient)
+		t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
-	}
-
-	err = writeConfigSet(out, path, "age_recipients.0", "notakey")
-	if err == nil || !strings.Contains(err.Error(), "not a valid recipient") {
-		t.Errorf("config set age_recipients.0 notakey gives error %v, "+
-			"want one saying it is not a valid recipient", err)
-	}
-
-	_, err = config.Load(path)
-	if err != nil {
-		t.Errorf("config.Load after the refused change: %v", err)
-	}
-}
-
-// configWithoutRecipients loads, but has no age_recipients key and no
-// snapshot named work.
-const configWithoutRecipients = `storage_url: file:///mnt/backups
-snapshots:
-  home:
-    paths:
-      - "~"
-`
-
-// configWithUnsetReference loads only where VAULTIK_TEST_UNSET_SECRET is
-// set, as a backup job's environment might set it. No test sets it.
-const configWithUnsetReference = configWithoutRecipients + `s3:
-  secret_access_key: ${ENV:VAULTIK_TEST_UNSET_SECRET}
-`
-
-// configWithLevelReference returns configWithoutRecipients with a ${FILE:...}
-// reference on compression_level to a file holding 9, so that the config
-// loads once the reference is filled in.
-func configWithLevelReference(t *testing.T) string {
-	t.Helper()
-
-	levelPath := filepath.Join(t.TempDir(), "level")
-
-	err := os.WriteFile(levelPath, []byte("9"), configFileMode)
-	if err != nil {
-		t.Fatalf("write level: %v", err)
-	}
-
-	return configWithoutRecipients +
-		"compression_level: ${FILE:" + levelPath + "}\n"
-}
-
-// TestConfigSetIndexUnderMissingKey checks that a list index under a key
-// the file does not have creates a list, and that the result loads.
-func TestConfigSetIndexUnderMissingKey(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		key   string
-		value string
-		field func(cfg *config.Config) []string
-	}{
-		{"age_recipients.0", testRecipient,
-			func(cfg *config.Config) []string { return cfg.AgeRecipients }},
-		{"snapshots.work.paths.0", "/srv/work",
-			func(cfg *config.Config) []string { return cfg.Snapshots["work"].Paths }},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.key, func(t *testing.T) {
-			t.Parallel()
-
-			path := filepath.Join(t.TempDir(), "config.yml")
-
-			err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
-			if err != nil {
-				t.Fatalf("write config: %v", err)
-			}
-
-			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
-				path, tt.key, tt.value)
-			if err != nil {
-				t.Fatalf("config set %s: %v", tt.key, err)
-			}
-
-			cfg, err := config.Load(path)
-			if err != nil {
-				t.Fatalf("config.Load: %v", err)
-			}
-
-			got := tt.field(cfg)
-			if len(got) != 1 || got[0] != tt.value {
-				t.Errorf("%s gives %v, want [%s]", tt.key, got, tt.value)
-			}
-		})
-	}
-}
-
-// TestConfigSetKeepsFileThatWouldNotLoad checks that config set refuses a
-// change after which the config would not load, with an error giving the
-// reason, and leaves the file as it was. On the file `config init` writes,
-// which does not load until storage_url is set, and on one that cannot be
-// read here, only a value of the wrong type is refused.
-func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
-	t.Parallel()
-
-	levelReference := configWithLevelReference(t)
-
-	tests := []struct {
-		config string
-		key    string
-		value  string
-		reason string
-	}{
-		// Index 1 of the empty list the missing key gives.
-		{configWithoutRecipients, "age_recipients.1", testRecipient,
-			"index out of range"},
-		// A list set to a single value.
-		{configWithoutRecipients, "snapshots.home.paths", "/srv",
-			"into []string"},
-		{configWithoutRecipients, "chunk_size", "fast",
-			"invalid size format"},
-		// Values of the right type that fail validation.
-		{configWithoutRecipients, "compression_level", "99",
-			"compression_level must be between 1 and 19"},
-		{configWithoutRecipients, "chunk_size", "1KB",
-			"chunk_size must be at least 1MB"},
-		// With its reference filled in, the config loads.
-		{levelReference, "blob_size_limit", "1MB",
-			"blob_size_limit must be at least"},
-		// The type check on a config that does not load yet.
-		{defaultConfigTemplate, "snapshots.apps.paths", "/srv",
-			"into []string"},
-		// The type check on a config that cannot be read here.
-		{configWithUnsetReference, "snapshots.home.paths", "/x",
-			"cannot unmarshal !!str `/x`"},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.key+" "+tt.value, func(t *testing.T) {
-			t.Parallel()
-
-			path := filepath.Join(t.TempDir(), "config.yml")
-
-			err := os.WriteFile(path, []byte(tt.config), configFileMode)
-			if err != nil {
-				t.Fatalf("write config: %v", err)
-			}
-
-			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
-				path, tt.key, tt.value)
-			if err == nil || !strings.Contains(err.Error(), tt.reason) {
-				t.Errorf("config set %s %s gives error %v, want one saying %q",
-					tt.key, tt.value, err, tt.reason)
-			}
-
-			data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
-			if err != nil {
-				t.Fatalf("read config: %v", err)
-			}
-
-			if string(data) != tt.config {
-				t.Errorf("config set %s changed the file to:\n%s",
-					tt.key, data)
-			}
-		})
-	}
-}
-
-// TestConfigSetRefusesOnlyItsOwnFailure checks that setting another key
-// works on a config with a ${...} reference on compression_level, which
-// loads because loading fills the reference in, on one whose
-// compression_level already fails to load, and on one with a reference
-// that cannot be filled in here.
-func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
-	t.Parallel()
-
-	configs := map[string]string{
-		"reference":       configWithLevelReference(t),
-		"failing":         configWithoutRecipients + "compression_level: fast\n",
-		"unset reference": configWithUnsetReference,
-	}
-
-	for name, configYAML := range configs {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			path := filepath.Join(t.TempDir(), "config.yml")
-
-			err := os.WriteFile(path, []byte(configYAML), configFileMode)
-			if err != nil {
-				t.Fatalf("write config: %v", err)
-			}
-
-			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
-				path, "hostname", "myhost")
-			if err != nil {
-				t.Fatalf("config set hostname: %v", err)
-			}
-
-			data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
-			if err != nil {
-				t.Fatalf("read config: %v", err)
-			}
-
-			if !strings.Contains(string(data), "hostname: myhost") {
-				t.Errorf("config set hostname wrote:\n%s", data)
-			}
-		})
 	}
 }
 
@@ -647,30 +441,6 @@ func TestWriteConfigSetTightensMode(t *testing.T) {
 	}
 }
 
-// TestConfigEditEditorWithArgument checks that an $EDITOR carrying an
-// argument works and that a config path with a space reaches it whole.
-// The "editor" is cp with the replacement file as its argument, so the
-// config ends up with the replacement's content only if both arrived.
-func TestConfigEditEditorWithArgument(t *testing.T) {
-	dir := t.TempDir()
-	path := seedFile(t, dir, "my config.yml", "compression_level: 3\n")
-	edited := seedFile(t, dir, "edited.yml", "compression_level: 9\n")
-
-	setRootFlags(t, RootFlags{ConfigPath: path})
-	t.Setenv("EDITOR", "cp '"+edited+"'")
-
-	mustExecute(t, newConfigEditCommand())
-
-	data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
-	if err != nil {
-		t.Fatalf("reading config back: %v", err)
-	}
-
-	if string(data) != "compression_level: 9\n" {
-		t.Errorf("config after edit = %q, want the edited content", data)
-	}
-}
-
 func splitPath(s string) []string {
 	return strings.Split(s, ".")
 }

internal/cli/entry_prune_quiet_test.go

@@ -1,123 +0,0 @@
-package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// flagCron is used to build the argument vectors below.
-const flagCron = "--cron"
-
-// plantUnreferencedBlob writes a blob that no snapshot references to the
-// destination store at storeDir, so that a prune deletes it. Returns its
-// path.
-func plantUnreferencedBlob(t *testing.T, storeDir string) string {
-	t.Helper()
-
-	hash := strings.Repeat("b", 64)
-	path := filepath.Join(storeDir, "blobs", hash[:2], hash[2:4], hash)
-
-	require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750))
-	require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600))
-
-	return path
-}
-
-// TestEntrySnapshotCreatePruneIsSilent checks that a successful
-// `snapshot create --prune` under --cron or --quiet prints nothing on
-// stdout or stderr, whether or not the prune finds a blob to delete.
-//
-//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
-func TestEntrySnapshotCreatePruneIsSilent(t *testing.T) {
-	for _, testCase := range []struct {
-		name      string
-		args      []string
-		plantBlob bool
-	}{
-		{
-			name: "cron, no blob to delete",
-			args: []string{cmdSnapshot, cmdCreate, flagCron, flagPrune},
-		},
-		{
-			name:      "cron, blob to delete",
-			args:      []string{cmdSnapshot, cmdCreate, flagCron, flagPrune},
-			plantBlob: true,
-		},
-		{
-			name: "cron with keep-newer-than",
-			args: []string{
-				cmdSnapshot, cmdCreate, flagCron, flagPrune,
-				flagKeepNewerThan, "4w",
-			},
-			plantBlob: true,
-		},
-		{
-			name:      "quiet",
-			args:      []string{flagQuiet, cmdSnapshot, cmdCreate, flagPrune},
-			plantBlob: true,
-		},
-	} {
-		t.Run(testCase.name, func(t *testing.T) {
-			configPath, storeDir := writeBackupConfig(t)
-
-			var blob string
-			if testCase.plantBlob {
-				blob = plantUnreferencedBlob(t, storeDir)
-			}
-
-			code, stdout, stderr := runEntry(t,
-				append([]string{flagConfig, configPath}, testCase.args...)...)
-
-			assert.Equal(t, 0, code)
-			assert.Empty(t, stdout)
-			assert.Empty(t, stderr)
-
-			if testCase.plantBlob {
-				assert.NoFileExists(t, blob, "the prune must delete the blob")
-			}
-		})
-	}
-}
-
-// TestEntryPruneQuietIsSilent checks that a successful `prune --force`
-// under --quiet prints nothing on stdout or stderr, both when there is
-// nothing to remove and when it removes a stale local snapshot record
-// and an unreferenced blob.
-//
-//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
-func TestEntryPruneQuietIsSilent(t *testing.T) {
-	for _, testCase := range []struct {
-		name               string
-		staleRecordAndBlob bool
-	}{
-		{name: "nothing to remove"},
-		{name: "stale record and blob to remove", staleRecordAndBlob: true},
-	} {
-		t.Run(testCase.name, func(t *testing.T) {
-			configPath := writeHermeticPruneConfig(t, testCase.staleRecordAndBlob)
-
-			var blob string
-			if testCase.staleRecordAndBlob {
-				// writeHermeticPruneConfig puts the store beside the config.
-				blob = plantUnreferencedBlob(t,
-					filepath.Join(filepath.Dir(configPath), "store"))
-			}
-
-			code, stdout, stderr := runEntry(t,
-				flagConfig, configPath, flagQuiet, cmdPrune, flagForce)
-
-			assert.Equal(t, 0, code)
-			assert.Empty(t, stdout)
-			assert.Empty(t, stderr)
-
-			if testCase.staleRecordAndBlob {
-				assert.NoFileExists(t, blob, "the prune must delete the blob")
-			}
-		})
-	}
-}

internal/cli/entry_snapshot_purge_test.go

@@ -1,68 +0,0 @@
-package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/snapshot"
-)
-
-// Words and flags of `snapshot purge` used to build argument vectors
-// below.
-const (
-	cmdPurge      = "purge"
-	flagOlderThan = "--older-than"
-	flagForce     = "--force"
-)
-
-// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
-// the local index refuse to delete an old snapshot and checks that
-// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
-// the destination store. It used to log the error, delete the metadata
-// and exit 0.
-//
-// `snapshot create --prune` runs the same purge, but cannot be driven
-// this way: the refusal also breaks the backup's metadata export, which
-// deletes the other snapshots from its copy of the local index.
-//
-//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
-func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
-	configPath := writeHermeticPruneConfig(t, false)
-	dir := filepath.Dir(configPath)
-	indexPath := filepath.Join(dir, "index.sqlite")
-
-	// seedStaleSnapshotRecord writes an old snapshot to the local index
-	// only; the manifest planted here puts it on the destination store
-	// too, so the purge selects it rather than dropping it as stale.
-	seedStaleSnapshotRecord(t, indexPath)
-
-	manifestDir := filepath.Join(dir, "store", "metadata",
-		snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
-	require.NoError(t, os.MkdirAll(manifestDir, 0o750))
-
-	manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
-	require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
-
-	ctx := context.Background()
-
-	db, err := database.New(ctx, indexPath)
-	require.NoError(t, err)
-
-	_, err = db.Conn().ExecContext(ctx, `
-		CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
-		BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
-	require.NoError(t, err)
-	require.NoError(t, db.Close())
-
-	code, _, _ := runEntry(t, flagConfig, configPath,
-		cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
-
-	assert.Equal(t, 1, code)
-	assert.FileExists(t, manifestPath,
-		"the snapshot is still in the local index, so its metadata must stay")
-}

internal/cli/snapshot.go

@@ -270,9 +270,8 @@ By default, this removes the snapshot from the local index database and
 strips the snapshot's metadata from the backup destination store. Blobs
 are NOT touched: deleting them requires reading every remaining remote
 manifest (the destination store may hold snapshots this host doesn't
-know about), which is what 'vaultik prune' does. The command prints the
+know about), which is what 'vaultik prune' does. On success the command
-'vaultik prune' invocation to run as a follow-up only when it removed the
+prints the exact 'vaultik prune' invocation to run as a follow-up.
-snapshot's metadata from the destination store, and never under --json.
 
 An ID that names no snapshot in the local index or on the destination
 store is an error.
@@ -281,12 +280,10 @@ Use --local-only to skip the remote half (e.g. when you want to forget a
 snapshot locally without touching the destination store); the snapshot
 must then be in the local index.
 
-If removing the snapshot's metadata from the destination store fails (for
+If the remote is unreachable, the local-database removal still completes
-example because the remote is unreachable), the local-database removal
+and a warning is emitted; run 'vaultik snapshot remove <snapshot-id>' again
-still completes and a warning is emitted; run 'vaultik snapshot remove
+once the destination store is reachable to remove the snapshot's metadata
-<snapshot-id>' again once the destination store is reachable to remove
+from it ('vaultik prune' does not).
-the snapshot's metadata from it ('vaultik prune' does not). For a
-snapshot that is only on the destination store, that failure is an error.
 
 To wipe the entire destination store and start over, use 'vaultik remote
 nuke --force' — it is the single supported entry point for that.`,

internal/cli/snapshot_restore.go

@@ -30,8 +30,6 @@ This command will restore files from the specified snapshot to the
 target directory.
 If no paths are specified, all files are restored.
 If paths are specified, only matching files/directories are restored.
-A path that matches nothing in the snapshot is named in a warning, and
-the command exits non-zero after restoring the paths that match.
 
 The snapshot may be named by its ID or, when restoring on a host with no
 local index, by the remote key that 'snapshot list' prints for a

internal/config/config.go

@@ -4,7 +4,6 @@
 package config
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -30,15 +29,13 @@ const appName = "vaultik"
 //nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
 const secretKeyPrefix = "AGE-SECRET-KEY-"
 
-// Defaults and validation bounds for tunable settings. MB and GB are
+// Defaults and validation bounds for tunable settings.
-// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
-// limits, which are 1024-based (MiB, GiB).
 const (
-	defaultBlobSizeLimit    = Size(10 * 1000 * 1000 * 1000) // 10GB
+	defaultBlobSizeLimit    = Size(10 * 1024 * 1024 * 1024) // 10GB
-	defaultChunkSize        = Size(10 * 1000 * 1000)        // 10MB
+	defaultChunkSize        = Size(10 * 1024 * 1024)        // 10MB
 	defaultS3PartSize       = Size(5 * 1024 * 1024)         // 5MiB
 	defaultCompressionLevel = 3
-	minChunkSize            = 1000 * 1000 // 1MB
+	minChunkSize            = 1024 * 1024 // 1MB
 	minCompressionLevel     = 1
 	maxCompressionLevel     = 19
 	// S3 accepts a multipart upload part from 5MiB to 5GiB.
@@ -236,48 +233,14 @@ func New(path Path) (*Config, error) {
 	return cfg, nil
 }
 
-// Load reads the configuration file at path, parses it with Parse, warns
+// Load reads and parses the configuration file from the specified path.
-// when others can read the file, and validates the configuration.
+// It applies default values for optional fields, performs environment variable
-// Returns an error if the file cannot be read, parsed, or if validation
+// substitution using smartconfig, and validates the configuration.
-// fails.
+// The configuration file should be in YAML format. Returns an error if the file
+// cannot be read, parsed, or if validation fails.
 func Load(path string) (*Config, error) {
-	data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
-	if err != nil {
-		return nil, fmt.Errorf("failed to load config file: %w", err)
-	}
-
-	cfg, err := Parse(data)
-	if err != nil {
-		return nil, err
-	}
-
-	// Check config file permissions (warn if world or group readable)
-	//nolint:gosec // G703: config path is operator-supplied by design
-	info, statErr := os.Stat(path)
-	if statErr == nil {
-		mode := info.Mode().Perm()
-		if mode&0044 != 0 { // group or world readable
-			log.Warn(cfg.readableByOthersWarning(),
-				"path", path,
-				"mode", fmt.Sprintf("%04o", mode),
-				"recommendation", "chmod 600 "+path)
-		}
-	}
-
-	err = cfg.Validate()
-	if err != nil {
-		return nil, fmt.Errorf("invalid config: %w", err)
-	}
-
-	return cfg, nil
-}
-
-// Parse reads YAML config data as Load does, without validating it. It
-// applies default values for optional fields, fills in ${...} references
-// using smartconfig, and expands ~ in paths.
-func Parse(data []byte) (*Config, error) {
 	// Load config using smartconfig for interpolation
-	sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
+	sc, err := smartconfig.NewFromConfigPath(path)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load config file: %w", err)
 	}
@@ -339,6 +302,24 @@ func Parse(data []byte) (*Config, error) {
 		cfg.S3.Region = "us-east-1"
 	}
 
+	// Check config file permissions (warn if world or group readable)
+	//nolint:gosec // G703: config path is operator-supplied by design
+	info, statErr := os.Stat(path)
+	if statErr == nil {
+		mode := info.Mode().Perm()
+		if mode&0044 != 0 { // group or world readable
+			log.Warn(cfg.readableByOthersWarning(),
+				"path", path,
+				"mode", fmt.Sprintf("%04o", mode),
+				"recommendation", "chmod 600 "+path)
+		}
+	}
+
+	err = cfg.Validate()
+	if err != nil {
+		return nil, fmt.Errorf("invalid config: %w", err)
+	}
+
 	return cfg, nil
 }
 

internal/config/config_test.go

@@ -337,81 +337,6 @@ func TestLoadS3PartSize(t *testing.T) {
 	})
 }
 
-// TestDocumentedSizeDefaults checks that the defaults are the documented
-// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
-// (1000-based), so writing a default out in the config file changes nothing.
-func TestDocumentedSizeDefaults(t *testing.T) {
-	t.Parallel()
-
-	cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
-	if err != nil {
-		t.Fatalf("Parse() unexpected error: %v", err)
-	}
-
-	if cfg.ChunkSize != defaultChunkSize {
-		t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
-			cfg.ChunkSize, defaultChunkSize)
-	}
-
-	if cfg.BlobSizeLimit != defaultBlobSizeLimit {
-		t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
-			cfg.BlobSizeLimit, defaultBlobSizeLimit)
-	}
-}
-
-// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
-// every size the README, config.example.yml and the config init template
-// list as an example is accepted.
-func TestDocumentedSizeExamples(t *testing.T) {
-	t.Parallel()
-
-	const withoutSizes = "snapshots:\n" +
-		"  test:\n" +
-		"    paths: [/tmp/vaultik-test-source]\n" +
-		"storage_url: file:///tmp/vaultik-test-storage\n"
-
-	tests := []struct {
-		setting string
-		wantErr error
-	}{
-		{"chunk_size: 1MB", nil},
-		{"chunk_size: 999999", errChunkSizeTooSmall},
-		{"chunk_size: 5M", nil},
-		{"chunk_size: 10M", nil},
-		{"chunk_size: 10MB", nil},
-		{"chunk_size: 64MiB", nil},
-		{"chunk_size: 1GB", nil},
-		// Four times the default chunk_size, the documented minimum.
-		{"blob_size_limit: 40MB", nil},
-		{"blob_size_limit: 500MB", nil},
-		{"blob_size_limit: 1GB", nil},
-		{"blob_size_limit: 1GiB", nil},
-		{"blob_size_limit: 10G", nil},
-		{"blob_size_limit: 10GB", nil},
-		{"s3: {part_size: 5MiB}", nil},
-		{"s3: {part_size: 10MB}", nil},
-		{"s3: {part_size: 16MiB}", nil},
-		{"s3: {part_size: 100MiB}", nil},
-		{"s3: {part_size: 5GiB}", nil},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.setting, func(t *testing.T) {
-			t.Parallel()
-
-			cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
-			if err != nil {
-				t.Fatalf("Parse() unexpected error: %v", err)
-			}
-
-			err = cfg.Validate()
-			if !errors.Is(err, tt.wantErr) {
-				t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-
 // TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
 // relative path fails at load. Accepted, it would resolve against the
 // working directory, so a backup run from another directory would write

internal/database/blob_chunks.go

@@ -58,7 +58,12 @@ func (r *BlobChunkRepository) GetByBlobID(
 		return nil, fmt.Errorf("querying blob chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var blobChunks []*BlobChunk
 

internal/database/blobs.go

@@ -83,7 +83,12 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
 		return nil, fmt.Errorf("querying blobs: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	out := make(map[string]*Blob)
 
@@ -154,16 +159,13 @@ func (r *BlobRepository) UpdateFinished(
 	return nil
 }
 
-// UpdateUploaded marks a blob as uploaded and records the size of the
+// UpdateUploaded marks a blob as uploaded
-// stored object as its compressed size. That size can differ from the
-// local copy's when an identical blob was already stored with another
-// compression level or other recipients.
 func (r *BlobRepository) UpdateUploaded(
-	ctx context.Context, tx *sql.Tx, id string, storedSize int64,
+	ctx context.Context, tx *sql.Tx, id string,
 ) error {
 	query := `
-		UPDATE blobs
+		UPDATE blobs 
-		SET uploaded_ts = ?, compressed_size = ?
+		SET uploaded_ts = ?
 		WHERE id = ?
 	`
 
@@ -171,9 +173,9 @@ func (r *BlobRepository) UpdateUploaded(
 
 	var err error
 	if tx != nil {
-		_, err = tx.ExecContext(ctx, query, now, storedSize, id)
+		_, err = tx.ExecContext(ctx, query, now, id)
 	} else {
-		_, err = r.db.ExecWithLog(ctx, query, now, storedSize, id)
+		_, err = r.db.ExecWithLog(ctx, query, now, id)
 	}
 
 	if err != nil {

internal/database/blobs_test.go

@@ -122,8 +122,8 @@ func TestBlobRepositoryUpdates(t *testing.T) {
 		t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
 	}
 
-	// Test UpdateUploaded, with a stored object larger than the local copy
+	// Test UpdateUploaded
-	err = repo.UpdateUploaded(ctx, nil, blob.ID.String(), 600)
+	err = repo.UpdateUploaded(ctx, nil, blob.ID.String())
 	if err != nil {
 		t.Fatalf("failed to update blob as uploaded: %v", err)
 	}
@@ -141,10 +141,6 @@ func TestBlobRepositoryUpdates(t *testing.T) {
 	if uploaded.UploadedTS.Before(now.Add(-1 * time.Second)) {
 		t.Error("uploaded timestamp should be around test time")
 	}
-
-	if uploaded.CompressedSize != 600 {
-		t.Errorf("expected the stored size 600, got %d", uploaded.CompressedSize)
-	}
 }
 
 func TestBlobRepositoryDuplicate(t *testing.T) {

internal/database/chunk_files.go

@@ -61,7 +61,12 @@ func (r *ChunkFileRepository) GetByChunkHash(
 		return nil, fmt.Errorf("querying chunk files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanChunkFiles(rows)
 }
@@ -82,7 +87,12 @@ func (r *ChunkFileRepository) GetByFilePath(
 		return nil, fmt.Errorf("querying chunk files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanChunkFiles(rows)
 }
@@ -102,7 +112,12 @@ func (r *ChunkFileRepository) GetByFileID(
 		return nil, fmt.Errorf("querying chunk files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanChunkFiles(rows)
 }

internal/database/chunks.go

@@ -107,7 +107,12 @@ func (r *ChunkRepository) GetByHashes(
 		return nil, fmt.Errorf("querying chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var chunks []*Chunk
 
@@ -147,7 +152,12 @@ func (r *ChunkRepository) ListUnpacked(
 		return nil, fmt.Errorf("querying unpacked chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var chunks []*Chunk
 

internal/database/chunks_ext.go

@@ -38,7 +38,12 @@ func (r *ChunkRepository) list(ctx context.Context, query string) ([]*Chunk, err
 		return nil, fmt.Errorf("querying chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var chunks []*Chunk
 

internal/database/errors.go

@@ -0,0 +1,12 @@
+package database
+
+import (
+	"fmt"
+	"os"
+)
+
+// Fatalf prints an error message to stderr and exits with status 1
+func Fatalf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
+	os.Exit(1)
+}

internal/database/file_chunks.go

@@ -62,7 +62,12 @@ func (r *FileChunkRepository) GetByPath(
 		return nil, fmt.Errorf("querying file chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanFileChunks(rows)
 }
@@ -83,7 +88,12 @@ func (r *FileChunkRepository) GetByFileID(
 		return nil, fmt.Errorf("querying file chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanFileChunks(rows)
 }
@@ -107,7 +117,12 @@ func (r *FileChunkRepository) GetByPathTx(
 		return nil, fmt.Errorf("querying file chunks: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	fileChunks, err := r.scanFileChunks(rows)
 	LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))

internal/database/files.go

@@ -175,7 +175,12 @@ func (r *FileRepository) ListModifiedSince(
 		return nil, fmt.Errorf("querying files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var files []*File
 
@@ -251,46 +256,12 @@ func (r *FileRepository) ListUnderPath(
 		return nil, fmt.Errorf("querying files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
-
+		err := rows.Close()
-	var files []*File
-
-	for rows.Next() {
-		file, err := r.scanFileRows(rows)
 		if err != nil {
-			return nil, fmt.Errorf("scanning file: %w", err)
+			Fatalf("failed to close rows: %v", err)
 		}
-
+	}()
-		files = append(files, file)
-	}
-
-	return files, rows.Err()
-}
-
-// ListUnderPathInSnapshot returns the files that ListUnderPath returns
-// for path and that the snapshot lists, in no particular order.
-func (r *FileRepository) ListUnderPathInSnapshot(
-	ctx context.Context, path, snapshotID string,
-) ([]*File, error) {
-	path = strings.TrimRight(path, "/")
-	dirPrefix := path + "/"
-
-	query := `
-		SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
-			f.mode, f.uid, f.gid, f.link_target
-		FROM files f
-		JOIN snapshot_files sf ON sf.file_id = f.id
-		WHERE sf.snapshot_id = ?
-		  AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
-	`
-
-	rows, err := r.db.conn.QueryContext(ctx, query,
-		snapshotID, path, dirPrefix, dirPrefix)
-	if err != nil {
-		return nil, fmt.Errorf("querying files: %w", err)
-	}
-
-	defer func() { _ = rows.Close() }()
 
 	var files []*File
 
@@ -333,7 +304,12 @@ func (r *FileRepository) ListIDsWithChunksNotInUploadedBlobs(
 		return nil, fmt.Errorf("querying files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var ids []types.FileID
 
@@ -364,7 +340,12 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
 		return nil, fmt.Errorf("querying files: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var files []*File
 

internal/database/files_test.go

@@ -212,65 +212,6 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
 	}
 }
 
-// ListUnderPathInSnapshot leaves out files outside path and files the
-// snapshot does not list, though another snapshot lists them.
-func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
-	t.Parallel()
-
-	db, cleanup := setupTestDB(t)
-	defer cleanup()
-
-	ctx := context.Background()
-	files := database.NewFileRepository(db)
-	snapshots := database.NewSnapshotRepository(db)
-
-	listed := map[string][]string{
-		"one": {"/src/a.txt", "/src2/c.txt"},
-		"two": {"/src/b.txt"},
-	}
-
-	for snapshotID, paths := range listed {
-		err := snapshots.Create(ctx, nil, &database.Snapshot{
-			ID:        types.SnapshotID(snapshotID),
-			Hostname:  "host",
-			StartedAt: time.Now(),
-		})
-		if err != nil {
-			t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
-		}
-
-		for _, path := range paths {
-			err := files.Create(ctx, nil, &database.File{
-				Path:  types.FilePath(path),
-				MTime: time.Now().Truncate(time.Second),
-				Mode:  0644,
-			})
-			if err != nil {
-				t.Fatalf("failed to create %s: %v", path, err)
-			}
-
-			err = snapshots.AddFile(ctx, nil, snapshotID, path)
-			if err != nil {
-				t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
-			}
-		}
-	}
-
-	found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
-	if err != nil {
-		t.Fatalf("failed to list files: %v", err)
-	}
-
-	got := make([]string, 0, len(found))
-	for _, f := range found {
-		got = append(got, f.Path.String())
-	}
-
-	if !slices.Equal(got, []string{"/src/a.txt"}) {
-		t.Errorf("got %q, want only /src/a.txt", got)
-	}
-}
-
 func TestFileRepositorySymlink(t *testing.T) {
 	t.Parallel()
 

internal/database/repositories.go

@@ -106,24 +106,22 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
 
 	defer func() {
 		if p := recover(); p != nil {
-			// The panic is the failure to report, so a rollback error
+			rollbackErr := tx.Rollback()
-			// is dropped.
+			if rollbackErr != nil {
-			_ = tx.Rollback()
+				Fatalf("failed to rollback transaction: %v", rollbackErr)
+			}
 
 			panic(p)
+		} else if err != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				Fatalf("failed to rollback transaction: %v", rollbackErr)
+			}
 		}
 	}()
 
 	err = fn(ctx, tx)
 	if err != nil {
-		// Once ctx is cancelled, database/sql rolls the transaction
-		// back itself and Rollback returns sql.ErrTxDone.
-		rollbackErr := tx.Rollback()
-		if rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) {
-			return fmt.Errorf("%w; rolling back transaction: %w",
-				err, rollbackErr)
-		}
-
 		return err
 	}
 

internal/database/repositories_test.go

@@ -411,41 +411,3 @@ func TestRepositoriesReadTransaction(t *testing.T) {
 		t.Error("expected to retrieve file in read transaction")
 	}
 }
-
-// When the context is cancelled during a read transaction's body,
-// database/sql rolls the transaction back itself, so WithReadTx's own
-// rollback finds it already done. WithReadTx must return the body's error
-// alone, without that rollback.
-func TestRepositoriesReadTransactionContextCancelled(t *testing.T) {
-	t.Parallel()
-
-	db, cleanup := setupTestDB(t)
-	defer cleanup()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	repos := database.NewRepositories(db)
-
-	err := repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
-		cancel()
-
-		// database/sql rolls back from a goroutine of its own; wait
-		// until it has, so the test does not depend on that race.
-		for {
-			_, err := tx.ExecContext(context.WithoutCancel(ctx), "SELECT 1")
-			if errors.Is(err, sql.ErrTxDone) {
-				return ctx.Err()
-			}
-
-			time.Sleep(time.Millisecond)
-		}
-	})
-	if !errors.Is(err, context.Canceled) {
-		t.Fatalf("expected context.Canceled, got: %v", err)
-	}
-
-	if errors.Is(err, sql.ErrTxDone) {
-		t.Fatalf("the already-done rollback was reported: %v", err)
-	}
-}

internal/database/schema/001.sql

@@ -89,7 +89,6 @@ CREATE TABLE IF NOT EXISTS snapshots (
     new_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_count INTEGER NOT NULL DEFAULT 0,
     total_size INTEGER NOT NULL DEFAULT 0,
-    uncompressed_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_size INTEGER NOT NULL DEFAULT 0,
     blob_uncompressed_size INTEGER NOT NULL DEFAULT 0,
     compression_ratio REAL NOT NULL DEFAULT 1.0,

internal/database/snapshots.go

@@ -120,11 +120,7 @@ func (r *SnapshotRepository) UpdateCounts(
 	return nil
 }
 
-// UpdateExtendedStats updates extended statistics for a snapshot. It also
+// UpdateExtendedStats updates extended statistics for a snapshot
-// records the total plaintext size of the distinct chunks the snapshot's
-// files reference. Only the snapshot's own backup can take that total: a
-// file's row is shared by every snapshot that lists it, and a later
-// backup of a changed file rewrites the row with the new chunks.
 func (r *SnapshotRepository) UpdateExtendedStats(
 	ctx context.Context,
 	tx *sql.Tx,
@@ -149,17 +145,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
 		    compression_level = ?,
 		    upload_bytes = ?,
 		    upload_duration_ms = ?,
-		    new_chunk_size = ?,
+		    new_chunk_size = ?
-		    uncompressed_chunk_size = (
-		        SELECT COALESCE(SUM(c.size), 0)
-		        FROM (
-		            SELECT DISTINCT fc.chunk_hash
-		            FROM snapshot_files sf
-		            JOIN file_chunks fc ON sf.file_id = fc.file_id
-		            WHERE sf.snapshot_id = snapshots.id
-		        ) sc
-		        JOIN chunks c ON sc.chunk_hash = c.chunk_hash
-		    )
 		WHERE id = ?
 	`
 
@@ -255,7 +241,12 @@ func (r *SnapshotRepository) GetOnlySnapshot(ctx context.Context) (*Snapshot, er
 		return nil, fmt.Errorf("querying snapshots: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	snapshots, err := r.scanSnapshotRows(rows)
 	if err != nil {
@@ -290,7 +281,12 @@ func (r *SnapshotRepository) ListRecent(
 		return nil, fmt.Errorf("querying snapshots: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanSnapshotRows(rows)
 }
@@ -505,7 +501,12 @@ func (r *SnapshotRepository) GetBlobHashes(
 		return nil, fmt.Errorf("querying blob hashes: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	var blobs []string
 
@@ -569,22 +570,30 @@ func (r *SnapshotRepository) GetSnapshotBlobSizes(
 	return compressed, uncompressed, nil
 }
 
-// GetSnapshotUncompressedChunkSize returns the total plaintext size of the
+// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
-// distinct chunks the snapshot's files reference, as the snapshot's backup
+// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
-// recorded it (see UpdateExtendedStats).
 func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
 	ctx context.Context, snapshotID string,
 ) (int64, error) {
-	query := `SELECT uncompressed_chunk_size FROM snapshots WHERE id = ?`
+	query := `
+		SELECT COALESCE(SUM(c.size), 0)
+		FROM (
+			SELECT DISTINCT fc.chunk_hash
+			FROM snapshot_files sf
+			JOIN file_chunks fc ON sf.file_id = fc.file_id
+			WHERE sf.snapshot_id = ?
+		) sc
+		JOIN chunks c ON sc.chunk_hash = c.chunk_hash
+	`
 
-	var uncompressedChunkSize int64
+	var totalSize int64
 
-	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&uncompressedChunkSize)
+	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
 	if err != nil {
 		return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
 	}
 
-	return uncompressedChunkSize, nil
+	return totalSize, nil
 }
 
 // GetSnapshotNewChunkSize returns the total plaintext size of the chunks the
@@ -625,7 +634,12 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(
 		return nil, fmt.Errorf("querying incomplete snapshots: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	return r.scanSnapshotRows(rows)
 }
@@ -648,7 +662,12 @@ func (r *SnapshotRepository) GetIncompleteByHostname(
 		return nil, fmt.Errorf("querying incomplete snapshots: %w", err)
 	}
 
-	defer func() { _ = rows.Close() }()
+	defer func() {
+		err := rows.Close()
+		if err != nil {
+			Fatalf("failed to close rows: %v", err)
+		}
+	}()
 
 	// Same column set as every other multi-row snapshot query, so the
 	// shared scanner applies — including its timestamp normalization.

internal/s3/client.go

@@ -18,13 +18,6 @@ import (
 	"github.com/aws/smithy-go/logging"
 )
 
-// ErrNoContinuationToken is returned by ListObjects and ListObjectsStream
-// when the server marks a page of a listing truncated but sends no
-// continuation token to fetch the rest with, as a server answering V2
-// listings V1-style does.
-var ErrNoContinuationToken = errors.New(
-	"server marked the listing truncated but sent no continuation token")
-
 // Client wraps the AWS S3 client for vaultik operations.
 // It provides a simplified interface for S3 operations with automatic
 // prefix handling and connection management. All operations are performed
@@ -221,7 +214,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
 	})
 
 	for paginator.HasMorePages() {
-		page, err := nextListPage(ctx, paginator)
+		page, err := paginator.NextPage(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -242,26 +235,6 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
 	return keys, nil
 }
 
-// nextListPage fetches the next page of a listing. The SDK paginator takes
-// a page marked truncated without a continuation token for the last page
-// and stops without an error, so such a page is ErrNoContinuationToken
-// here instead.
-func nextListPage(
-	ctx context.Context, paginator *s3.ListObjectsV2Paginator,
-) (*s3.ListObjectsV2Output, error) {
-	page, err := paginator.NextPage(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	truncated := aws.ToBool(page.IsTruncated)
-	if truncated && aws.ToString(page.NextContinuationToken) == "" {
-		return nil, ErrNoContinuationToken
-	}
-
-	return page, nil
-}
-
 // HeadObject checks if an object exists in S3.
 // Returns true if the object exists, false otherwise.
 // The key is automatically prefixed with the configured prefix.
@@ -327,7 +300,7 @@ func (c *Client) ListObjectsStream(
 		})
 
 		for paginator.HasMorePages() {
-			page, err := nextListPage(ctx, paginator)
+			page, err := paginator.NextPage(ctx)
 			if err != nil {
 				ch <- ObjectInfo{Err: err}
 

internal/snapshot/blob_temp_file_test.go

@@ -1,62 +0,0 @@
-//nolint:testpackage // exercises the unexported cleanupBlobTempFile
-package snapshot
-
-import (
-	"errors"
-	"os"
-	"testing"
-
-	"github.com/spf13/afero"
-	"sneak.berlin/go/vaultik/internal/blob"
-)
-
-// A blob temp file that cannot be closed or removed must make the cleanup
-// return an error, not end the process.
-func TestCleanupBlobTempFileReturnsErrors(t *testing.T) {
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	s := &Scanner{fs: fs}
-
-	tests := []struct {
-		name string
-		// prepare runs before the cleanup and makes its close or remove
-		// fail.
-		prepare func(tempFile afero.File) error
-		want    error
-	}{
-		{
-			name:    "close fails",
-			prepare: func(tempFile afero.File) error { return tempFile.Close() },
-			want:    os.ErrClosed,
-		},
-		{
-			name: "remove fails",
-			prepare: func(tempFile afero.File) error {
-				return fs.Remove(tempFile.Name())
-			},
-			want: os.ErrNotExist,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			tempFile, err := afero.TempFile(fs, t.TempDir(), "blob-*.tmp")
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = tt.prepare(tempFile)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.cleanupBlobTempFile(&blob.WithReader{TempFile: tempFile})
-			if !errors.Is(err, tt.want) {
-				t.Fatalf("expected an error wrapping %v, got: %v", tt.want, err)
-			}
-		})
-	}
-}

internal/snapshot/exclude_test.go

@@ -578,54 +578,3 @@ func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
 	require.Equal(t, 1, result.FilesScanned,
 		"**/ file pattern should match at the top level and nested")
 }
-
-func TestExcludePatterns_AnchoredDoubleStarAtEveryDepth(t *testing.T) {
-	t.Parallel()
-
-	// As in gitignore, /**/ also matches directly under the source dir
-	fs := setupAnchoredTestFS(t)
-
-	scanner, repos, cleanup := createTestScanner(t, fs, []string{"/**/projectname"})
-	defer cleanup()
-
-	ctx := context.Background()
-	createSnapshotRecord(ctx, t, repos, "test-snapshot")
-
-	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
-	require.NoError(t, err)
-
-	// /**/projectname should exclude BOTH:
-	// - /backup/projectname/file.txt
-	// - /backup/otherproject/projectname/file.txt
-	// Total files: 4, excluded: 2
-	require.Equal(t, 2, result.FilesScanned,
-		"/**/ pattern should match at the top level and nested")
-}
-
-func TestExcludePatterns_DoubleStarInMiddleMatchesZeroDirectories(t *testing.T) {
-	t.Parallel()
-
-	// As in gitignore, a/**/b matches a/b as well as a/c/b. The last part
-	// is a glob: with a plain name the glob library already matched a/b.
-	fs := &afero.MemMapFs{}
-	files := []string{"/backup/a/x.log", "/backup/a/c/y.log", "/backup/a/c/kept.txt"}
-
-	for _, path := range files {
-		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0755))
-		require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0644))
-	}
-
-	scanner, repos, cleanup := createTestScanner(t, fs, []string{"a/**/*.log"})
-	defer cleanup()
-
-	ctx := context.Background()
-	createSnapshotRecord(ctx, t, repos, "test-snapshot")
-
-	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
-	require.NoError(t, err)
-
-	// a/**/*.log should exclude /backup/a/x.log and /backup/a/c/y.log
-	// Total files: 3, excluded: 2
-	require.Equal(t, 1, result.FilesScanned,
-		"a/**/*.log should match a/x.log as well as a/c/y.log")
-}

internal/snapshot/module.go

@@ -16,9 +16,6 @@ type ScannerParams struct {
 	Fs             afero.Fs
 	Exclude        []string // Exclude patterns (combined global + snapshot-specific)
 	SkipErrors     bool     // Skip file read errors (log loudly but continue)
-	// PreviousSnapshotID is the most recent completed snapshot of the
-	// same name, "" for none.
-	PreviousSnapshotID string
 }
 
 // Module exports backup functionality as an fx module.
@@ -47,18 +44,17 @@ func provideScannerFactory(
 		}
 
 		return NewScanner(ScannerConfig{
-			FS:                 params.Fs,
+			FS:               params.Fs,
-			ChunkSize:          cfg.ChunkSize.Int64(),
+			ChunkSize:        cfg.ChunkSize.Int64(),
-			Repositories:       repos,
+			Repositories:     repos,
-			Storage:            storer,
+			Storage:          storer,
-			MaxBlobSize:        cfg.BlobSizeLimit.Int64(),
+			MaxBlobSize:      cfg.BlobSizeLimit.Int64(),
-			CompressionLevel:   cfg.CompressionLevel,
+			CompressionLevel: cfg.CompressionLevel,
-			AgeRecipients:      cfg.AgeRecipients,
+			AgeRecipients:    cfg.AgeRecipients,
-			EnableProgress:     params.EnableProgress,
+			EnableProgress:   params.EnableProgress,
-			UI:                 params.UI,
+			UI:               params.UI,
-			Exclude:            excludes,
+			Exclude:          excludes,
-			SkipErrors:         params.SkipErrors,
+			SkipErrors:       params.SkipErrors,
-			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/snapshot/scanner.go

@@ -58,10 +58,6 @@ type Scanner struct {
 	compressionLevel int
 	ageRecipient     string
 	snapshotID       string // Current snapshot being processed
-	// previousSnapshotID is the most recent completed snapshot of the
-	// same name, "" for none. Only a file it lists can be counted as
-	// deleted.
-	previousSnapshotID string
 	// currentSourcePath is the source directory being scanned, stored with
 	// each file record.
 	currentSourcePath string
@@ -133,9 +129,6 @@ type ScannerConfig struct {
 	// SkipErrors skips files that cannot be opened or read (log loudly but
 	// continue); packer, database, encryption, and upload errors still abort.
 	SkipErrors bool
-	// PreviousSnapshotID is the most recent completed snapshot of the
-	// same name, "" for none.
-	PreviousSnapshotID string
 }
 
 // ScanResult contains the results of a scan operation. Files and bytes
@@ -214,7 +207,6 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		compiledExclude:    compiledExclude,
 		progress:           progress,
 		skipErrors:         cfg.SkipErrors,
-		previousSnapshotID: cfg.PreviousSnapshotID,
 		ui:                 uiw,
 		pendingChunkHashes: make(map[string]struct{}),
 	}
@@ -266,9 +258,8 @@ func (s *Scanner) Scan(
 
 	filesToProcess := scanResult.FilesToProcess
 
-	// Phase 1b: Detect deleted files by comparing the previous snapshot
+	// Phase 1b: Detect deleted files by comparing DB against scanned files
-	// of this name against scanned files
+	err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
-	err = s.detectDeletedFiles(ctx, path, existingFiles, result)
 	if err != nil {
 		return nil, fmt.Errorf("detecting deleted files: %w", err)
 	}
@@ -448,8 +439,9 @@ func (s *Scanner) summarizeScanPhase(
 }
 
 // loadKnownFiles loads the known files at and beneath path from the
-// database into a map for fast lookup. This avoids per-file database
+// database into a map for fast lookup. Every loaded file the scan does
-// queries during the scan phase.
+// not find is counted as deleted. This avoids per-file database queries
+// during the scan phase.
 func (s *Scanner) loadKnownFiles(
 	ctx context.Context, path string,
 ) (map[string]*database.File, error) {
@@ -1525,8 +1517,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
 			// snapshot references has uploaded_ts set, so deduplication and
 			// interrupted-run repair treat these blobs as trustworthy.
 			err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
-				err := s.repos.Blobs.UpdateUploaded(
+				err := s.repos.Blobs.UpdateUploaded(ctx, tx, b.ID)
-					ctx, tx, b.ID, b.Compressed)
 				if err != nil {
 					return fmt.Errorf("marking blob uploaded: %w", err)
 				}
@@ -1562,28 +1553,23 @@ func (s *Scanner) handleBlobReady(
 	blobPath := fmt.Sprintf("blobs/%s/%s/%s",
 		finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
 
-	storedSize, blobExists, err := s.uploadBlobIfNeeded(
+	blobExists, err := s.uploadBlobIfNeeded(
 		ctx, blobPath, blobWithReader, startTime, result)
 	if err != nil {
-		cleanupErr := s.cleanupBlobTempFile(blobWithReader)
+		s.cleanupBlobTempFile(blobWithReader)
 
-		return errors.Join(
+		return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
-			fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err), cleanupErr)
 	}
 
-	err = s.recordBlobMetadata(
+	err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime)
-		ctx, finishedBlob, storedSize, blobExists, startTime)
 	if err != nil {
-		cleanupErr := s.cleanupBlobTempFile(blobWithReader)
+		s.cleanupBlobTempFile(blobWithReader)
 
-		return errors.Join(err, cleanupErr)
-	}
-
-	err = s.cleanupBlobTempFile(blobWithReader)
-	if err != nil {
 		return err
 	}
 
+	s.cleanupBlobTempFile(blobWithReader)
+
 	// Chunks from this blob are now committed to DB - remove from pending set
 	s.removePendingChunkHashes(blobWithReader.InsertedChunkHashes)
 
@@ -1597,22 +1583,20 @@ func (s *Scanner) handleBlobReady(
 }
 
 // uploadBlobIfNeeded uploads the blob to storage if it doesn't already
-// exist. It returns the size of the object in storage, which for an
+// exist, returns whether it existed
-// existing object can differ from the local copy's, and whether it
-// already existed.
 func (s *Scanner) uploadBlobIfNeeded(
 	ctx context.Context,
 	blobPath string,
 	blobWithReader *blob.WithReader,
 	startTime time.Time,
 	result *ScanResult,
-) (int64, bool, error) {
+) (bool, error) {
 	finishedBlob := blobWithReader.FinishedBlob
 
 	// Check if blob already exists (deduplication after restart)
 	destination := s.storage.Info().Location
 
-	stored, err := s.storage.Stat(ctx, blobPath)
+	_, err := s.storage.Stat(ctx, blobPath)
 	if err == nil {
 		log.Info("Blob already exists in storage, skipping upload",
 			"hash", finishedBlob.Hash,
@@ -1621,7 +1605,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 			s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
 			s.ui.Path(destination))
 
-		return stored.Size, true, nil
+		return true, nil
 	}
 
 	s.ui.Beginf("Uploading blob %s (%s) to %s.",
@@ -1635,7 +1619,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 	if err != nil {
 		log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
 
-		return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
+		return false, fmt.Errorf("uploading blob to storage: %w", err)
 	}
 
 	uploadDuration := time.Since(startTime)
@@ -1666,7 +1650,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 		stats.BytesUploaded.Add(finishedBlob.Compressed)
 	}
 
-	return finishedBlob.Compressed, false, nil
+	return false, nil
 }
 
 // makeUploadProgressCallback creates a progress callback for blob uploads.
@@ -1736,7 +1720,6 @@ func (s *Scanner) makeUploadProgressCallback(
 func (s *Scanner) recordBlobMetadata(
 	ctx context.Context,
 	finishedBlob *blob.FinishedBlob,
-	storedSize int64,
 	blobExists bool,
 	startTime time.Time,
 ) error {
@@ -1748,8 +1731,7 @@ func (s *Scanner) recordBlobMetadata(
 	uploadDuration := time.Since(startTime)
 
 	return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
-		err := s.repos.Blobs.UpdateUploaded(
+		err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
-			txCtx, tx, finishedBlob.ID, storedSize)
 		if err != nil {
 			return fmt.Errorf("updating blob upload timestamp: %w", err)
 		}
@@ -1780,22 +1762,20 @@ func (s *Scanner) recordBlobMetadata(
 }
 
 // cleanupBlobTempFile closes and removes the blob's temporary file
-func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.WithReader) error {
+func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.WithReader) {
 	if blobWithReader.TempFile != nil {
 		tempName := blobWithReader.TempFile.Name()
 
 		err := blobWithReader.TempFile.Close()
 		if err != nil {
-			return fmt.Errorf("closing blob temp file: %w", err)
+			log.Fatal("Failed to close temp file", "file", tempName, "error", err)
 		}
 
 		err = s.fs.Remove(tempName)
 		if err != nil {
-			return fmt.Errorf("removing blob temp file: %w", err)
+			log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
 		}
 	}
-
-	return nil
 }
 
 // streamingChunkInfo tracks chunk metadata collected during streaming
@@ -1962,29 +1942,21 @@ func (s *Scanner) queueFileForBatchInsert(
 	})
 }
 
-// detectDeletedFiles counts as deleted each file at or beneath path that
+// detectDeletedFilesFromMap finds files that existed in previous snapshots
-// the previous snapshot of this name lists, the walk did not find, and
+// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
-// is gone from disk. A file that is excluded, or inside an excluded
+// database access.
-// directory, is not counted: the walk skips it whether or not it is
+func (s *Scanner) detectDeletedFilesFromMap(
-// still on disk.
-func (s *Scanner) detectDeletedFiles(
 	ctx context.Context,
-	path string,
+	knownFiles map[string]*database.File,
 	existingFiles map[string]struct{},
 	result *ScanResult,
 ) error {
-	if s.previousSnapshotID == "" {
+	if len(knownFiles) == 0 {
 		return nil
 	}
 
-	previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
+	// Check each known file against the enumerated set (no filesystem access needed)
-		ctx, path, s.previousSnapshotID)
+	for path, file := range knownFiles {
-	if err != nil {
-		return fmt.Errorf("listing files of snapshot %s: %w",
-			s.previousSnapshotID, err)
-	}
-
-	for _, file := range previousFiles {
 		// Check context cancellation periodically
 		select {
 		case <-ctx.Done():
@@ -1992,22 +1964,13 @@ func (s *Scanner) detectDeletedFiles(
 		default:
 		}
 
-		filePath := file.Path.String()
+		// Check if the file exists in our enumerated set
-
+		if _, exists := existingFiles[path]; !exists {
-		_, exists := existingFiles[filePath]
+			// File has been deleted
-		if exists || s.excludedOrInExcludedDir(filePath, path) {
+			result.FilesDeleted++
-			continue
+			result.BytesDeleted += file.Size
+			log.Debug("Detected deleted file", "path", path, "size", file.Size)
 		}
-
-		// Under --skip-errors the walk also misses what it cannot reach,
-		// such as the files in a directory it cannot open.
-		if !s.goneFromDisk(filePath) {
-			continue
-		}
-
-		result.FilesDeleted++
-		result.BytesDeleted += file.Size
-		log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
 	}
 
 	if result.FilesDeleted > 0 {
@@ -2018,36 +1981,6 @@ func (s *Scanner) detectDeletedFiles(
 	return nil
 }
 
-// excludedOrInExcludedDir reports whether filePath, which is at or
-// beneath rootPath, or any directory between rootPath and it, matches an
-// exclude pattern. A pattern such as */.local/share/Trash matches the
-// directory but not the files inside it, which the walk skips too.
-func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
-	for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
-		if s.shouldExclude(p, rootPath) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// goneFromDisk reports whether looking filePath up on disk finds that
-// it does not exist. Like the walk, it does not follow a symlink, so a
-// symlink whose target is gone is still on disk. A file whose directory
-// was replaced by a regular file is gone too, though the lookup reports
-// "not a directory".
-func (s *Scanner) goneFromDisk(filePath string) bool {
-	var err error
-	if lstater, ok := s.fs.(afero.Lstater); ok {
-		_, _, err = lstater.LstatIfPossible(filePath)
-	} else {
-		_, err = s.fs.Stat(filePath)
-	}
-
-	return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR)
-}
-
 // wrapPermissionError augments permission errors with platform-specific
 // remediation instructions. On macOS, TCC-protected directories (Calendars,
 // Reminders, Photos, etc.) return EPERM unless the running application has
@@ -2090,60 +2023,33 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
 		pattern = strings.TrimSuffix(pattern, "/")
 
 		// A leading "**/" demands a "/" before the name and so misses an
-		// entry directly under the source path. It means "at every depth",
+		// entry directly under the source path. An unanchored pattern
-		// which is what an unanchored pattern already does, so drop the
+		// already matches at every depth, so drop the prefix: as in
-		// prefix and match unanchored: as in gitignore, "**/x" and
+		// gitignore, "**/x" then also matches a top-level "x".
-		// "/**/x" then also match a top-level "x".
+		if !anchored {
-		if rest, found := strings.CutPrefix(pattern, "**/"); found {
+			pattern = strings.TrimPrefix(pattern, "**/")
-			pattern = rest
-			anchored = false
 		}
 
-		// Likewise "/**/" inside a pattern demands two slashes, so
+		// Compile the glob pattern
-		// "a/**/*.log" misses "a/x.log". As in gitignore it stands for
+		// For patterns without path separators, we need to match them as components
-		// zero or more directories, so also compile the pattern with it
+		// e.g., ".git" should match ".git" anywhere in the path
-		// made a single "/". The glob library's {a,b} is no help: with a
+		g, err := glob.Compile(pattern, '/')
-		// "**" inside it in mid-pattern, it misses paths it should match.
+		if err != nil {
-		for _, variant := range doubleStarVariants(pattern) {
+			log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
-			// For patterns without path separators, we need to match them as components
-			// e.g., ".git" should match ".git" anywhere in the path
-			g, err := glob.Compile(variant, '/')
-			if err != nil {
-				log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
 
-				break
+			continue
-			}
-
-			compiled = append(compiled, compiledPattern{
-				pattern:  g,
-				anchored: anchored,
-				original: p,
-			})
 		}
+
+		compiled = append(compiled, compiledPattern{
+			pattern:  g,
+			anchored: anchored,
+			original: p,
+		})
 	}
 
 	return compiled
 }
 
-// doubleStarVariants returns pattern once for every way of keeping each
-// "/**/" in it or making it a single "/".
-func doubleStarVariants(pattern string) []string {
-	const doubleStar = "/**/"
-
-	before, after, found := strings.Cut(pattern, doubleStar)
-	if !found {
-		return []string{pattern}
-	}
-
-	var variants []string
-
-	for _, rest := range doubleStarVariants(after) {
-		variants = append(variants, before+doubleStar+rest, before+"/"+rest)
-	}
-
-	return variants
-}
-
 // shouldExclude checks if a path should be excluded based on exclude patterns
 // filePath is the full path to the file
 // rootPath is the root of the backup source directory

internal/storage/faultstore/faultstore.go

@@ -1,16 +1,15 @@
 // Package faultstore provides a storage.Storer wrapper that injects
 // faults on demand, so tests can reproduce the failure modes a real
 // backend exhibits: an upload that fails partway, a backend that reports
-// success while storing nothing, reads that return corrupt or truncated
+// success while storing nothing, and reads that return corrupt or
-// bytes, and a delete the backend refuses. It is the seam called for by
+// truncated bytes. It is the seam called for by the fault-injection
-// the fault-injection tests (sneak/vaultik issue 72) and is meant to be
+// tests (sneak/vaultik issue 72) and is meant to be reused by future
-// reused by future tests rather than re-implemented per case.
+// tests rather than re-implemented per case.
 //
-// The wrapper delegates every method to the inner Storer. Three hooks
+// The wrapper delegates every method to the inner Storer. Two hooks
-// change that: OnPut decides the fate of each write, OnGet decides how
+// change that: OnPut decides the fate of each write, and OnGet decides
-// each read's bytes are returned, and OnDelete decides whether each
+// how each read's bytes are returned. Both are keyed by the object key,
-// delete fails. All are keyed by the object key, so a test can fault
+// so a test can fault only blobs, only metadata, or a single object.
-// only blobs, only metadata, or a single object.
 package faultstore
 
 import (
@@ -26,10 +25,6 @@ import (
 // ErrInjectedUpload is returned by a Put the OnPut hook chose to fail.
 var ErrInjectedUpload = errors.New("faultstore: injected upload failure")
 
-// ErrInjectedDelete is returned by a Delete the OnDelete hook chose to
-// fail.
-var ErrInjectedDelete = errors.New("faultstore: injected delete failure")
-
 // PutAction is the disposition OnPut assigns to a write.
 type PutAction int
 
@@ -71,11 +66,6 @@ type Storer struct {
 	// OnGet, when set, is consulted for every Get with the object key
 	// and damages the returned bytes accordingly.
 	OnGet func(key string) GetFault
-
-	// OnDelete, when set, is consulted before every Delete with the
-	// object key. Returning true fails the delete and leaves the object
-	// in place.
-	OnDelete func(key string) bool
 }
 
 // New wraps inner. inner must be non-nil.
@@ -164,12 +154,8 @@ func (f *Storer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, err
 	return f.inner.Stat(ctx, key)
 }
 
-// Delete removes the object unless OnDelete fails the delete.
+// Delete delegates unchanged.
 func (f *Storer) Delete(ctx context.Context, key string) error {
-	if f.OnDelete != nil && f.OnDelete(key) {
-		return fmt.Errorf("%w for %q", ErrInjectedDelete, key)
-	}
-
 	return f.inner.Delete(ctx, key)
 }
 

internal/storage/file.go

@@ -120,10 +120,9 @@ func (f *FileStorer) Delete(_ context.Context, key string) error {
 	return nil
 }
 
-// List returns all keys with the given prefix, reading through symlinked
+// List returns all keys with the given prefix. It fails when the
-// directories. It fails when the destination directory is missing, or
+// destination directory is missing; a missing prefix under it is an
-// when the prefix or a directory above it is a symlink whose target is
+// empty listing.
-// missing; a prefix that is truly missing is an empty listing.
 func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
 	var keys []string
 
@@ -134,7 +133,8 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 
 	basePath := f.fullPath(prefix)
 
-	exists, err := f.prefixExists(prefix)
+	// Check if the prefix exists
+	exists, err := afero.Exists(f.fs, basePath)
 	if err != nil {
 		return nil, fmt.Errorf("checking path: %w", err)
 	}
@@ -143,7 +143,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 		return keys, nil // Empty list for non-existent prefix
 	}
 
-	err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
+	err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -176,9 +176,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 }
 
 // ListStream returns a channel of ObjectInfo for large result sets. Like
-// List, it reads through symlinked directories and sends an error when
+// List, it sends an error when the destination directory is missing.
-// the destination directory is missing, or when the prefix or a directory
-// above it is a symlink whose target is missing.
 func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo {
 	ch := make(chan ObjectInfo)
 
@@ -194,7 +192,8 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 
 		basePath := f.fullPath(prefix)
 
-		exists, err := f.prefixExists(prefix)
+		// Check if the prefix exists
+		exists, err := afero.Exists(f.fs, basePath)
 		if err != nil {
 			ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
 
@@ -205,7 +204,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 			return // Empty channel for non-existent prefix
 		}
 
-		_ = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
+		_ = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
 			// Check context cancellation
 			select {
 			case <-ctx.Done():
@@ -244,12 +243,11 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 }
 
 // DeletePartialUploads removes every file under prefix whose name ends in
-// tempSuffix. Like List, it reads through symlinked directories and fails
+// tempSuffix. A missing prefix has none to remove.
-// on one whose target is missing. A missing prefix has none to remove.
 func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
 	basePath := f.fullPath(prefix)
 
-	exists, err := f.prefixExists(prefix)
+	exists, err := afero.Exists(f.fs, basePath)
 	if err != nil {
 		return fmt.Errorf("checking path: %w", err)
 	}
@@ -258,7 +256,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
 		return nil
 	}
 
-	err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
+	err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -354,54 +352,6 @@ func (f *FileStorer) writeAtomic(
 	return nil
 }
 
-// walk is afero.Walk, except that it reads through a symlinked directory
-// the way Get and Stat do, where afero.Walk reports the link as one file.
-// afero.Walk uses Lstat, which does not follow a link, whenever the
-// filesystem has one; the struct below carries only the afero.Fs methods,
-// so the walk uses Stat instead. A link back up the tree is followed until
-// the operating system refuses the path for too many levels of links, and
-// the walk then fails.
-func (f *FileStorer) walk(root string, fn filepath.WalkFunc) error {
-	return afero.Walk(struct{ afero.Fs }{f.fs}, root, fn)
-}
-
-// prefixExists reports whether prefix exists under the destination
-// directory. It checks each directory on the way down to prefix in turn,
-// and the first one Stat cannot find decides: if it is not there at all,
-// prefix is missing; if it is a symlink whose target is missing, such as
-// a link to a volume that is not mounted, prefixExists fails, since
-// listing it as empty would let prune delete every blob.
-func (f *FileStorer) prefixExists(prefix string) (bool, error) {
-	lstater, canLstat := f.fs.(afero.Lstater)
-	path := f.basePath
-
-	for name := range strings.SplitSeq(prefix, "/") {
-		path = filepath.Join(path, name)
-
-		_, err := f.fs.Stat(path)
-		if err == nil {
-			continue
-		}
-
-		if !os.IsNotExist(err) {
-			return false, err
-		}
-
-		// Lstat does not follow a symlink at path, so it finds the link
-		// that Stat could not follow.
-		if canLstat {
-			_, _, lstatErr := lstater.LstatIfPossible(path)
-			if lstatErr == nil {
-				return false, err
-			}
-		}
-
-		return false, nil
-	}
-
-	return true, nil
-}
-
 // fullPath returns the full filesystem path for a key.
 func (f *FileStorer) fullPath(key string) string {
 	return filepath.Join(f.basePath, key)

internal/storage/file_backend_test.go

@@ -4,10 +4,7 @@ import (
 	"context"
 	"errors"
 	"io/fs"
-	"os"
 	"path/filepath"
-	"reflect"
-	"strings"
 	"testing"
 
 	"sneak.berlin/go/vaultik/internal/storage"
@@ -63,232 +60,3 @@ func TestFileStorerListMissingDestination(t *testing.T) {
 		t.Errorf("ListStream error = %v, want a not-exist error", streamErr)
 	}
 }
-
-// The two directories at the top of a destination, either of which the
-// tests below replace with a symlink.
-const (
-	metadataDir = "metadata"
-	blobsDir    = "blobs"
-)
-
-// moveBehindSymlink moves base/dir to a directory elsewhere and puts a
-// symlink to it in its place.
-func moveBehindSymlink(t *testing.T, base, dir string) {
-	t.Helper()
-
-	linkPath := filepath.Join(base, dir)
-	movedPath := filepath.Join(t.TempDir(), dir)
-
-	err := os.Rename(linkPath, movedPath)
-	if err != nil {
-		t.Fatalf("moving %s away: %v", dir, err)
-	}
-
-	err = os.Symlink(movedPath, linkPath)
-	if err != nil {
-		t.Fatalf("linking %s: %v", dir, err)
-	}
-}
-
-// TestFileStorerListsThroughSymlinkedDirectory checks that List and
-// ListStream read through a symlinked metadata/ or blobs/ directory, as Get
-// and Stat do, instead of listing the link as one key. Listing the whole
-// destination meets the link part-way through the walk; listing its prefix
-// starts the walk at the link. A prefix missing behind the link lists as
-// empty.
-func TestFileStorerListsThroughSymlinkedDirectory(t *testing.T) {
-	t.Parallel()
-
-	for _, dir := range []string{metadataDir, blobsDir} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			base := t.TempDir()
-			key := dir + "/aa/bb/object"
-
-			s, err := storage.NewFileStorer(base)
-			if err != nil {
-				t.Fatalf("NewFileStorer: %v", err)
-			}
-
-			err = s.Put(context.Background(), key, strings.NewReader("data"))
-			if err != nil {
-				t.Fatalf("Put: %v", err)
-			}
-
-			moveBehindSymlink(t, base, dir)
-
-			want := []string{key}
-
-			for _, prefix := range []string{"", dir + "/"} {
-				got := listSorted(t, s, prefix)
-				if !reflect.DeepEqual(got, want) {
-					t.Errorf("List(%q) = %v, want %v", prefix, got, want)
-				}
-
-				got = listStreamKeys(t, s, prefix)
-				if !reflect.DeepEqual(got, want) {
-					t.Errorf("ListStream(%q) = %v, want %v", prefix, got, want)
-				}
-			}
-
-			absent := dir + "/absent/"
-
-			if got := listSorted(t, s, absent); len(got) != 0 {
-				t.Errorf("List(%q) = %v, want empty", absent, got)
-			}
-
-			if got := listStreamKeys(t, s, absent); len(got) != 0 {
-				t.Errorf("ListStream(%q) = %v, want empty", absent, got)
-			}
-		})
-	}
-}
-
-// TestFileStorerListFailsOnSymlinkWithMissingTarget checks that List and
-// ListStream fail on a symlinked metadata/ or blobs/ directory whose target
-// is missing, such as a link to a volume that is not mounted, as they do on
-// a missing destination directory, instead of listing it as empty. That
-// holds for a prefix below the link too.
-func TestFileStorerListFailsOnSymlinkWithMissingTarget(t *testing.T) {
-	t.Parallel()
-
-	ctx := context.Background()
-
-	for _, dir := range []string{metadataDir, blobsDir} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			base := t.TempDir()
-
-			s, err := storage.NewFileStorer(base)
-			if err != nil {
-				t.Fatalf("NewFileStorer: %v", err)
-			}
-
-			unmounted := filepath.Join(t.TempDir(), "unmounted")
-
-			err = os.Symlink(unmounted, filepath.Join(base, dir))
-			if err != nil {
-				t.Fatalf("Symlink: %v", err)
-			}
-
-			for _, prefix := range []string{"", dir + "/", dir + "/aa/"} {
-				keys, err := s.List(ctx, prefix)
-				if !errors.Is(err, fs.ErrNotExist) {
-					t.Errorf("List(%q) = %v, %v; want a not-exist error",
-						prefix, keys, err)
-				}
-
-				var streamErr error
-
-				for object := range s.ListStream(ctx, prefix) {
-					if object.Err != nil {
-						streamErr = object.Err
-					}
-				}
-
-				if !errors.Is(streamErr, fs.ErrNotExist) {
-					t.Errorf("ListStream(%q) error = %v, want a not-exist error",
-						prefix, streamErr)
-				}
-			}
-		})
-	}
-}
-
-// TestFileStorerListFailsOnSymlinkLoop checks that a symlink pointing back
-// at the destination makes List and ListStream fail instead of walking it
-// forever.
-func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
-	t.Parallel()
-
-	ctx := context.Background()
-	base := t.TempDir()
-
-	s, err := storage.NewFileStorer(base)
-	if err != nil {
-		t.Fatalf("NewFileStorer: %v", err)
-	}
-
-	err = s.Put(ctx, testBlobKey, strings.NewReader("data"))
-	if err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-
-	err = os.Symlink(base, filepath.Join(base, "blobs", "loop"))
-	if err != nil {
-		t.Fatalf("Symlink: %v", err)
-	}
-
-	keys, err := s.List(ctx, "blobs/")
-	if err == nil {
-		t.Errorf("List = %d keys and no error, want an error", len(keys))
-	}
-
-	var streamErr error
-
-	for object := range s.ListStream(ctx, "blobs/") {
-		if object.Err != nil {
-			streamErr = object.Err
-		}
-	}
-
-	if streamErr == nil {
-		t.Error("ListStream sent no error, want one")
-	}
-}
-
-// TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory checks that
-// DeletePartialUploads, which remote nuke uses, removes a leftover temp
-// file behind a symlinked metadata/ or blobs/ directory and keeps the
-// object at the real key.
-func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
-	t.Parallel()
-
-	ctx := context.Background()
-
-	for _, dir := range []string{metadataDir, blobsDir} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			base := t.TempDir()
-			key := dir + "/aa/bb/object"
-
-			s, err := storage.NewFileStorer(base)
-			if err != nil {
-				t.Fatalf("NewFileStorer: %v", err)
-			}
-
-			err = s.Put(ctx, key, strings.NewReader("data"))
-			if err != nil {
-				t.Fatalf("Put: %v", err)
-			}
-
-			leftover := filepath.Join(base, key+"-123456.partial")
-
-			err = os.WriteFile(leftover, []byte("half"), 0o600)
-			if err != nil {
-				t.Fatalf("writing leftover temp file: %v", err)
-			}
-
-			moveBehindSymlink(t, base, dir)
-
-			err = s.DeletePartialUploads(ctx, dir+"/")
-			if err != nil {
-				t.Fatalf("DeletePartialUploads: %v", err)
-			}
-
-			// leftover runs through the link.
-			_, err = os.Stat(leftover)
-			if !os.IsNotExist(err) {
-				t.Errorf("leftover temp file was not removed: %v", err)
-			}
-
-			_, err = s.Stat(ctx, key)
-			if err != nil {
-				t.Errorf("Stat of the real key: %v", err)
-			}
-		})
-	}
-}

internal/storage/rclone.go

@@ -7,12 +7,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 	"time"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/operations"
 
@@ -35,15 +33,6 @@ type RcloneStorer struct {
 // The remote parameter is the rclone remote name (as configured via `rclone config`).
 // The path parameter is the path within the remote.
 func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, error) {
-	// rclone reads RCLONE_CONFIG only as the value of its --config flag,
-	// which vaultik does not have, so it is applied here as that flag would.
-	if configPath, ok := os.LookupEnv("RCLONE_CONFIG"); ok {
-		err := config.SetConfigPath(configPath)
-		if err != nil {
-			return nil, fmt.Errorf("using RCLONE_CONFIG %q: %w", configPath, err)
-		}
-	}
-
 	// Install the default config file handler
 	configfile.Install()
 

internal/storage/rclone_test.go

@@ -11,7 +11,6 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"sneak.berlin/go/vaultik/internal/storage"
 )
@@ -391,41 +390,3 @@ func TestNewRcloneStorerUnknownRemote(t *testing.T) {
 		t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
 	}
 }
-
-// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only
-// in the file RCLONE_CONFIG names is found and written to, as rclone itself
-// would.
-func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
-	// rclone keeps the config file path in a process-global variable, which
-	// NewRcloneStorer leaves pointing at this test's file.
-	previous := config.GetConfigPath()
-
-	t.Cleanup(func() { _ = config.SetConfigPath(previous) })
-
-	confFile := filepath.Join(t.TempDir(), "rclone.conf")
-
-	err := os.WriteFile(confFile, []byte("[loc]\ntype = local\n"), 0o600)
-	if err != nil {
-		t.Fatalf("writing rclone config: %v", err)
-	}
-
-	t.Setenv("RCLONE_CONFIG", confFile)
-
-	dir := t.TempDir()
-	ctx := context.Background()
-
-	s, err := storage.NewRcloneStorer(ctx, "loc", dir)
-	if err != nil {
-		t.Fatalf("NewRcloneStorer: %v", err)
-	}
-
-	err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
-	if err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-
-	_, err = os.Stat(filepath.Join(dir, testBlobKey))
-	if err != nil {
-		t.Errorf("object is not in the remote's directory: %v", err)
-	}
-}

internal/storage/s3_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"slices"
@@ -23,16 +22,13 @@ import (
 // s3TestBucket is the bucket created for each in-process S3 server.
 const s3TestBucket = "test-bucket"
 
-// Credentials for the tests' S3 clients. The test servers accept any.
+// Credentials for the tests that build a storer from a config.Config. The
+// in-process S3 server accepts any.
 const (
 	s3TestAccessKeyID     = "key"
 	s3TestSecretAccessKey = "secret"
 )
 
-// s3TestRegion is the region of the S3 clients built here with
-// s3.NewClient. The test servers ignore it.
-const s3TestRegion = "us-east-1"
-
 // newS3Storer builds an s3:// backend backed by a fresh in-process
 // S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
 // over httptest) that internal/s3 and the not-found regression test use,
@@ -59,7 +55,7 @@ func newS3Storer(t *testing.T) storage.Storer {
 		Bucket:          s3TestBucket,
 		AccessKeyID:     "test",
 		SecretAccessKey: "test",
-		Region:          s3TestRegion,
+		Region:          "us-east-1",
 	})
 	if err != nil {
 		t.Fatalf("new client: %v", err)
@@ -97,68 +93,6 @@ func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
 	}
 }
 
-// TestS3ListingCutShortIsAnError checks that List and ListStream fail on a
-// listing page marked truncated that carries no continuation token, as a
-// server answering V2 listings V1-style sends it. Taken as the last page,
-// it would hide every later key, and prune would treat the blobs of the
-// snapshots listed there as unreferenced.
-func TestS3ListingCutShortIsAnError(t *testing.T) {
-	t.Parallel()
-
-	const (
-		prefix        = "metadata/"
-		truncatedPage = `<?xml version="1.0" encoding="UTF-8"?>
-<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-  <Name>test-bucket</Name>
-  <Prefix>metadata/</Prefix>
-  <KeyCount>1</KeyCount>
-  <MaxKeys>1</MaxKeys>
-  <IsTruncated>true</IsTruncated>
-  <Contents>
-    <Key>metadata/snap/manifest.json.zst</Key>
-    <Size>8</Size>
-  </Contents>
-</ListBucketResult>`
-	)
-
-	srv := httptest.NewServer(http.HandlerFunc(
-		func(w http.ResponseWriter, _ *http.Request) {
-			_, _ = io.WriteString(w, truncatedPage)
-		}))
-	t.Cleanup(srv.Close)
-
-	client, err := s3.NewClient(context.Background(), s3.Config{
-		Endpoint:        srv.URL,
-		Bucket:          s3TestBucket,
-		AccessKeyID:     s3TestAccessKeyID,
-		SecretAccessKey: s3TestSecretAccessKey,
-		Region:          s3TestRegion,
-	})
-	if err != nil {
-		t.Fatalf("new client: %v", err)
-	}
-
-	storer := storage.NewS3Storer(client)
-	ctx := context.Background()
-
-	keys, err := storer.List(ctx, prefix)
-	if !errors.Is(err, s3.ErrNoContinuationToken) {
-		t.Errorf("List = %q, %v; want ErrNoContinuationToken", keys, err)
-	}
-
-	var streamErr error
-
-	for obj := range storer.ListStream(ctx, prefix) {
-		if obj.Err != nil {
-			streamErr = obj.Err
-		}
-	}
-
-	if !errors.Is(streamErr, s3.ErrNoContinuationToken) {
-		t.Errorf("ListStream error = %v, want ErrNoContinuationToken", streamErr)
-	}
-}
-
 // TestS3URLPrefixKeyLayout pins the bucket keys an s3:// URL reads and
 // writes: the README's remote storage layout, with the prefix joined to
 // each key by one "/". s3://b/p and s3://b/p/ must be the same

internal/storage/url.go

@@ -30,12 +30,6 @@ var (
 		"storage URL must not carry credentials; " +
 			"set s3.access_key_id and s3.secret_access_key in the config instead")
 	ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
-	ErrURLBadQuery     = errors.New(
-		"storage URL query cannot be decoded; " +
-			"check it for a ; or a % not followed by two hex digits")
-	ErrRcloneRemoteParams = errors.New(
-		"rclone URL must not carry parameters in the remote name; " +
-			"set them on the remote with rclone config instead")
 )
 
 // URL represents a parsed storage URL.
@@ -96,9 +90,8 @@ func parseFileURL(rawPath string) (*URL, error) {
 }
 
 // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
-// the userinfo or in the endpoint parameter's userinfo, a query that
+// the userinfo and any query parameter other than endpoint, region and
-// cannot be decoded, and any query parameter other than endpoint, region
+// ssl, so a credential-bearing URL is never stored or echoed.
-// and ssl, so a credential-bearing URL is never stored or echoed.
 func parseS3URL(rawURL string) (*URL, error) {
 	u, err := url.Parse(rawURL)
 	if err != nil {
@@ -114,26 +107,13 @@ func parseS3URL(rawURL string) (*URL, error) {
 		return nil, ErrMissingBucket
 	}
 
-	// u.Query() would silently drop a part it cannot decode, endpoint
+	query := u.Query()
-	// included. The decode error is not wrapped: it can quote the value.
-	query, err := url.ParseQuery(u.RawQuery)
-	if err != nil {
-		return nil, ErrURLBadQuery
-	}
 
 	err = rejectUnknownParams(query, "endpoint", "region", "ssl")
 	if err != nil {
 		return nil, err
 	}
 
-	// A host name never contains "@", so one in an endpoint, with or
-	// without a scheme, can only introduce userinfo.
-	for _, endpoint := range query["endpoint"] {
-		if strings.Contains(endpoint, "@") {
-			return nil, ErrURLCredentials
-		}
-	}
-
 	return &URL{
 		Scheme:   schemeS3,
 		Bucket:   bucket,
@@ -145,23 +125,9 @@ func parseS3URL(rawURL string) (*URL, error) {
 }
 
 // parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
-// query parameters, so credentials in the userinfo, parameters in the
+// query parameters, so credentials in the userinfo and any parameter at
-// remote name, a query that cannot be decoded and any query parameter at
 // all are rejected rather than silently ignored.
 func parseRcloneURL(rawURL string) (*URL, error) {
-	// The remote name runs up to the first "/", "?" or "#". It never
-	// contains a comma; one starts rclone's remote,param=value form,
-	// whose values can be credentials. This is checked before url.Parse,
-	// whose error for such a name can print part of it.
-	remote := strings.TrimPrefix(rawURL, "rclone://")
-	if end := strings.IndexAny(remote, "/?#"); end >= 0 {
-		remote = remote[:end]
-	}
-
-	if strings.Contains(remote, ",") {
-		return nil, ErrRcloneRemoteParams
-	}
-
 	u, err := url.Parse(rawURL)
 	if err != nil {
 		return nil, wrapParseError(err)
@@ -171,19 +137,12 @@ func parseRcloneURL(rawURL string) (*URL, error) {
 		return nil, ErrURLCredentials
 	}
 
-	remote = u.Host
+	remote := u.Host
 	if remote == "" {
 		return nil, ErrMissingRemote
 	}
 
-	// As in parseS3URL: u.Query() would drop a part it cannot decode, and
+	err = rejectUnknownParams(u.Query())
-	// the decode error can quote the value.
-	query, err := url.ParseQuery(u.RawQuery)
-	if err != nil {
-		return nil, ErrURLBadQuery
-	}
-
-	err = rejectUnknownParams(query)
 	if err != nil {
 		return nil, err
 	}

internal/storage/url_parse_test.go

@@ -150,19 +150,6 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
 			wantErr: storage.ErrURLCredentials,
 			secrets: []string{key, secret, "mybucket"},
 		},
-		{
-			name: "s3 userinfo in endpoint",
-			raw: "s3://mybucket?endpoint=https://" +
-				key + ":" + secret + "@minio.example.com",
-			wantErr: storage.ErrURLCredentials,
-			secrets: []string{key, secret},
-		},
-		{
-			name:    "s3 userinfo in endpoint without scheme",
-			raw:     "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
-			wantErr: storage.ErrURLCredentials,
-			secrets: []string{key, secret},
-		},
 		{
 			name:    "s3 unknown query param",
 			raw:     "s3://mybucket?access_key=" + key + "&secret=" + secret,
@@ -216,112 +203,6 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
 	}
 }
 
-// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
-// remote name carrying parameters (rclone's "remote,param=value" form,
-// which can hold a credential) is rejected without echoing the value,
-// including the URLs that url.Parse itself refuses.
-func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
-	t.Parallel()
-
-	cases := []struct {
-		name    string
-		raw     string
-		secrets []string // must not appear in the error message
-	}{
-		{
-			name:    "configured remote",
-			raw:     "rclone://gdrive,client_secret=topsecret/backups",
-			secrets: []string{"topsecret"},
-		},
-		{
-			name:    "remote defined inline",
-			raw:     "rclone://:s3,access_key_id=KEY,secret_access_key=SECRET/bucket",
-			secrets: []string{"KEY", "SECRET"},
-		},
-		{
-			name:    "colon in a parameter value",
-			raw:     "rclone://gdrive,client_secret=ab:cd/backups",
-			secrets: []string{"cd"},
-		},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-
-			_, err := storage.ParseStorageURL(tc.raw)
-			if !errors.Is(err, storage.ErrRcloneRemoteParams) {
-				t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
-					tc.raw, err, storage.ErrRcloneRemoteParams)
-			}
-
-			for _, secret := range tc.secrets {
-				if strings.Contains(err.Error(), secret) {
-					t.Errorf("error message leaked %q: %v", secret, err.Error())
-				}
-			}
-		})
-	}
-}
-
-// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
-// URL whose query cannot be decoded is rejected, not read with that part
-// of the query left out, and that the error does not echo it.
-func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
-	t.Parallel()
-
-	const (
-		key    = "AKIAKEY"
-		secret = "topsecret"
-	)
-
-	cases := []struct {
-		name string
-		raw  string
-	}{
-		{
-			name: "semicolon",
-			raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
-				"@minio.example.com:9000;region=us-east-1",
-		},
-		{
-			name: "bad percent escape",
-			raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
-				"%zz@minio.example.com:9000",
-		},
-		{
-			name: "rclone semicolon",
-			raw:  "rclone://gdrive/backups?token=" + secret + ";x=1",
-		},
-		{
-			name: "rclone bad percent escape",
-			raw:  "rclone://gdrive/backups?token=" + secret + "%zz",
-		},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-
-			_, err := storage.ParseStorageURL(tc.raw)
-			if err == nil {
-				t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
-			}
-
-			if !errors.Is(err, storage.ErrURLBadQuery) {
-				t.Errorf("ParseStorageURL(%q) error = %v, want %v",
-					tc.raw, err, storage.ErrURLBadQuery)
-			}
-
-			for _, s := range []string{key, secret, "minio.example.com"} {
-				if strings.Contains(err.Error(), s) {
-					t.Errorf("error message leaked %q: %v", s, err.Error())
-				}
-			}
-		})
-	}
-}
-
 // TestParseStorageURLParseFailureHidesURL checks that when url.Parse
 // itself fails, the wrapped error carries only the inner cause, not the
 // *url.Error whose text embeds the raw (possibly credential-bearing) URL.

internal/vaultik/blob_fetch.go

@@ -104,8 +104,7 @@ func (v *Vaultik) FetchAndDecryptBlob(
 
 // FetchBlob downloads a blob and returns a reader for the encrypted data.
 // The reader is closed as soon as ctx is cancelled, which ends a read
-// blocked inside it. A storage error is returned as is: restore and deep
+// blocked inside it.
-// verify name the blob in their own errors.
 func (v *Vaultik) FetchBlob(
 	ctx context.Context, blobHash string,
 ) (io.ReadCloser, error) {
@@ -120,7 +119,7 @@ func (v *Vaultik) FetchBlob(
 
 	rc, err := v.Storage.Get(ctx, blobPath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
 	}
 
 	return newCloseOnCancelReader(ctx, rc), nil

internal/vaultik/blobcache.go

@@ -39,9 +39,9 @@ type blobDiskCacheEntry struct {
 // internally and will fall back to LRU eviction if curBytes exceeds
 // maxBytes. Restore passes math.MaxInt64 as maxBytes and drives
 // eviction itself via Delete() through restoreSweeper, which deletes
-// each blob once no file still to be restored in this run needs it.
+// each blob the moment every file that references its chunks has been
-// LRU never fires under that configuration; it is kept as a safety net
+// written. LRU never fires under that configuration; it is kept as a
-// for callers that don't manage eviction themselves.
+// safety net for callers that don't manage eviction themselves.
 //
 // Get/ReadAt/peak-Len counters are debugging instrumentation used by
 // tests to assert that the restore code path uses ReadAt rather than

internal/vaultik/deep_verify_test.go

@@ -1,20 +1,15 @@
 package vaultik_test
 
 import (
-	"bytes"
 	"context"
-	"crypto/rand"
 	"io"
 	"os"
-	"path"
 	"path/filepath"
 	"testing"
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
@@ -82,180 +77,6 @@ func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
 		"deep verify should fail on a corrupted blob")
 }
 
-// TestDeepVerifyJSONCountsBlobs checks the verified and missing counts
-// that deep verify reports under --json: on a healthy snapshot, with a
-// damaged blob, and with missing blobs. Deep verify stops at the first
-// damaged blob, so the blobs read before it are the ones verified; when
-// blobs are missing it reads no blob contents, but counts every missing
-// blob.
-func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "remote")
-
-	cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
-
-	healthy, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.NoError(t, err)
-	require.Equal(t, "ok", healthy.Status)
-	require.Len(t, keys, healthy.BlobCount)
-	require.Equal(t, healthy.BlobCount, healthy.Verified)
-	require.Zero(t, healthy.Missing)
-
-	// Damage the last blob read, keeping its length, so every blob before
-	// it is read and passes first.
-	lastPath := filepath.Join(storeDir, keys[len(keys)-1])
-	blob, err := afero.ReadFile(fs, lastPath)
-	require.NoError(t, err)
-
-	blob[len(blob)/2] ^= 0xff
-	require.NoError(t, afero.WriteFile(fs, lastPath, blob, 0o600))
-
-	got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Equal(t, healthy.BlobCount-1, got.Verified)
-	require.Zero(t, got.Missing)
-
-	// Delete the first blob and the damaged last one: both must be
-	// counted as missing, not only the first one found.
-	var deletedSize int64
-
-	for _, key := range []string{keys[0], keys[len(keys)-1]} {
-		info, statErr := storer.Stat(ctx, key)
-		require.NoError(t, statErr)
-
-		deletedSize += info.Size
-
-		require.NoError(t, storer.Delete(ctx, key))
-	}
-
-	got, err = deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Equal(t, 2, got.Missing)
-	require.Equal(t, deletedSize, got.MissingSize)
-	require.Zero(t, got.Verified)
-}
-
-// TestDeepVerifyJSONCountsWrongSizeBlob checks that under --deep --json a
-// blob of the wrong size is counted in mismatched and fails the run, and
-// that the check goes on past it to count a missing blob after it. The
-// store is on disk, so the message must not call it S3.
-func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "remote")
-
-	cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
-
-	// Make the first blob checked longer, and delete the last one.
-	firstPath := filepath.Join(storeDir, keys[0])
-	blob, err := afero.ReadFile(fs, firstPath)
-	require.NoError(t, err)
-
-	blob = append(blob, []byte("extra")...)
-	require.NoError(t, afero.WriteFile(fs, firstPath, blob, 0o600))
-	require.NoError(t, storer.Delete(ctx, keys[len(keys)-1]))
-
-	got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Equal(t, 1, got.Mismatched)
-	require.Equal(t, 1, got.Missing)
-	require.Zero(t, got.Verified)
-	require.Contains(t, got.ErrorMessage, "with the wrong size")
-	require.NotContains(t, got.ErrorMessage, "S3")
-}
-
-// TestDeepVerifyStopsOnBlobThatCannotBeChecked checks that a blob that is
-// present in storage but cannot be checked is not counted as missing:
-// deep verify stops and fails, saying it could not check the blob, with
-// the storage error.
-func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "remote")
-
-	cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
-	failing := statFailingStorer{Storer: storer, failKeys: []string{keys[0]}}
-
-	got, err := deepVerifyJSON(ctx, t, cfg, failing, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Zero(t, got.Missing)
-	require.Contains(t, got.ErrorMessage,
-		"could not check blob "+path.Base(keys[0]))
-	require.Contains(t, got.ErrorMessage, errStatDenied.Error())
-}
-
-// backUpSeveralBlobs backs up random data, which does not compress or
-// deduplicate, so it fills at least three blobs in a store on disk at
-// storeDir. It returns the blobs' storage keys sorted, which is hash
-// order, the order deep verify reads them in.
-func backUpSeveralBlobs(
-	ctx context.Context, t *testing.T, storeDir string,
-) (*config.Config, *storage.FileStorer, string, []string) {
-	t.Helper()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "source")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	chunkSize := int64(32 * 1024)
-	maxBlobSize := int64(128 * 1024)
-
-	data := make([]byte, maxBlobSize*4)
-	_, err := rand.Read(data)
-	require.NoError(t, err)
-	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
-	require.NoError(t, afero.WriteFile(fs,
-		filepath.Join(dataDir, "data.bin"), data, 0o644))
-
-	cfg, storer, snapshotID := runFileStorageBackup(
-		ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
-
-	keys := storedBlobKeys(t, storeDir)
-	require.GreaterOrEqual(t, len(keys), 3)
-
-	return cfg, storer, snapshotID, keys
-}
-
-// deepVerifyJSON runs deep verify with --json on snapshotID and returns
-// the decoded result and the error RunDeepVerify returned.
-func deepVerifyJSON(
-	ctx context.Context, t *testing.T,
-	cfg *config.Config, storer storage.Storer, snapshotID string,
-) (vaultik.VerifyResult, error) {
-	t.Helper()
-
-	var out bytes.Buffer
-
-	v := &vaultik.Vaultik{
-		Config:  cfg,
-		Storage: storer,
-		Fs:      afero.NewOsFs(),
-		Stdout:  &out,
-		Stderr:  io.Discard,
-		UI:      ui.NewWithColor(io.Discard, false),
-	}
-	v.SetContext(ctx)
-
-	err := v.RunDeepVerify(snapshotID,
-		&vaultik.VerifyOptions{Deep: true, JSON: true})
-
-	return decodeVerifyResult(t, out.Bytes()), err
-}
-
 // corruptOneBlob flips a middle byte of the first blob file found under
 // blobsDir, leaving the file length unchanged.
 func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {

internal/vaultik/fault_injection_test.go

@@ -661,18 +661,17 @@ func faultScannerFactory(
 ) snapshot.ScannerFactory {
 	return func(params snapshot.ScannerParams) *snapshot.Scanner {
 		return snapshot.NewScanner(snapshot.ScannerConfig{
-			FS:                 params.Fs,
+			FS:               params.Fs,
-			Storage:            storer,
+			Storage:          storer,
-			ChunkSize:          faultChunkSize,
+			ChunkSize:        faultChunkSize,
-			MaxBlobSize:        faultMaxBlobSize,
+			MaxBlobSize:      faultMaxBlobSize,
-			CompressionLevel:   cfg.CompressionLevel,
+			CompressionLevel: cfg.CompressionLevel,
-			AgeRecipients:      cfg.AgeRecipients,
+			AgeRecipients:    cfg.AgeRecipients,
-			Repositories:       repos,
+			Repositories:     repos,
-			EnableProgress:     params.EnableProgress,
+			EnableProgress:   params.EnableProgress,
-			UI:                 params.UI,
+			UI:               params.UI,
-			Exclude:            params.Exclude,
+			Exclude:          params.Exclude,
-			SkipErrors:         params.SkipErrors,
+			SkipErrors:       params.SkipErrors,
-			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/vaultik/nuke_remote_test.go

@@ -5,28 +5,25 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
-	"sneak.berlin/go/vaultik/internal/storage/faultstore"
 )
 
 // TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file
 // under a file:// destination, including the `.partial` files uploads
 // killed part-way leave next to a snapshot's metadata and next to where a
-// blob would have been, a snapshot's database whose manifest was never
+// blob would have been.
-// uploaded, and a file under blobs/ whose name is not a blob hash.
 func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
 
 	ctx := context.Background()
 	storeDir := filepath.Join(t.TempDir(), "store")
-	v, repos, out := backUpToFileDestination(ctx, t, storeDir)
+	v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
 
 	snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
 	require.NoError(t, err)
@@ -39,9 +36,6 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 			"db.zst.age-123456.partial"),
 		filepath.Join(storeDir, "blobs", hash[:2], hash[2:4],
 			hash+"-123456.partial"),
-		filepath.Join(storeDir, "metadata",
-			snapshot.RemoteSnapshotKey("no-manifest"), "db.zst.age"),
-		filepath.Join(storeDir, "blobs", "0b", "01", ".DS_Store"),
 	}
 
 	for _, leftover := range leftovers {
@@ -67,110 +61,4 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 		})
 	require.NoError(t, err)
 	assert.Empty(t, files)
-
-	// Nuke deletes the blobs itself, so advice to run prune is wrong.
-	assert.Contains(t, out.String(), "Backup destination store is now empty.")
-	assert.NotContains(t, out.String(), "vaultik prune")
-}
-
-// TestNukeRemoteFailsWhenADeleteFails checks that remote nuke returns an
-// error when a snapshot's metadata or a blob cannot be deleted, does not
-// report the store empty, and advises running nuke again: prune deletes
-// no metadata.
-func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	for _, refusedPrefix := range []string{"metadata/", "blobs/"} {
-		t.Run(refusedPrefix, func(t *testing.T) {
-			t.Parallel()
-
-			ctx := context.Background()
-			storeDir := filepath.Join(t.TempDir(), "store")
-			v, _, out := backUpToFileDestination(ctx, t, storeDir)
-
-			store := faultstore.New(v.Storage)
-			store.OnDelete = func(key string) bool {
-				return strings.HasPrefix(key, refusedPrefix)
-			}
-			v.Storage = store
-
-			require.Error(t, v.NukeRemote(true))
-
-			assert.NotContains(t, out.String(), "now empty")
-			assert.Contains(t, out.String(), "vaultik remote nuke --force")
-			assert.NotContains(t, out.String(), "vaultik prune")
-		})
-	}
-}
-
-// TestNukeRemoteNamesWhatIsLeft checks that remote nuke, when an object
-// it does not recognise cannot be deleted, does not report the store
-// empty but names that object and fails.
-func TestNukeRemoteNamesWhatIsLeft(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, _, out := backUpToFileDestination(ctx, t, storeDir)
-
-	const leftKey = "blobs/0b/01/.DS_Store"
-
-	leftPath := filepath.Join(storeDir, filepath.FromSlash(leftKey))
-	require.NoError(t, os.MkdirAll(filepath.Dir(leftPath), 0o750))
-	require.NoError(t, os.WriteFile(leftPath, []byte("finder"), 0o600))
-
-	store := faultstore.New(v.Storage)
-	store.OnDelete = func(key string) bool { return key == leftKey }
-	v.Storage = store
-
-	require.Error(t, v.NukeRemote(true))
-
-	assert.Contains(t, out.String(), leftKey)
-	assert.NotContains(t, out.String(), "now empty")
-	assert.Contains(t, out.String(), "vaultik remote nuke --force")
-}
-
-// TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice checks that
-// remote nuke, when the local index belongs to a different destination
-// than the configured one, fails without advising to run nuke again:
-// that run would fail the same way.
-func TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, _, out := backUpToFileDestination(ctx, t, storeDir)
-
-	v.Config.StorageURL = "file://" + filepath.Join(t.TempDir(), "elsewhere")
-
-	err := v.NukeRemote(true)
-
-	require.ErrorContains(t, err, "bound to a different backup destination")
-	assert.NotContains(t, out.String(), "vaultik remote nuke --force")
-}
-
-// TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved checks that a
-// snapshot remote nuke cannot remove from the local index fails the
-// run: nuke then leaves that snapshot's metadata on the destination
-// store.
-func TestNukeRemoteFailsWhenALocalSnapshotCannotBeRemoved(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, _, out := backUpToFileDestination(ctx, t, storeDir)
-
-	_, err := v.DB.Conn().ExecContext(ctx, `
-		CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
-		BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
-	require.NoError(t, err)
-
-	require.Error(t, v.NukeRemote(true))
-
-	assert.NotContains(t, out.String(), "now empty")
-	assert.Contains(t, out.String(), "vaultik remote nuke --force")
 }

internal/vaultik/prune.go

@@ -19,19 +19,6 @@ type PruneOptions struct {
 var errNukeRequiresForce = errors.New(
 	"nuke requires --force (this deletes ALL remote snapshots and blobs)")
 
-// errBlobDeleteFailed reports unreferenced blobs prune could not delete.
-var errBlobDeleteFailed = errors.New("blob(s) could not be deleted")
-
-// errObjectsLeft reports objects remote nuke still finds on the backup
-// destination store after deleting.
-var errObjectsLeft = errors.New(
-	"object(s) still on the backup destination store")
-
-// nukeCommandHint is the command suggested when a remote nuke could not
-// delete everything: running it again deletes what is left, which
-// `vaultik prune` does not do for snapshot metadata.
-const nukeCommandHint = "vaultik remote nuke --force"
-
 // metadataDirName is the top-level remote directory holding snapshot
 // metadata.
 const metadataDirName = "metadata"
@@ -40,11 +27,10 @@ const metadataDirName = "metadata"
 // (blobs/<aa>/<bb>/<hash>).
 const blobKeyParts = 4
 
-// NukeRemote deletes every object under metadata/ and blobs/ in remote
+// NukeRemote deletes every snapshot's metadata and every blob from remote
-// storage, whatever its name, along with any object an upload cut off
+// storage, along with any object an upload cut off part-way left under a
-// part-way left under a temporary `.partial` name. It returns
+// temporary `.partial` name. After this returns successfully the bucket
-// successfully only when a listing afterwards finds nothing left there,
+// prefix is empty and the next backup starts from scratch.
-// and the next backup then starts from scratch.
 //
 // Refuses to run unless force is true. The caller is responsible for
 // confirming with the user.
@@ -53,30 +39,6 @@ func (v *Vaultik) NukeRemote(force bool) error {
 		return errNukeRequiresForce
 	}
 
-	// Checked before anything is deleted: running nuke again would fail
-	// the same way, so the advice below must not follow this error.
-	err := v.EnsureStorageBinding()
-	if err != nil {
-		return err
-	}
-
-	err = v.emptyDestinationStore()
-	if err != nil {
-		v.UI.Warningf("Run '%s' again to delete what is left on the "+
-			"backup destination store.", nukeCommandHint)
-
-		return err
-	}
-
-	v.UI.Completef("Backup destination store is now empty.")
-
-	return nil
-}
-
-// emptyDestinationStore does the deleting for NukeRemote and stops at
-// the first step that fails. It also fails when objects are still
-// listed at the end, naming each one.
-func (v *Vaultik) emptyDestinationStore() error {
 	v.UI.Beginf("Removing all snapshot metadata from backup destination store.")
 
 	_, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true})
@@ -84,62 +46,27 @@ func (v *Vaultik) emptyDestinationStore() error {
 		return fmt.Errorf("removing all snapshots: %w", err)
 	}
 
-	v.UI.Beginf("Removing everything still present in backup destination store.")
+	v.UI.Beginf("Removing any blobs still present in backup destination store.")
 
-	var left []string
+	err = v.PruneBlobs(&PruneOptions{Force: true})
+	if err != nil {
+		return fmt.Errorf("pruning blobs: %w", err)
+	}
 
+	// The file and rclone listings skip `.partial` objects, so the two
+	// steps above never delete them.
 	for _, prefix := range []string{"metadata/", "blobs/"} {
-		keys, err := v.deleteEverythingUnder(prefix)
+		err = v.Storage.DeletePartialUploads(v.ctx, prefix)
 		if err != nil {
-			return err
+			return fmt.Errorf("deleting partial uploads: %w", err)
 		}
-
-		left = append(left, keys...)
 	}
 
-	for _, key := range left {
+	v.UI.Completef("Backup destination store is now empty.")
-		v.UI.Warningf("Still on the backup destination store: %s", key)
-	}
-
-	if len(left) > 0 {
-		return fmt.Errorf("%d %w", len(left), errObjectsLeft)
-	}
 
 	return nil
 }
 
-// deleteEverythingUnder deletes every object under prefix, whatever its
-// name, and returns the keys a listing afterwards still finds. A delete
-// that fails is logged and the others still run; that object then
-// shows up in the listing afterwards.
-func (v *Vaultik) deleteEverythingUnder(prefix string) ([]string, error) {
-	keys, err := v.Storage.List(v.ctx, prefix)
-	if err != nil {
-		return nil, fmt.Errorf("listing %s: %w", prefix, err)
-	}
-
-	for _, key := range keys {
-		err = v.Storage.Delete(v.ctx, key)
-		if err != nil {
-			log.Error("Failed to delete object", "key", key, "error", err)
-		}
-	}
-
-	// The file and rclone listings skip `.partial` objects, so the
-	// deletes above never reach them.
-	err = v.Storage.DeletePartialUploads(v.ctx, prefix)
-	if err != nil {
-		return nil, fmt.Errorf("deleting partial uploads: %w", err)
-	}
-
-	left, err := v.Storage.List(v.ctx, prefix)
-	if err != nil {
-		return nil, fmt.Errorf("listing %s: %w", prefix, err)
-	}
-
-	return left, nil
-}
-
 // PruneBlobsResult contains the result of a blob prune operation
 //
 //nolint:tagliatelle // snake_case is the established JSON output format
@@ -204,7 +131,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 			return v.outputPruneBlobsJSON(result)
 		}
 
-		v.UI.Infof("No unreferenced blobs to remove.")
+		v.printlnStdout("No unreferenced blobs to remove.")
 
 		return nil
 	}
@@ -212,17 +139,12 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	log.Info("Found unreferenced blobs",
 		"count", len(unreferencedBlobs), "total_size", ubytes(totalSize))
 
-	if opts.Force && !opts.JSON {
+	if !opts.JSON {
-		v.UI.Infof("Found %d unreferenced blob(s) totaling %s.",
+		v.stdoutf("Found %d unreferenced blob(s) totaling %s\n",
 			len(unreferencedBlobs), ubytes(totalSize))
 	}
 
 	if !opts.Force && !opts.JSON {
-		// The count, the prompt and its answer bypass the UI: they are an
-		// interactive exchange the operator must see, so --quiet does not
-		// silence them.
-		v.stdoutf("Found %d unreferenced blob(s) totaling %s.\n",
-			len(unreferencedBlobs), ubytes(totalSize))
 		v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
 
 		var confirm string
@@ -244,17 +166,14 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
 
 	if opts.JSON {
-		err = v.outputPruneBlobsJSON(result)
+		return v.outputPruneBlobsJSON(result)
-		if err != nil {
-			return err
-		}
-	} else {
-		v.UI.Infof("Deleted %d blob(s) totaling %s.",
-			result.BlobsDeleted, ubytes(result.BytesFreed))
 	}
 
+	v.stdoutf("\nDeleted %d blob(s) totaling %s\n",
+		result.BlobsDeleted, ubytes(result.BytesFreed))
+
 	if result.BlobsFailed > 0 {
-		return fmt.Errorf("%d %w", result.BlobsFailed, errBlobDeleteFailed)
+		v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed)
 	}
 
 	return nil

internal/vaultik/prune_count_test.go

@@ -1,17 +1,13 @@
 package vaultik //nolint:testpackage // exercises unexported count helpers
 
 import (
-	"bytes"
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/snapshot"
-	"sneak.berlin/go/vaultik/internal/ui"
 )
 
 // TestTableCountForReportSurfacesReadFailure is the regression guard for
@@ -80,59 +76,3 @@ func TestCountDiffUnknownWhenEitherSideUnknown(t *testing.T) {
 	assert.Nil(t, countDiff(&before, nil), "unknown after yields unknown delta")
 	assert.Nil(t, countDiff(nil, nil))
 }
-
-// TestPruneDatabaseSummaryLabelsTotals removes the only incomplete
-// snapshot from an index that also holds a complete one. The figure in
-// brackets is every snapshot left in the local index, so the line must
-// say so rather than read as incomplete snapshots that remain.
-func TestPruneDatabaseSummaryLabelsTotals(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-
-	db, err := database.New(ctx, ":memory:")
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	repos := database.NewRepositories(db)
-
-	startedAt := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-	completedAt := startedAt.Add(time.Minute)
-
-	require.NoError(t, repos.Snapshots.Create(ctx, nil, &database.Snapshot{
-		ID:          "host_complete_2026-01-02T03:04:05Z",
-		Hostname:    "host",
-		StartedAt:   startedAt,
-		CompletedAt: &completedAt,
-	}))
-	require.NoError(t, repos.Snapshots.Create(ctx, nil, &database.Snapshot{
-		ID:        "host_incomplete_2026-01-02T03:04:05Z",
-		Hostname:  "host",
-		StartedAt: startedAt,
-	}))
-
-	var out bytes.Buffer
-
-	v := &Vaultik{
-		DB:           db,
-		Repositories: repos,
-		SnapshotManager: snapshot.NewSnapshotManager(
-			snapshot.SnapshotManagerParams{Repos: repos}),
-		UI: ui.NewWithColor(&out, false),
-	}
-	v.SetContext(ctx)
-
-	result, err := v.PruneDatabase()
-	require.NoError(t, err)
-	require.Equal(t, int64(1), result.SnapshotsDeleted)
-
-	assert.Contains(t, out.String(), "Incomplete snapshots: 1 removed "+
-		"(total snapshots left in the local index: 1).")
-	assert.Contains(t, out.String(), "Orphaned files: 0 removed "+
-		"(total files left in the local index: 0).")
-	assert.Contains(t, out.String(), "Orphaned chunks: 0 removed "+
-		"(total chunks left in the local index: 0).")
-	assert.Contains(t, out.String(), "Orphaned blobs: 0 removed "+
-		"(total blobs left in the local index: 0).")
-}

internal/vaultik/prune_delete_failure_test.go

@@ -1,98 +0,0 @@
-package vaultik_test
-
-import (
-	"context"
-	"encoding/json"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage/faultstore"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// refuseBlobDeletes is an OnDelete hook that fails every blob delete.
-func refuseBlobDeletes(key string) bool {
-	return strings.HasPrefix(key, "blobs/")
-}
-
-// TestPruneBlobs_FailedDeleteReturnsError checks that prune returns an
-// error, so the command exits non-zero, when an unreferenced blob cannot
-// be deleted.
-func TestPruneBlobs_FailedDeleteReturnsError(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newListEnv(t)
-
-	// The store holds no manifest, so nothing references this blob.
-	addBlob(t, env.store.testStorer, testBlobHashA)
-
-	store := faultstore.New(env.store)
-	store.OnDelete = refuseBlobDeletes
-	env.v.Storage = store
-
-	err := env.v.PruneBlobs(&vaultik.PruneOptions{Force: true})
-
-	require.ErrorContains(t, err, "1 blob(s) could not be deleted")
-}
-
-// TestPruneBlobs_JSONFailedDeleteReturnsError checks that prune --json
-// prints its document and then returns an error when an unreferenced
-// blob cannot be deleted.
-func TestPruneBlobs_JSONFailedDeleteReturnsError(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newListEnv(t)
-	addBlob(t, env.store.testStorer, testBlobHashA)
-
-	store := faultstore.New(env.store)
-	store.OnDelete = refuseBlobDeletes
-	env.v.Storage = store
-
-	err := env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true})
-
-	require.Error(t, err, "prune must fail when a blob cannot be deleted")
-
-	var result vaultik.PruneBlobsResult
-
-	require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result),
-		"stdout must hold the JSON document, got:\n%s", env.stdout.String())
-	assert.Equal(t, 1, result.BlobsFailed)
-	assert.Equal(t, 0, result.BlobsDeleted)
-}
-
-// TestCreateSnapshotPrune_FailedBlobDeleteReturnsError checks that the
-// prune step of snapshot create --prune fails the run when an
-// unreferenced blob cannot be deleted.
-func TestCreateSnapshotPrune_FailedBlobDeleteReturnsError(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, _, _ := backUpToFileDestination(ctx, t, storeDir)
-
-	// No snapshot references this blob, so the prune step deletes it.
-	orphan := filepath.Join(storeDir, "blobs", testBlobHashA[:2],
-		testBlobHashA[2:4], testBlobHashA)
-	require.NoError(t, os.MkdirAll(filepath.Dir(orphan), 0o750))
-	require.NoError(t, os.WriteFile(orphan, []byte("orphan"), 0o600))
-
-	store := faultstore.New(v.Storage)
-	store.OnDelete = refuseBlobDeletes
-	v.Storage = store
-
-	err := v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-		Cron:  true,
-		Prune: true,
-	})
-
-	require.ErrorContains(t, err, "1 blob(s) could not be deleted")
-	assert.FileExists(t, orphan)
-}

internal/vaultik/purge_local_remote_consistency_test.go

@@ -14,7 +14,6 @@ import (
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/types"
-	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
 
@@ -59,16 +58,13 @@ func setupConsistencyTest(
 		require.NoError(t, err)
 	}
 
-	stdout := &bytes.Buffer{}
-
 	v := &vaultik.Vaultik{
 		Storage:      mockStorage,
 		Repositories: repos,
 		DB:           db,
-		Stdout:       stdout,
+		Stdout:       &bytes.Buffer{},
 		Stderr:       &bytes.Buffer{},
 		Stdin:        &bytes.Buffer{},
-		UI:           ui.NewWithColor(stdout, false),
 	}
 	v.SetContext(ctx)
 

internal/vaultik/purge_per_name_test.go

@@ -14,7 +14,6 @@ import (
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/types"
-	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
 
@@ -87,7 +86,6 @@ func setupPurgeTest(
 		Stdout:       stdout,
 		Stderr:       stderr,
 		Stdin:        stdin,
-		UI:           ui.NewWithColor(stdout, false),
 	}
 	v.SetContext(ctx)
 
@@ -281,8 +279,7 @@ func TestPurgeOlderThan_TooLargeDeletesNothing(t *testing.T) {
 		OlderThan: "1000y",
 		Force:     true,
 	})
-	require.EqualError(t, err,
+	require.Error(t, err)
-		`invalid duration: "1000y" is longer than about 292 years`)
 
 	err = v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
 		OlderThan: "292y",

internal/vaultik/remove_quiet_test.go

@@ -1,148 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/ui"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// TestPurgeSnapshots_QuietSuppressesReport checks that `snapshot purge
-// --keep-latest --force` under --quiet deletes the older snapshot and
-// prints nothing.
-func TestPurgeSnapshots_QuietSuppressesReport(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	v := setupPurgeTest(t, snapHostname, []string{snapHomeT0, snapHomeT1})
-
-	var out bytes.Buffer
-
-	v.Stdout = &out
-	v.UI = ui.NewWithColor(&out, false)
-	v.UI.SetQuiet(true)
-
-	require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
-		KeepLatest: true,
-		Force:      true,
-	}))
-
-	assert.Equal(t, []string{snapHomeT1}, listRemainingSnapshots(t, v))
-	assert.Empty(t, out.String())
-}
-
-// TestPurgeSnapshots_QuietPromptListsSnapshots checks that `snapshot
-// purge --keep-latest` without --force still lists the snapshots its
-// prompt asks to delete under --quiet.
-func TestPurgeSnapshots_QuietPromptListsSnapshots(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	v := setupPurgeTest(t, snapHostname, []string{snapHomeT0, snapHomeT1})
-
-	var out bytes.Buffer
-
-	v.Stdout = &out
-	v.UI = ui.NewWithColor(&out, false)
-	v.UI.SetQuiet(true)
-
-	// Stdin is empty, so the prompt reads no answer and cancels.
-	require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
-		KeepLatest: true,
-	}))
-
-	assert.Contains(t, out.String(), snapHomeT0)
-	assert.Contains(t, out.String(), "Delete 1 snapshot(s)? [y/N]")
-	assert.Len(t, listRemainingSnapshots(t, v), 2)
-}
-
-// TestPruneBlobs_QuietPromptShowsBlobTotal checks that `prune` without
-// --force still states the count and total size of the blobs its prompt
-// asks to delete under --quiet.
-func TestPruneBlobs_QuietPromptShowsBlobTotal(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newListEnv(t)
-	env.v.UI.SetQuiet(true)
-
-	// The store holds no manifest, so nothing references this blob.
-	addBlob(t, env.store.testStorer, testBlobHashA)
-
-	// Stdin is empty, so the prompt reads no answer and cancels.
-	require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
-
-	assert.Contains(t, env.stdout.String(),
-		"Found 1 unreferenced blob(s) totaling")
-	assert.Contains(t, env.stdout.String(),
-		"Delete 1 unreferenced blob(s)? [y/N]")
-}
-
-// TestRemoveSnapshot_QuietSuppressesReport checks that `snapshot remove`
-// under --quiet prints nothing with --dry-run or with --force, and that
-// the --json document still prints.
-func TestRemoveSnapshot_QuietSuppressesReport(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	newQuietRemover := func() (*vaultik.TestVaultik, *testStorer) {
-		store := newTestStorer()
-		addManifest(t, store, "snapshot-001", nil)
-
-		tv := vaultik.NewForTesting(store)
-		tv.Repositories = newLocalIndex(t, "snapshot-001")
-		tv.UI.SetQuiet(true)
-
-		return tv, store
-	}
-
-	tv, store := newQuietRemover()
-
-	_, err := tv.RemoveSnapshot("snapshot-001",
-		&vaultik.RemoveOptions{DryRun: true})
-	require.NoError(t, err)
-	assert.Empty(t, tv.Stdout.String(),
-		"--dry-run must print nothing under --quiet")
-
-	_, err = tv.RemoveSnapshot("snapshot-001",
-		&vaultik.RemoveOptions{Force: true})
-	require.NoError(t, err)
-	assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
-	assert.Empty(t, tv.Stdout.String(),
-		"--force must print nothing under --quiet")
-
-	tv, _ = newQuietRemover()
-
-	_, err = tv.RemoveSnapshot("snapshot-001",
-		&vaultik.RemoveOptions{JSON: true})
-	require.NoError(t, err)
-
-	var result vaultik.RemoveResult
-
-	require.NoError(t, json.Unmarshal(tv.Stdout.Bytes(), &result),
-		"the --json document must still print under --quiet")
-	assert.True(t, result.RemoteRemoved)
-}
-
-// TestNukeRemote_QuietSuppressesReport checks that `remote nuke --force`
-// under --quiet prints nothing while it removes a snapshot's metadata
-// and its blobs.
-func TestNukeRemote_QuietSuppressesReport(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, _, out := backUpToFileDestination(ctx, t, storeDir)
-	v.UI.SetQuiet(true)
-
-	require.NoError(t, v.NukeRemote(true))
-	assert.Empty(t, out.String())
-}

internal/vaultik/remove_snapshot_test.go

@@ -17,7 +17,6 @@ import (
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/storage/faultstore"
 	"sneak.berlin/go/vaultik/internal/types"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
@@ -351,7 +350,7 @@ func TestRemoveSnapshot_DryRun(t *testing.T) {
 	assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
 	assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
 
-	assert.Contains(t, tv.Stdout.String(), "Dry run: no changes made.")
+	assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
 }
 
 // TestRemoveSnapshot_UnknownIDIsError checks that an ID naming no
@@ -432,40 +431,6 @@ func TestRemoveSnapshot_NotInLocalIndex(t *testing.T) {
 	assert.Contains(t, out, "Removed snapshot metadata from remote storage")
 }
 
-// TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a
-// snapshot only the destination store holds fails when its metadata
-// cannot be deleted there, under --force and under --json: nothing was
-// removed, so the command must not succeed.
-func TestRemoveSnapshot_NotInLocalIndexDeleteFails(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	for name, opts := range map[string]vaultik.RemoveOptions{
-		"force": {Force: true},
-		"json":  {JSON: true},
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			store := newTestStorer()
-			addManifest(t, store, "snapshot-001", nil)
-
-			faulty := faultstore.New(store)
-			faulty.OnDelete = func(string) bool { return true }
-
-			tv := vaultik.NewForTesting(faulty)
-			tv.Repositories = newLocalIndex(t)
-
-			result, err := tv.RemoveSnapshot("snapshot-001", &opts)
-
-			require.ErrorIs(t, err, faultstore.ErrInjectedDelete)
-			assert.False(t, result.RemoteRemoved)
-			assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
-			assert.Empty(t, tv.Stdout.String())
-		})
-	}
-}
-
 func TestRemoveAllSnapshots_RequiresForce(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -511,8 +476,7 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) {
 	out := tv.Stdout.String()
 	assert.Contains(t, out, "Removed 2 snapshot(s)")
 	assert.Contains(t, out, "Removed snapshot metadata from remote storage")
-	// Remote nuke, the only caller, deletes the blobs next itself.
+	assert.Contains(t, out, "vaultik prune")
-	assert.NotContains(t, out, "vaultik prune")
 }
 
 func TestRemoveAllSnapshots_DryRun(t *testing.T) {
@@ -539,7 +503,7 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) {
 
 	assert.Equal(t, initialCount, store.keyCount())
 
-	assert.Contains(t, tv.Stdout.String(), "Dry run: no changes made.")
+	assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
 }
 
 func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) {

internal/vaultik/restore.go

@@ -27,7 +27,6 @@ import (
 // Sentinel errors for restore failures.
 var (
 	errFilesFailedRestore    = errors.New("file(s) failed to restore")
-	errPathsNotInSnapshot    = errors.New("requested path(s) not in the snapshot")
 	errFilesFailedVerify     = errors.New("files failed verification")
 	errDecryptionKeyRequired = errors.New(
 		"decryption key required for restore\n\n" +
@@ -104,12 +103,11 @@ type RestoreResult struct {
 	BlobsDownloaded int
 	BytesDownloaded int64
 	Duration        time.Duration
-	FilesFailed     int
-	FailedFiles     []string // Paths of files that failed to restore
 	// Verification results (only populated if Verify option is set)
-	FilesVerified     int
+	FilesVerified int
-	BytesVerified     int64
+	BytesVerified int64
-	VerifyFailedFiles []string // Paths of restored files that failed verification
+	FilesFailed   int
+	FailedFiles   []string // Paths of files that failed verification
 }
 
 // Restore restores files from a snapshot to the target directory
@@ -151,20 +149,16 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 	repos := database.NewRepositories(tempDB)
 
 	// Step 2: Get list of files to restore
-	files, unmatched, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
+	files, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
 	if err != nil {
 		return fmt.Errorf("getting files to restore: %w", err)
 	}
 
-	// Named before anything is restored, so the names are shown even
-	// when the restore stops on an error.
-	unmatchedErr := v.reportUnmatchedPaths(unmatched)
-
 	if len(files) == 0 {
 		log.Warn("No files found to restore")
 		v.UI.Warningf("No files found to restore.")
 
-		return unmatchedErr
+		return nil
 	}
 
 	log.Info("Found files to restore", "count", len(files))
@@ -190,20 +184,6 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 
 	result.Duration = time.Since(startTime)
 
-	return v.finishRestore(repos, files, unmatchedErr, opts, result)
-}
-
-// finishRestore prints the restore summary and the post-restore
-// warnings, runs optional verification, and converts any failed-file
-// count into an error. Otherwise it returns unmatchedErr, the error for
-// the requested paths that matched nothing (nil when there are none).
-func (v *Vaultik) finishRestore(
-	repos *database.Repositories,
-	files []*database.File,
-	unmatchedErr error,
-	opts *RestoreOptions,
-	result *RestoreResult,
-) error {
 	log.Info("Restore complete",
 		"files_restored", result.FilesRestored,
 		"bytes_restored", ubytes(result.BytesRestored),
@@ -218,6 +198,17 @@ func (v *Vaultik) finishRestore(
 		v.UI.Duration(result.Duration),
 	)
 
+	return v.finishRestore(repos, files, opts, result)
+}
+
+// finishRestore emits the post-restore warnings, runs optional
+// verification, and converts any failed-file count into an error.
+func (v *Vaultik) finishRestore(
+	repos *database.Repositories,
+	files []*database.File,
+	opts *RestoreOptions,
+	result *RestoreResult,
+) error {
 	if os.Geteuid() != 0 {
 		v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
 			"requires root. Re-run as root (e.g. with sudo) if you need " +
@@ -244,23 +235,7 @@ func (v *Vaultik) finishRestore(
 		return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
 	}
 
-	return unmatchedErr
+	return nil
-}
-
-// reportUnmatchedPaths warns, by name, about each requested path that
-// matched nothing in the snapshot, and returns an error counting them,
-// or nil when there are none.
-func (v *Vaultik) reportUnmatchedPaths(unmatched []string) error {
-	if len(unmatched) == 0 {
-		return nil
-	}
-
-	for _, path := range unmatched {
-		log.Warn("Requested path matches nothing in the snapshot", "path", path)
-		v.UI.Warningf("%s matches nothing in the snapshot.", v.UI.Path(path))
-	}
-
-	return fmt.Errorf("%d %w", len(unmatched), errPathsNotInSnapshot)
 }
 
 // restoreIdentities parses the configured age secret key once into every
@@ -323,33 +298,22 @@ func (v *Vaultik) restoreAllFiles(
 		_ = blobCache.Close()
 	}()
 
+	// Per-restore sweep state: every blob_size_limit/100 bytes written,
+	// scan the cache and delete any blob whose remaining file references
+	// are all already restored.
+	sweeper := newRestoreSweeper(v.ctx, repos, blobCache,
+		v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor)
+
 	blobByHash, blobIDToHash, err := v.buildBlobIndexes(repos)
 	if err != nil {
 		return nil, err
 	}
 
-	plan, unplanned, err := newRestorePlan(
+	plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
-		v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
 	if err != nil {
 		return nil, fmt.Errorf("building restore plan: %w", err)
 	}
 
-	// Per-restore sweep state: every blob_size_limit/100 bytes written,
-	// scan the cache and delete any blob that no pending file in the
-	// plan references.
-	sweeper := newRestoreSweeper(plan, blobCache,
-		v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor)
-
-	// Ranging over files rather than the map reports them in a stable order.
-	for _, file := range files {
-		if planErr, ok := unplanned[file.ID]; ok {
-			err = v.handleRestoreFileError(plan, opts, result, file, file.ID, planErr)
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
 	filesByID, totalBytesExpected := indexRestoreFiles(files)
 
 	v.UI.Beginf("Restoring %s files (%s) to %s.",
@@ -427,8 +391,10 @@ func (v *Vaultik) runRestoreLoop(
 			continue
 		}
 
-		// Drop the file from the plan's indexes so future picks ignore
+		// Record the file as restored so the sweeper can free blobs
-		// it and the sweeper can free blobs no pending file needs.
+		// once all referencing files are done, and drop it from the
+		// plan's indexes so future picks ignore it.
+		session.sweeper.fileRestored(fileID.String())
 		plan.finishFile(fileID)
 
 		processed++
@@ -681,34 +647,20 @@ func (v *Vaultik) handleRestoreVerification(
 	opts *RestoreOptions,
 	result *RestoreResult,
 ) error {
-	// A file that failed to restore was not written and is already
+	err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result)
-	// reported as such, so it is not verified.
-	failedToRestore := make(map[string]bool, len(result.FailedFiles))
-	for _, path := range result.FailedFiles {
-		failedToRestore[path] = true
-	}
-
-	restored := make([]*database.File, 0, len(files))
-	for _, file := range files {
-		if !failedToRestore[file.Path.String()] {
-			restored = append(restored, file)
-		}
-	}
-
-	err := v.verifyRestoredFiles(v.ctx, repos, restored, opts.TargetDir, result)
 	if err != nil {
 		return fmt.Errorf("verification failed: %w", err)
 	}
 
-	if len(result.VerifyFailedFiles) > 0 {
+	if result.FilesFailed > 0 {
 		v.UI.Errorf("Verification failed: %s files did not match expected checksums.",
-			v.UI.Count(len(result.VerifyFailedFiles)))
+			v.UI.Count(result.FilesFailed))
 
-		for _, path := range result.VerifyFailedFiles {
+		for _, path := range result.FailedFiles {
 			v.UI.Detailf("%s", v.UI.Path(path))
 		}
 
-		return fmt.Errorf("%d %w", len(result.VerifyFailedFiles), errFilesFailedVerify)
+		return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedVerify)
 	}
 
 	v.UI.Completef("Verified %s files (%s).",
@@ -739,8 +691,6 @@ func (v *Vaultik) downloadSnapshotDB(
 		return nil, "", fmt.Errorf("downloading %s: %w", dbKey, err)
 	}
 
-	reader = newCloseOnCancelReader(v.ctx, reader)
-
 	defer func() { _ = reader.Close() }()
 
 	// Decrypt and decompress straight from the storage stream, then stream
@@ -862,34 +812,27 @@ func (v *Vaultik) materializeSnapshotDB(
 	return db, tempDir, nil
 }
 
-// getFilesToRestore returns the list of files to restore based on path
+// getFilesToRestore returns the list of files to restore based on path filters
-// filters, and the filters, as given, that matched no file.
 func (v *Vaultik) getFilesToRestore(
 	ctx context.Context, repos *database.Repositories, pathFilters []string,
-) ([]*database.File, []string, error) {
+) ([]*database.File, error) {
 	// If no filters, get all files
 	if len(pathFilters) == 0 {
-		files, err := repos.Files.ListAll(ctx)
+		return repos.Files.ListAll(ctx)
-
-		return files, nil, err
 	}
 
 	// Get files matching the path filters
-	var (
+	var result []*database.File
-		result    []*database.File
-		unmatched []string
-	)
 
 	seen := make(map[string]bool)
 
 	for _, filter := range pathFilters {
-		files, err := repos.Files.ListUnderPath(ctx, filepath.Clean(filter))
+		// Normalize the filter path
-		if err != nil {
+		filter = filepath.Clean(filter)
-			return nil, nil, fmt.Errorf("listing files under %s: %w", filter, err)
-		}
 
-		if len(files) == 0 {
+		files, err := repos.Files.ListUnderPath(ctx, filter)
-			unmatched = append(unmatched, filter)
+		if err != nil {
+			return nil, fmt.Errorf("listing files under %s: %w", filter, err)
 		}
 
 		for _, file := range files {
@@ -900,7 +843,7 @@ func (v *Vaultik) getFilesToRestore(
 		}
 	}
 
-	return result, unmatched, nil
+	return result, nil
 }
 
 // buildChunkToBlobMap creates a mapping from chunk hash to blob information
@@ -1477,8 +1420,8 @@ func (v *Vaultik) verifyRestoredFiles(
 		if err != nil {
 			log.Error("File verification failed", "path", file.Path, "error", err)
 
-			result.VerifyFailedFiles = append(
+			result.FilesFailed++
-				result.VerifyFailedFiles, file.Path.String())
+			result.FailedFiles = append(result.FailedFiles, file.Path.String())
 		}
 
 		bytesProcessed += file.Size
@@ -1494,7 +1437,7 @@ func (v *Vaultik) verifyRestoredFiles(
 	log.Info("Verification complete",
 		"files_verified", result.FilesVerified,
 		"bytes_verified", ubytes(result.BytesVerified),
-		"files_failed", len(result.VerifyFailedFiles),
+		"files_failed", result.FilesFailed,
 	)
 
 	return nil

internal/vaultik/restore_interrupt_test.go

@@ -74,8 +74,7 @@ func TestRestoreCleansTempDirOnInterrupt(t *testing.T) {
 	buildLocalityFixture(t, fs, dataDir)
 
 	cfg, storer, snapshotID := setupLocalityBackup(
-		context.Background(), t, fs, dataDir, storeDir, dbPath,
+		context.Background(), t, fs, dataDir, storeDir, dbPath)
-		localitySourceBytes)
 
 	// Point the "" temp paths (the blob cache directory and the
 	// snapshot-database directory) at a private directory so the test can
@@ -208,37 +207,30 @@ func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
 		"the cancel was reported as a failed file")
 }
 
-// stallingStorer wraps a Storer so that each download of a key chosen by
+// stallingBlobStorer wraps a Storer so that every blob download hands
-// stalls hands over the first half of the object and then stalls,
+// over the first half of the blob and then stalls, ignoring the context,
-// ignoring the context, as a file:// read on a hung USB stick or network
+// as a file:// blob on a hung USB stick or network share does. Only
-// share does. Only closing the returned reader ends the stall.
+// closing the returned reader ends the stall.
-type stallingStorer struct {
+type stallingBlobStorer struct {
 	storage.Storer
 
-	stalls  func(key string) bool
 	once    sync.Once
 	stalled chan struct{}
 }
 
-func newStallingStorer(
+func newStallingBlobStorer(inner storage.Storer) *stallingBlobStorer {
-	inner storage.Storer, stalls func(key string) bool,
+	return &stallingBlobStorer{Storer: inner, stalled: make(chan struct{})}
-) *stallingStorer {
-	return &stallingStorer{
-		Storer:  inner,
-		stalls:  stalls,
-		stalled: make(chan struct{}),
-	}
 }
 
-func (s *stallingStorer) Get(
+func (s *stallingBlobStorer) Get(
 	ctx context.Context, key string,
 ) (io.ReadCloser, error) {
 	rc, err := s.Storer.Get(ctx, key)
-	if err != nil || !s.stalls(key) {
+	if err != nil || !strings.HasPrefix(key, "blobs/") {
 		return rc, err
 	}
 
-	object, err := io.ReadAll(rc)
+	blob, err := io.ReadAll(rc)
 	_ = rc.Close()
 
 	if err != nil {
@@ -250,7 +242,7 @@ func (s *stallingStorer) Get(
 	pr, pw := io.Pipe()
 
 	go func() {
-		_, _ = pw.Write(object[:len(object)/2])
+		_, _ = pw.Write(blob[:len(blob)/2])
 
 		s.once.Do(func() { close(s.stalled) })
 	}()
@@ -258,26 +250,17 @@ func (s *stallingStorer) Get(
 	return pr, nil
 }
 
-// TestInterruptDuringStalledReadCleansTempDir stops a restore and a deep
+// TestInterruptDuringStalledBlobReadCleansTempDir stops a restore and a
-// verify while a read of a blob or of the snapshot database is stalled
+// deep verify while a blob read is stalled on a source that ignores the
-// on a source that ignores the context. Stopping must end that read, so
+// context. Stopping must end that read, so the operation unwinds and
-// the operation unwinds and removes its decrypted scratch files well
+// removes its decrypted scratch files well within the shutdown timeout
-// within the shutdown timeout (issues #325 and #356).
+// (issue #325).
 //
 // Not parallel: it points TMPDIR at a private directory (via t.Setenv)
 // so it can assert on exactly the scratch files the operation created.
-func TestInterruptDuringStalledReadCleansTempDir(t *testing.T) {
+func TestInterruptDuringStalledBlobReadCleansTempDir(t *testing.T) {
 	log.Initialize(log.Config{})
 
-	stalledReads := map[string]func(key string) bool{
-		"blob": func(key string) bool {
-			return strings.HasPrefix(key, "blobs/")
-		},
-		"snapshot database": func(key string) bool {
-			return strings.HasSuffix(key, "/db.zst.age")
-		},
-	}
-
 	operations := map[string]func(v *Vaultik, snapshotID, targetDir string) error{
 		"restore": func(v *Vaultik, snapshotID, targetDir string) error {
 			return v.Restore(&RestoreOptions{
@@ -290,58 +273,52 @@ func TestInterruptDuringStalledReadCleansTempDir(t *testing.T) {
 		},
 	}
 
-	for readName, stalls := range stalledReads {
+	for name, op := range operations {
-		for opName, op := range operations {
+		t.Run(name, func(t *testing.T) {
-			t.Run(readName+" read during "+opName, func(t *testing.T) {
+			fs := afero.NewOsFs()
-				fs := afero.NewOsFs()
+			root := t.TempDir()
-				root := t.TempDir()
 
-				// Random content, so the blob stays large enough to stall
+			// Random content, so the blob stays large enough to stall
-				// part-way through its compressed, encrypted body.
+			// part-way through its compressed, encrypted body.
-				cfg, storer, snapshotID, _ := backupOneFile(context.Background(),
+			cfg, storer, snapshotID, _ := backupOneFile(context.Background(),
-					t, fs, root, "a.bin", randomBytes(t, 256*1024), 0o644)
+				t, fs, root, "a.bin", randomBytes(t, 256*1024), 0o644)
 
-				scratch := filepath.Join(root, "scratch")
+			scratch := filepath.Join(root, "scratch")
-				require.NoError(t, fs.MkdirAll(scratch, 0o755))
+			require.NoError(t, fs.MkdirAll(scratch, 0o755))
-				t.Setenv("TMPDIR", scratch)
+			t.Setenv("TMPDIR", scratch)
 
-				stalling := newStallingStorer(storer, stalls)
+			stalling := newStallingBlobStorer(storer)
 
-				ctx, cancel := context.WithCancel(context.Background())
+			ctx, cancel := context.WithCancel(context.Background())
-				defer cancel()
+			defer cancel()
 
-				v := newRestoreVaultik(ctx, cfg, stalling, fs)
+			v := newRestoreVaultik(ctx, cfg, stalling, fs)
-				v.cancel = cancel
+			v.cancel = cancel
 
-				var opErr error
+			var opErr error
 
-				stop := v.StartOperation(func() {
+			stop := v.StartOperation(func() {
-					opErr = op(v, snapshotID, filepath.Join(root, "restored"))
+				opErr = op(v, snapshotID, filepath.Join(root, "restored"))
-				})
-
-				select {
-				case <-stalling.stalled:
-				case <-time.After(30 * time.Second):
-					t.Fatal("the operation never reached the stalled read")
-				}
-
-				// The snapshot database's temp directory is created just
-				// after its read starts, so it can appear after the stall.
-				require.Eventually(t, func() bool {
-					return len(scratchEntries(t, scratch)) > 0
-				}, 10*time.Second, 10*time.Millisecond,
-					"expected decrypted scratch files to exist mid-read")
-
-				stopCtx, stopCancel := context.WithTimeout(
-					context.Background(), 10*time.Second)
-				defer stopCancel()
-
-				require.True(t, stop(stopCtx),
-					"stop timed out; the stalled read was not interrupted")
-				require.ErrorIs(t, opErr, context.Canceled)
-				require.Empty(t, scratchEntries(t, scratch),
-					"decrypted scratch files remained after the interrupt")
 			})
-		}
+
+			select {
+			case <-stalling.stalled:
+			case <-time.After(30 * time.Second):
+				t.Fatal("the operation never reached a blob read")
+			}
+
+			require.NotEmpty(t, scratchEntries(t, scratch),
+				"expected decrypted scratch files to exist mid-read")
+
+			stopCtx, stopCancel := context.WithTimeout(
+				context.Background(), 10*time.Second)
+			defer stopCancel()
+
+			require.True(t, stop(stopCtx),
+				"stop timed out; the stalled blob read was not interrupted")
+			require.ErrorIs(t, opErr, context.Canceled)
+			require.Empty(t, scratchEntries(t, scratch),
+				"decrypted scratch files remained after the interrupt")
+		})
 	}
 }

internal/vaultik/restore_locality_test.go

@@ -63,10 +63,6 @@ type localityCopy struct {
 	data []byte
 }
 
-// localitySourceBytes is the size of each source file of the locality
-// fixture.
-const localitySourceBytes = 1024 * 1024
-
 // buildLocalityFixture writes the adversarial source layout described
 // in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the
 // backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
@@ -77,6 +73,7 @@ func buildLocalityFixture(
 	t.Helper()
 
 	const (
+		srcBytes   = 1024 * 1024
 		srcCount   = 15
 		blobsCount = 3
 		perBlob    = srcCount / blobsCount
@@ -86,7 +83,7 @@ func buildLocalityFixture(
 	for i := range srcCount {
 		s := &localitySource{
 			path: fmt.Sprintf("src-%03d.bin", i+1),
-			data: randomBytes(t, localitySourceBytes),
+			data: randomBytes(t, srcBytes),
 		}
 		sources[i] = s
 		require.NoError(t, afero.WriteFile(
@@ -118,20 +115,19 @@ func buildLocalityFixture(
 
 // setupLocalityBackup runs the backup half of the locality test: a
 // snapshot of dataDir into a file storer with sizes tuned so 15 one-chunk
-// files of srcBytes pack into 3 blobs. The index database is closed
+// files pack into 3 blobs. The index database is closed before returning
-// before returning so the restore half runs from remote bytes only.
+// so the restore half runs from remote bytes only.
 func setupLocalityBackup(
 	ctx context.Context, t *testing.T, fs afero.Fs,
-	dataDir, storeDir, dbPath string, srcBytes int64,
+	dataDir, storeDir, dbPath string,
 ) (*config.Config, *storage.FileStorer, string) {
 	t.Helper()
 
-	// An average chunk size of four times srcBytes makes the minimum
+	// chunkSize avg = 4 MiB makes minSize = 1 MiB, so a 1 MiB file
-	// chunk srcBytes, so each source file becomes one chunk. A blob limit
+	// becomes one chunk. maxBlobSize = 5 MiB packs exactly 5 chunks
-	// of five times srcBytes packs exactly 5 chunks per blob, yielding 3
+	// per blob, yielding 3 blobs from 15 source files.
-	// blobs from 15 source files.
+	chunkSize := int64(4 * 1024 * 1024)
-	chunkSize := 4 * srcBytes
+	maxBlobSize := int64(5 * 1024 * 1024)
-	maxBlobSize := 5 * srcBytes
 
 	storer, err := storage.NewFileStorer(storeDir)
 	require.NoError(t, err)
@@ -212,7 +208,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) {
 	ctx := context.Background()
 
 	cfg, storer, snapshotID := setupLocalityBackup(
-		ctx, t, fs, dataDir, storeDir, dbPath, localitySourceBytes)
+		ctx, t, fs, dataDir, storeDir, dbPath)
 
 	// Wrap the storer so we can count downloads per blob key.
 	counter := newCountingStorer(storer)

internal/vaultik/restore_malformed_db_test.go

@@ -61,8 +61,8 @@ func craftedFile(
 }
 
 // TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash
-// has no blob_chunks row is left out of the restore plan with an error,
+// has no blob_chunks row fails restore planning and the chunk write with
-// and fails the chunk write with an error.
+// an error.
 func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
 	t.Parallel()
 
@@ -77,11 +77,9 @@ func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
 	chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
 	require.NoError(t, err)
 
-	plan, unplanned, err := newRestorePlan(ctx, repos, []*database.File{file},
+	_, err = newRestorePlan(ctx, repos, []*database.File{file},
 		chunkToBlobMap, map[string]string{})
-	require.NoError(t, err)
+	require.ErrorIs(t, err, errPlanChunkMissing)
-	require.ErrorIs(t, unplanned[file.ID], errPlanChunkMissing)
-	require.False(t, plan.hasPending())
 
 	fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
 	require.NoError(t, err)

internal/vaultik/restore_plan.go

@@ -31,8 +31,7 @@ var (
 //     with no further downloads.
 //   - blobFiles: for each blob, the set of pending files referencing
 //     it. Used to short-circuit "when this blob lands, which files
-//     become ready" without a global scan, and by the sweeper to tell
+//     become ready" without a global scan.
-//     whether a cached blob is still needed.
 type restorePlan struct {
 	fileBlobs map[types.FileID]map[string]struct{}
 	blobFiles map[string]map[types.FileID]struct{}
@@ -42,25 +41,20 @@ type restorePlan struct {
 
 // newRestorePlan builds the file→blob index for the given files. Files
 // whose chunks reference no blobs (symlinks, directories) start in the
-// ready queue immediately. A file with a chunk that no blob holds cannot
+// ready queue immediately.
-// be restored: it is left out of the plan and returned, by ID, with the
-// reason.
 func newRestorePlan(
 	ctx context.Context,
 	repos *database.Repositories,
 	files []*database.File,
 	chunkToBlobMap map[string]*database.BlobChunk,
 	blobIDToHash map[string]string,
-) (*restorePlan, map[types.FileID]error, error) {
+) (*restorePlan, error) {
 	p := &restorePlan{
 		fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)),
 		blobFiles: make(map[string]map[types.FileID]struct{}),
 		ready:     make([]types.FileID, 0, len(files)),
 		cached:    make(map[string]struct{}),
 	}
-	unplanned := make(map[types.FileID]error)
-
-nextFile:
 	for _, f := range files {
 		if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 {
 			// No chunks to fetch — restore can run immediately.
@@ -72,7 +66,7 @@ nextFile:
 
 		fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID)
 		if err != nil {
-			return nil, nil, fmt.Errorf("planning %s: %w", f.Path, err)
+			return nil, fmt.Errorf("planning %s: %w", f.Path, err)
 		}
 
 		blobs := make(map[string]struct{})
@@ -80,18 +74,14 @@ nextFile:
 		for _, fc := range fileChunks {
 			bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
 			if !ok {
-				unplanned[f.ID] = fmt.Errorf("%w: %s",
+				return nil, fmt.Errorf("planning %s: %w: %s",
-					errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
+					f.Path, errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
-
-				continue nextFile
 			}
 
 			hash, ok := blobIDToHash[bc.BlobID.String()]
 			if !ok {
-				unplanned[f.ID] = fmt.Errorf("%w: %s",
+				return nil, fmt.Errorf("planning %s: %w: %s",
-					errPlanBlobIDMissing, bc.BlobID)
+					f.Path, errPlanBlobIDMissing, bc.BlobID)
-
-				continue nextFile
 			}
 
 			blobs[hash] = struct{}{}
@@ -113,7 +103,7 @@ nextFile:
 		}
 	}
 
-	return p, unplanned, nil
+	return p, nil
 }
 
 // markBlobCached records that the named blob is now resident in the
@@ -238,12 +228,6 @@ func (p *restorePlan) filesReferencingBlob(blobHash string) []types.FileID {
 	return out
 }
 
-// pendingFileUsesBlob reports whether any pending file references the
-// named blob, whether or not the blob is cached.
-func (p *restorePlan) pendingFileUsesBlob(blobHash string) bool {
-	return len(p.blobFiles[blobHash]) > 0
-}
-
 // hasPending reports whether any unfinished files remain.
 func (p *restorePlan) hasPending() bool {
 	return len(p.fileBlobs) > 0

internal/vaultik/restore_plan_test.go

@@ -63,6 +63,15 @@ func TestRunRestoreLoopFailsOnAbandonedFiles(t *testing.T) {
 
 	t.Cleanup(func() { _ = cache.Close() })
 
+	v := &Vaultik{ctx: ctx}
+	session := &restoreSession{
+		v:       v,
+		ctx:     ctx,
+		repos:   repos,
+		sweeper: newRestoreSweeper(ctx, repos, cache, 1),
+		result:  &RestoreResult{},
+	}
+
 	// A file that is still pending but whose uncached-blob set is empty
 	// and which was never queued as ready: the loop can neither restore
 	// nor download it. This is the abandonment the guard must catch.
@@ -74,15 +83,6 @@ func TestRunRestoreLoopFailsOnAbandonedFiles(t *testing.T) {
 		cached:    map[string]struct{}{},
 	}
 
-	v := &Vaultik{ctx: ctx}
-	session := &restoreSession{
-		v:       v,
-		ctx:     ctx,
-		repos:   repos,
-		sweeper: newRestoreSweeper(plan, cache, 1),
-		result:  &RestoreResult{},
-	}
-
 	err = v.runRestoreLoop(session, plan, map[types.FileID]*database.File{}, 0)
 	require.ErrorIs(t, err, errRestoreIncomplete)
 }

internal/vaultik/restore_skip_errors_test.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/spf13/afero"
@@ -30,8 +28,7 @@ const (
 )
 
 // missingBlobBackup is a snapshot of single-chunk files spread over two
-// blobs, with one of those blobs lost: deleted from the store, or missing
+// blobs, with one of those blobs deleted from the store.
-// from the snapshot metadata.
 type missingBlobBackup struct {
 	fs         afero.Fs
 	cfg        *config.Config
@@ -40,39 +37,20 @@ type missingBlobBackup struct {
 	restoreDir string
 	// files holds the original content by source path.
 	files map[string][]byte
-	// lost holds the source paths whose chunk is in the lost blob.
+	// lost holds the source paths whose chunk is in the deleted blob.
 	lost map[string]bool
 }
 
 // TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
-// after one blob of a two-blob snapshot was deleted from the store.
+// after one blob of a two-blob snapshot was deleted. Every file stored in
+// that blob must be reported as failed and left absent, every other file
+// must be restored intact, and Restore must still return an error.
 func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
 
 	ctx := context.Background()
-	assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
+	backup := backupThenDeleteOneBlob(ctx, t)
-}
-
-// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
-// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
-// earlier backup defects left behind.
-func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
-}
-
-// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
-// stored in the lost blob must be reported as failed and left absent,
-// every other file must be restored intact, and Restore must still return
-// an error.
-func assertRestoreSkipsLostFiles(
-	ctx context.Context, t *testing.T, backup *missingBlobBackup,
-) {
-	t.Helper()
 
 	var out bytes.Buffer
 
@@ -94,145 +72,20 @@ func assertRestoreSkipsLostFiles(
 
 		if backup.lost[path] {
 			assert.Containsf(t, out.String(), path,
-				"%s needs the lost blob and must be reported", path)
+				"%s needs the deleted blob and must be reported", path)
 			assert.NoFileExists(t, restored)
 
 			continue
 		}
 
 		got, err := afero.ReadFile(backup.fs, restored)
-		require.NoErrorf(t, err, "%s does not need the lost blob", path)
+		require.NoErrorf(t, err, "%s does not need the deleted blob", path)
 		assert.Equalf(t, content, got, "%s restored with wrong content", path)
 	}
 }
 
-// TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with
-// SkipErrors and Verify after one blob of a two-blob snapshot was
-// deleted. Verification must check only the files that were written, so
-// a file of the deleted blob is reported as failed to restore and not
-// also as failing verification, and Restore must still return an error.
-func TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	backup := backupThenDeleteOneBlob(ctx, t)
-
-	var out bytes.Buffer
-
-	v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
-	v.UI = ui.NewWithColor(&out, false)
-
-	err := v.Restore(&vaultik.RestoreOptions{
-		SnapshotID: backup.snapshotID,
-		TargetDir:  backup.restoreDir,
-		Verify:     true,
-		SkipErrors: true,
-	})
-
-	require.Error(t, err, "restore must fail when files were skipped")
-	assert.Contains(t, err.Error(),
-		fmt.Sprintf("%d file(s) failed to restore", len(backup.lost)))
-
-	written := len(backup.files) - len(backup.lost)
-
-	assert.Contains(t, out.String(), fmt.Sprintf("Verifying %d files", written))
-	assert.Contains(t, out.String(), fmt.Sprintf("Verified %d files", written))
-	assert.NotContains(t, out.String(), "Verification failed")
-}
-
-// TestRestoreSkipErrorsVerifyReportsFailuresApart restores with
-// SkipErrors and Verify after one blob of a two-blob snapshot was
-// deleted, while one file that does restore is written one byte too long.
-// The restore warning must list only the files of the deleted blob, the
-// verification summary only the long file, each path once, and Restore
-// must return an error.
-func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	backup := backupThenDeleteOneBlob(ctx, t)
-
-	var long string
-
-	for path := range backup.files {
-		if !backup.lost[path] {
-			long = path
-
-			break
-		}
-	}
-
-	var out bytes.Buffer
-
-	fs := trailingByteFs{Fs: backup.fs, suffix: long}
-	v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, fs)
-	v.UI = ui.NewWithColor(&out, false)
-
-	err := v.Restore(&vaultik.RestoreOptions{
-		SnapshotID: backup.snapshotID,
-		TargetDir:  backup.restoreDir,
-		Verify:     true,
-		SkipErrors: true,
-	})
-	require.Error(t, err, "restore must fail when files failed")
-
-	_, afterWarning, found := strings.Cut(out.String(),
-		fmt.Sprintf("%d file(s) failed to restore:", len(backup.lost)))
-	require.Truef(t, found, "no restore warning in:\n%s", out.String())
-
-	warning, summary, found := strings.Cut(afterWarning,
-		"Verification failed: 1 files did not match")
-	require.Truef(t, found, "no verification summary in:\n%s", out.String())
-
-	for path := range backup.lost {
-		assert.Equalf(t, 1, strings.Count(warning, path),
-			"%s must be listed once as failed to restore", path)
-		assert.NotContainsf(t, summary, path,
-			"%s was not restored and must not fail verification", path)
-	}
-
-	assert.NotContainsf(t, warning, long,
-		"%s was restored and must not be listed as failed to restore", long)
-	assert.Equalf(t, 1, strings.Count(summary, long),
-		"%s must be listed once as failing verification", long)
-}
-
-// trailingByteFs appends one byte, on close, to the file it opens whose
-// path ends in suffix, so that file is restored one byte too long and
-// fails verification.
-type trailingByteFs struct {
-	afero.Fs
-
-	suffix string
-}
-
-//nolint:ireturn // afero.Fs.OpenFile is defined to return the interface
-func (fs trailingByteFs) OpenFile(
-	name string, flag int, perm os.FileMode,
-) (afero.File, error) {
-	f, err := fs.Fs.OpenFile(name, flag, perm)
-	if err != nil || !strings.HasSuffix(name, fs.suffix) {
-		return f, err
-	}
-
-	return trailingByteFile{File: f}, nil
-}
-
-type trailingByteFile struct {
-	afero.File
-}
-
-func (f trailingByteFile) Close() error {
-	_, _ = f.Write([]byte{'x'})
-
-	return f.File.Close()
-}
-
 // TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
-// still ends the restore with an error when SkipErrors is not set, and
+// still ends the restore with an error when SkipErrors is not set.
-// that the error names the blob once.
 func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -248,32 +101,6 @@ func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
 	})
 
 	require.ErrorIs(t, err, storage.ErrNotFound)
-	assert.Equal(t, 1, strings.Count(err.Error(), "downloading blob"),
-		err.Error())
-}
-
-// TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held
-// by no blob still ends the restore, before any file is written, when
-// SkipErrors is not set.
-func TestRestoreChunkInNoBlobAbortsWithoutSkipErrors(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	backup := backupThenDropOneBlobFromMetadata(ctx, t)
-
-	v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
-
-	err := v.Restore(&vaultik.RestoreOptions{
-		SnapshotID: backup.snapshotID,
-		TargetDir:  backup.restoreDir,
-	})
-
-	require.ErrorContains(t, err, "chunk missing from blob map")
-
-	for path := range backup.files {
-		assert.NoFileExists(t, filepath.Join(backup.restoreDir, path))
-	}
 }
 
 // backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
@@ -283,54 +110,6 @@ func backupThenDeleteOneBlob(
 ) *missingBlobBackup {
 	t.Helper()
 
-	backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
-	require.NoError(t, db.Close())
-
-	require.NoError(t, backup.storer.Delete(ctx, fmt.Sprintf(
-		"blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)))
-
-	return backup
-}
-
-// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
-// single-chunk files, then removes the blob holding the first of them from
-// the snapshot and exports the snapshot metadata again. The exported
-// metadata then lists chunks that no blob holds.
-func backupThenDropOneBlobFromMetadata(
-	ctx context.Context, t *testing.T,
-) *missingBlobBackup {
-	t.Helper()
-
-	backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
-
-	// The uploads row refers to the blob row, so it would keep the export
-	// from deleting it.
-	for _, query := range []string{
-		"DELETE FROM snapshot_blobs WHERE blob_hash = ?",
-		"DELETE FROM uploads WHERE blob_hash = ?",
-	} {
-		_, err := db.Conn().ExecContext(ctx, query, lostBlob)
-		require.NoError(t, err)
-	}
-
-	sm := newFaultSnapshotManager(
-		backup.fs, backup.storer, backup.cfg, database.NewRepositories(db))
-	require.NoError(t,
-		sm.ExportSnapshotMetadata(ctx, db.Path(), backup.snapshotID))
-	require.NoError(t, db.Close())
-
-	return backup
-}
-
-// backupSingleChunkFiles backs up missingBlobFileCount single-chunk files.
-// It returns the backup, with lost naming the files stored in the same
-// blob as the first of them, the still open local index, and the hash of
-// that blob.
-func backupSingleChunkFiles(
-	ctx context.Context, t *testing.T,
-) (*missingBlobBackup, *database.DB, string) {
-	t.Helper()
-
 	fs := afero.NewOsFs()
 	tempDir := t.TempDir()
 	dataDir := filepath.Join(tempDir, "src")
@@ -364,11 +143,16 @@ func backupSingleChunkFiles(
 		blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
 	}
 
-	lostBlob := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
+	require.NoError(t, db.Close())
+
+	deleted := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
+	require.NoError(t, storer.Delete(ctx, fmt.Sprintf(
+		"blobs/%s/%s/%s", deleted[:2], deleted[2:4], deleted)))
+
 	lost := make(map[string]bool)
 
 	for path, hash := range blobOfFile {
-		if hash == lostBlob {
+		if hash == deleted {
 			lost[path] = true
 		}
 	}
@@ -384,7 +168,7 @@ func backupSingleChunkFiles(
 		restoreDir: filepath.Join(tempDir, "restored"),
 		files:      files,
 		lost:       lost,
-	}, db, lostBlob
+	}
 }
 
 // blobHashOfSingleChunkFile returns the hash of the blob holding the one

internal/vaultik/restore_sweeper.go

@@ -1,30 +1,46 @@
 package vaultik
 
-// restoreSweeper frees cached blobs once no file still to be restored in
+import (
-// this run needs them. It works as follows:
+	"context"
+	"fmt"
+
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+)
+
+// restoreSweeper frees cached blobs once all files that reference any of
+// their chunks have been restored. It works as follows:
 //
-//  1. After each chunk is restored, chunkRestored accumulates a running
+//  1. Callers add a file's ID to an in-memory restored set via
+//     fileRestored once the file is fully written to disk.
+//  2. After each chunk is restored, chunkRestored accumulates a running
 //     byte count.
-//  2. When the accumulator crosses a threshold (one hundredth of the
+//  3. When the accumulator crosses a threshold (one hundredth of the
 //     configured blob size — so a sweep runs about a hundred times per
 //     blob's worth of restored bytes), the sweeper iterates every key in
-//     the cache and deletes each blob that the restore plan no longer
+//     the cache. For each cached blob it asks the DB which files
-//     needs.
+//     reference any chunk in that blob, then compares that list against
+//     the in-memory restored set. If any referencing file is missing
+//     from the set the blob is kept; otherwise the cache entry is
+//     deleted.
 //
-// The plan drops a file once it is restored or has failed, and holds
+// All DB reads happen against the snapshot's temporary metadata DB,
-// only the files this run restores, so files outside the requested paths
+// which is local, indexed, and not under contention — the queries are
-// never keep a blob.
+// cheap and run at most once per blob per sweep interval.
 type restoreSweeper struct {
-	plan       *restorePlan
+	ctx        context.Context //nolint:containedctx // ctx bound at construction by design
+	repos      *database.Repositories
 	cache      *blobDiskCache
 	threshold  int64
 	bytesAccum int64
+	restored   map[string]struct{}
 }
 
 // newRestoreSweeper returns a sweeper that triggers eviction every
 // `threshold` bytes restored. Callers should pass blob_size_limit/100.
 func newRestoreSweeper(
-	plan *restorePlan,
+	ctx context.Context,
+	repos *database.Repositories,
 	cache *blobDiskCache,
 	threshold int64,
 ) *restoreSweeper {
@@ -33,12 +49,21 @@ func newRestoreSweeper(
 	}
 
 	return &restoreSweeper{
-		plan:      plan,
+		ctx:       ctx,
+		repos:     repos,
 		cache:     cache,
 		threshold: threshold,
+		restored:  make(map[string]struct{}),
 	}
 }
 
+// fileRestored records a file as fully restored. After this call, any
+// blob whose only remaining references come from files in the restored
+// set may be evicted on the next sweep.
+func (s *restoreSweeper) fileRestored(fileID string) {
+	s.restored[fileID] = struct{}{}
+}
+
 // chunkRestored accounts n bytes against the sweep threshold and runs a
 // sweep if the threshold has been crossed since the last sweep.
 func (s *restoreSweeper) chunkRestored(n int64) {
@@ -51,11 +76,60 @@ func (s *restoreSweeper) chunkRestored(n int64) {
 	s.sweep()
 }
 
-// sweep deletes every cached blob that no pending file references.
+// sweep deletes any cached blob whose chunks are no longer referenced
+// by an unrestored file. Per-blob DB failures are logged and the blob
+// is kept — we'd rather hold a blob longer than risk a re-download.
 func (s *restoreSweeper) sweep() {
 	for _, blobHash := range s.cache.Keys() {
-		if !s.plan.pendingFileUsesBlob(blobHash) {
+		needed, err := s.blobStillNeeded(blobHash)
+		if err != nil {
+			log.Debug("sweeper referencing-files query failed",
+				"blob_hash", blobHash[:16], "error", err)
+
+			continue
+		}
+
+		if !needed {
 			s.cache.Delete(blobHash)
 		}
 	}
 }
+
+// blobStillNeeded returns true if any file that references a chunk in
+// this blob has not yet been restored. On any error the function
+// returns true — keeping the blob is always the safe answer because we
+// can't prove we're done with it.
+func (s *restoreSweeper) blobStillNeeded(blobHash string) (bool, error) {
+	rows, err := s.repos.DB().Conn().QueryContext(s.ctx, `
+		SELECT DISTINCT fc.file_id
+		FROM file_chunks fc
+		JOIN blob_chunks bc ON bc.chunk_hash = fc.chunk_hash
+		JOIN blobs b ON b.id = bc.blob_id
+		WHERE b.blob_hash = ?
+	`, blobHash)
+	if err != nil {
+		return true, fmt.Errorf("querying referencing files: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		var fileID string
+
+		err := rows.Scan(&fileID)
+		if err != nil {
+			return true, fmt.Errorf("scanning file_id: %w", err)
+		}
+
+		if _, ok := s.restored[fileID]; !ok {
+			return true, nil
+		}
+	}
+
+	err = rows.Err()
+	if err != nil {
+		return true, err
+	}
+
+	return false, nil
+}

internal/vaultik/restore_sweeper_test.go

@@ -1,163 +0,0 @@
-package vaultik //nolint:testpackage // reads the restore blob cache's peak size
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"path/filepath"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/ui"
-)
-
-// TestRestoreFreesBlobsNoPendingFileNeeds asserts that the restore cache
-// frees a blob once no file still to be restored in this run needs it.
-// Every original file has a copy with the same content, so every blob
-// holds chunks of copies: a restore that leaves the copies out of its
-// paths, or fails on them, must not keep the blobs for them.
-func TestRestoreFreesBlobsNoPendingFileNeeds(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	// Small files keep the backup and both restores quick.
-	const fileBytes = 16 * 1024
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-
-	dataDir := filepath.Join(tempDir, "source")
-	originalsDir := filepath.Join(dataDir, "originals")
-	copiesDir := filepath.Join(dataDir, "copies")
-	storeDir := filepath.Join(tempDir, "remote")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	originals := writeFilesWithCopies(t, fs, originalsDir, copiesDir, fileBytes)
-
-	ctx := context.Background()
-
-	cfg, storer, snapshotID := setupLocalityBackup(
-		ctx, t, fs, dataDir, storeDir, dbPath, fileBytes)
-
-	t.Run("restore by path", func(t *testing.T) {
-		t.Parallel()
-
-		restoreDir := filepath.Join(tempDir, "by-path")
-
-		peak, err := restorePeakCachedBlobs(ctx, t, fs, cfg, storer,
-			&RestoreOptions{
-				SnapshotID: snapshotID,
-				TargetDir:  restoreDir,
-				Paths:      []string{originalsDir},
-			})
-		require.NoError(t, err)
-
-		requireRestoredFiles(t, fs, restoreDir, originals)
-		assert.LessOrEqualf(t, peak, 1,
-			"peak cached blobs was %d; blobs were kept for unselected files",
-			peak)
-	})
-
-	t.Run("restore with failed files", func(t *testing.T) {
-		t.Parallel()
-
-		restoreDir := filepath.Join(tempDir, "with-failures")
-
-		// A regular file where the copies directory belongs makes every
-		// copy fail to restore.
-		blocker := filepath.Join(restoreDir, copiesDir)
-		require.NoError(t, fs.MkdirAll(filepath.Dir(blocker), 0o755))
-		require.NoError(t, afero.WriteFile(fs, blocker, nil, 0o644))
-
-		peak, err := restorePeakCachedBlobs(ctx, t, fs, cfg, storer,
-			&RestoreOptions{
-				SnapshotID: snapshotID,
-				TargetDir:  restoreDir,
-				SkipErrors: true,
-			})
-		require.ErrorIs(t, err, errFilesFailedRestore)
-
-		requireRestoredFiles(t, fs, restoreDir, originals)
-		assert.LessOrEqualf(t, peak, 1,
-			"peak cached blobs was %d; blobs were kept for failed files", peak)
-	})
-}
-
-// writeFilesWithCopies writes 15 random files of fileBytes into
-// originalsDir and a copy of each into copiesDir. With
-// setupLocalityBackup given the same size, each file is one chunk and the
-// 15 chunks fill 3 blobs. Returns the originals' content by path.
-func writeFilesWithCopies(
-	t *testing.T, fs afero.Fs, originalsDir, copiesDir string, fileBytes int,
-) map[string][]byte {
-	t.Helper()
-
-	const fileCount = 15
-
-	require.NoError(t, fs.MkdirAll(originalsDir, 0o755))
-	require.NoError(t, fs.MkdirAll(copiesDir, 0o755))
-
-	originals := make(map[string][]byte, fileCount)
-
-	for i := range fileCount {
-		name := fmt.Sprintf("file-%02d.bin", i)
-		data := randomBytes(t, fileBytes)
-		original := filepath.Join(originalsDir, name)
-		originals[original] = data
-
-		require.NoError(t, afero.WriteFile(fs, original, data, 0o644))
-		require.NoError(t, afero.WriteFile(
-			fs, filepath.Join(copiesDir, name), data, 0o644))
-	}
-
-	return originals
-}
-
-// requireRestoredFiles requires every file in want to have been restored
-// under restoreDir with its content intact.
-func requireRestoredFiles(
-	t *testing.T, fs afero.Fs, restoreDir string, want map[string][]byte,
-) {
-	t.Helper()
-
-	for path, data := range want {
-		got, err := afero.ReadFile(fs, filepath.Join(restoreDir, path))
-		require.NoErrorf(t, err, "file missing after restore: %s", path)
-		require.Equalf(t, data, got, "byte mismatch for %s", path)
-	}
-}
-
-// restorePeakCachedBlobs runs a restore and returns the most blobs its
-// cache held at once, together with the restore's error.
-func restorePeakCachedBlobs(
-	ctx context.Context, t *testing.T, fs afero.Fs,
-	cfg *config.Config, storer storage.Storer, opts *RestoreOptions,
-) (int, error) {
-	t.Helper()
-
-	var cache *blobDiskCache
-
-	v := &Vaultik{
-		Config:  cfg,
-		Storage: storer,
-		Fs:      fs,
-		Stdout:  io.Discard,
-		Stderr:  io.Discard,
-		UI:      ui.NewWithColor(io.Discard, false),
-		restoreCacheObserver: func(c *blobDiskCache) {
-			cache = c
-		},
-	}
-	v.SetContext(ctx)
-
-	err := v.Restore(opts)
-
-	require.NotNil(t, cache, "restoreCacheObserver must fire during restore")
-
-	return cache.PeakLen(), err
-}

internal/vaultik/restore_unmatched_path_test.go

@@ -1,202 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/storage/faultstore"
-	"sneak.berlin/go/vaultik/internal/ui"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// These tests cover https://git.eeqj.de/sneak/vaultik/issues/309: a path
-// given to restore that matches nothing in the snapshot is named in a
-// warning and makes the restore fail, after the paths that do match are
-// restored.
-
-func TestRestoreNamesUnmatchedPathAndRestoresTheMatchingOne(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	storeDir := filepath.Join(tempDir, "remote")
-	restoreDir := filepath.Join(tempDir, "restored")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	ctx := context.Background()
-	cfg := faultTestConfig()
-	files := writeFaultSourceTree(t, fs, dataDir)
-
-	store, err := storage.NewFileStorer(storeDir)
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-
-	id := fullFaultBackup(ctx, t, fs, store, cfg,
-		database.NewRepositories(db), dataDir, dbPath, "unmatched")
-	require.NoError(t, db.Close())
-
-	matched := filepath.Join(dataDir, "small.txt")
-	unmatched := filepath.Join(dataDir, "typo.txt")
-
-	var out bytes.Buffer
-
-	v := newReaderVaultik(ctx, cfg, store, nil, fs)
-	v.UI = ui.NewWithColor(&out, false)
-
-	err = v.Restore(&vaultik.RestoreOptions{
-		SnapshotID: id,
-		TargetDir:  restoreDir,
-		Paths:      []string{matched, unmatched},
-	})
-
-	require.Error(t, err, "a path that matches nothing must fail the restore")
-	assert.Contains(t, out.String(), unmatched,
-		"the path that matches nothing must be named")
-	assert.NotContains(t, out.String(), matched,
-		"the path that matches must not be reported")
-
-	got, err := afero.ReadFile(fs, filepath.Join(restoreDir, matched))
-	require.NoError(t, err, "the path that matches must still be restored")
-	assert.Equal(t, files[matched], got)
-}
-
-// When no requested path matches, nothing is restored and each path is
-// named. One path is relative. The other goes through the symlink that
-// is the snapshot's configured path, while the entries are stored under
-// the directory the symlink points to.
-func TestRestoreNamesEveryPathWhenNothingMatches(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	const snapshotName = "linked"
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	linkDir := filepath.Join(tempDir, "srclink")
-	storeDir := filepath.Join(tempDir, "remote")
-	restoreDir := filepath.Join(tempDir, "restored")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	ctx := context.Background()
-
-	writeFaultSourceTree(t, fs, dataDir)
-	require.NoError(t, os.Symlink(dataDir, linkDir))
-
-	cfg := faultTestConfig()
-	cfg.IndexPath = dbPath
-	cfg.ChunkSize = config.Size(faultChunkSize)
-	cfg.Snapshots = map[string]config.SnapshotConfig{
-		snapshotName: {Paths: []string{linkDir}},
-	}
-
-	store, err := storage.NewFileStorer(storeDir)
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-
-	repos := database.NewRepositories(db)
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-
-	require.NoError(t, backUp(v, snapshotName))
-
-	id := localSnapshotID(ctx, t, repos, snapshotName)
-	require.NoError(t, db.Close())
-
-	throughLink := filepath.Join(linkDir, "sub")
-	relative := strings.TrimPrefix(filepath.Join(dataDir, "small.txt"), "/")
-
-	var out bytes.Buffer
-
-	reader := newReaderVaultik(ctx, cfg, store, nil, fs)
-	reader.UI = ui.NewWithColor(&out, false)
-
-	err = reader.Restore(&vaultik.RestoreOptions{
-		SnapshotID: id,
-		TargetDir:  restoreDir,
-		Paths:      []string{throughLink, relative},
-	})
-
-	require.Error(t, err, "a restore that matches nothing must fail")
-	assert.Contains(t, out.String(), throughLink,
-		"the path through the symlink must be named")
-	assert.Contains(t, out.String(), relative,
-		"the relative path must be named")
-
-	_, err = fs.Stat(restoreDir)
-	assert.True(t, os.IsNotExist(err), "nothing must be restored")
-}
-
-// Without --skip-errors a damaged blob stops the restore part way. The
-// path that matches nothing must still be named.
-func TestRestoreNamesUnmatchedPathWhenADamagedBlobStopsIt(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	storeDir := filepath.Join(tempDir, "remote")
-	restoreDir := filepath.Join(tempDir, "restored")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	ctx := context.Background()
-	cfg := faultTestConfig()
-
-	writeFaultSourceTree(t, fs, dataDir)
-
-	inner, err := storage.NewFileStorer(storeDir)
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-
-	id := fullFaultBackup(ctx, t, fs, inner, cfg,
-		database.NewRepositories(db), dataDir, dbPath, "damaged")
-	require.NoError(t, db.Close())
-
-	damaged := faultstore.New(inner)
-	damaged.OnGet = func(key string) faultstore.GetFault {
-		if strings.HasPrefix(key, "blobs/") {
-			return faultstore.GetCorrupt
-		}
-
-		return faultstore.GetNormal
-	}
-
-	matched := filepath.Join(dataDir, "small.txt")
-	unmatched := filepath.Join(dataDir, "typo.txt")
-
-	var out bytes.Buffer
-
-	v := newReaderVaultik(ctx, cfg, damaged, nil, fs)
-	v.UI = ui.NewWithColor(&out, false)
-
-	err = v.Restore(&vaultik.RestoreOptions{
-		SnapshotID: id,
-		TargetDir:  restoreDir,
-		Paths:      []string{matched, unmatched},
-	})
-
-	require.Error(t, err, "a damaged blob must stop the restore")
-	assert.Contains(t, err.Error(), "blob",
-		"the restore must stop on the damaged blob")
-	assert.Contains(t, out.String(), unmatched,
-		"the path that matches nothing must be named")
-}

internal/vaultik/reused_blob_verify_test.go

@@ -1,83 +0,0 @@
-package vaultik_test
-
-import (
-	"context"
-	"io"
-	"path/filepath"
-	"testing"
-
-	"filippo.io/age"
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/ui"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// TestVerifyPassesWhenBackupReusesStoredBlob backs up a file, then backs it
-// up again from an empty local index, as after `database delete`, with an
-// extra age recipient. The second backup builds the same blob, finds it
-// already stored and skips the upload, but its own copy is larger, because
-// age writes a header entry for each recipient. The second snapshot must
-// record the size of the stored object, so shallow and deep verify pass.
-func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-
-	dataDir := filepath.Join(tempDir, "source")
-	storeDir := filepath.Join(tempDir, "remote")
-	secondDBPath := filepath.Join(tempDir, "second.sqlite")
-
-	chunkSize := int64(64 * 1024)
-	maxBlobSize := int64(512 * 1024)
-
-	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
-	require.NoError(t, afero.WriteFile(fs,
-		filepath.Join(dataDir, "data.bin"),
-		bytesPattern("reuse-", int(chunkSize*3)), 0o644))
-
-	ctx := context.Background()
-
-	runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
-		filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
-
-	env := setupDedupBackupEnv(
-		ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
-
-	defer func() { _ = env.db.Close() }()
-
-	extra, err := age.GenerateX25519Identity()
-	require.NoError(t, err)
-
-	env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
-		extra.Recipient().String())
-
-	snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
-		env.cfg.Hostname, "second", dataDir, secondDBPath)
-	require.Positive(t, result.BlobsCreated)
-	require.Zero(t, result.BlobsUploaded,
-		"the second backup should find every blob already stored")
-
-	require.NoError(t, env.db.Close())
-
-	verifier := &vaultik.Vaultik{
-		Config:  env.cfg,
-		Storage: env.storer,
-		Fs:      fs,
-		Stdout:  io.Discard,
-		Stderr:  io.Discard,
-		UI:      ui.NewWithColor(io.Discard, false),
-	}
-	verifier.SetContext(ctx)
-
-	require.NoError(t,
-		verifier.VerifySnapshotWithOptions(snapshotID, &vaultik.VerifyOptions{}),
-		"shallow verify should pass")
-	require.NoError(t,
-		verifier.VerifySnapshotWithOptions(snapshotID,
-			&vaultik.VerifyOptions{Deep: true}),
-		"deep verify should pass")
-}

internal/vaultik/shallow_verify_test.go

@@ -27,7 +27,7 @@ import (
 // then grows one stored blob so its size no longer matches the manifest.
 // Shallow verify must then fail, count the grown blob as a size mismatch,
 // and drop it from the verified count rather than continuing to report it
-// as checked, and its error must not repeat the cli layer's prefix.
+// as checked.
 func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -96,12 +96,6 @@ func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) {
 	require.Equal(t, 1, bad.Mismatched)
 	require.Equal(t, healthy.BlobCount-1, bad.Verified,
 		"the wrong-sized blob must not be counted as verified")
-
-	// The cli layer prints this error after its own "Verification failed:".
-	err = newVerifier(io.Discard).VerifySnapshotWithOptions(
-		snapshotID, &vaultik.VerifyOptions{})
-	require.EqualError(t, err,
-		"problems found in storage: 1 blobs have the wrong size")
 }
 
 // TestShallowVerifyDetectsMissingDatabase backs up a real snapshot,

internal/vaultik/snapshot.go

@@ -23,10 +23,8 @@ var (
 	errSnapshotNotInConfig  = errors.New("snapshot not found in config")
 	errNoSnapshotsInConfig  = errors.New("no snapshots configured")
 	errSnapshotVerifyFailed = errors.New("verification failed")
-	errProblemsInStorage    = errors.New("problems found in storage")
 	errRemoveAllNeedsForce  = errors.New("--all requires --force")
 	errSnapshotNotFound     = errors.New("snapshot not found")
-	errSnapshotRemoveFailed = errors.New("snapshot(s) could not be removed")
 	errInvalidTableName     = errors.New("invalid table name")
 	errNoAgeRecipients      = errors.New(
 		"creating a snapshot needs at least one public key in " +
@@ -244,18 +242,12 @@ func (v *Vaultik) createNamedSnapshot(
 		return err
 	}
 
-	previousSnapshotID, err := v.previousSnapshotID(snapName)
-	if err != nil {
-		return err
-	}
-
 	scanner := v.ScannerFactory(snapshot.ScannerParams{
-		EnableProgress:     !opts.Cron,
+		EnableProgress: !opts.Cron,
-		UI:                 v.UI,
+		UI:             v.UI,
-		Fs:                 v.Fs,
+		Fs:             v.Fs,
-		Exclude:            v.Config.GetExcludes(snapName),
+		Exclude:        v.Config.GetExcludes(snapName),
-		SkipErrors:         opts.SkipErrors,
+		SkipErrors:     opts.SkipErrors,
-		PreviousSnapshotID: previousSnapshotID,
 	})
 
 	snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -290,24 +282,6 @@ func (v *Vaultik) createNamedSnapshot(
 	return nil
 }
 
-// previousSnapshotID returns the ID of the most recent completed snapshot
-// named snapName in the local index, or "" when there is none.
-func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
-	snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
-	if err != nil {
-		return "", fmt.Errorf("listing snapshots: %w", err)
-	}
-
-	for _, s := range snapshots {
-		if s.CompletedAt != nil &&
-			parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
-			return s.ID.String(), nil
-		}
-	}
-
-	return "", nil
-}
-
 // resolveSnapshotPaths resolves source directories to absolute paths
 // with symlink resolution.
 func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
@@ -599,7 +573,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
 
 	if len(toDelete) == 0 {
 		if !opts.Quiet {
-			v.UI.Infof("No snapshots to delete.")
+			v.printlnStdout("No snapshots to delete")
 		}
 
 		return nil
@@ -637,7 +611,7 @@ func selectSnapshotsToPurge(
 	case opts.OlderThan != "":
 		duration, err := parseDuration(opts.OlderThan)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("invalid duration: %w", err)
 		}
 
 		cutoff := time.Now().UTC().Add(-duration)
@@ -656,19 +630,19 @@ func selectSnapshotsToPurge(
 func (v *Vaultik) confirmAndExecutePurge(
 	toDelete []SnapshotInfo, force, quiet bool,
 ) error {
-	// Confirm unless --force is used. The list, the prompt and its answer
+	if !quiet {
-	// bypass the UI: they are an interactive exchange the operator must
-	// see, so --quiet does not silence them.
-	if !force {
 		v.stdoutf("The following snapshots will be deleted:\n\n")
 
 		for _, snap := range toDelete {
 			v.stdoutf("  %s (%s, %s)\n",
 				snap.ID,
-				snap.Timestamp.Format(time.DateTime),
+				snap.Timestamp.Format("2006-01-02 15:04:05"),
 				ubytes(snap.CompressedSize))
 		}
+	}
 
+	// Confirm unless --force is used
+	if !force {
 		v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete))
 
 		var confirm string
@@ -686,16 +660,7 @@ func (v *Vaultik) confirmAndExecutePurge(
 			return nil
 		}
 	} else if !quiet {
-		v.UI.Infof("The following snapshots will be deleted:")
+		v.stdoutf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete))
-
-		for _, snap := range toDelete {
-			v.UI.Detailf("%s (%s, %s)",
-				v.UI.Snapshot(snap.ID.String()),
-				snap.Timestamp.Format(time.DateTime),
-				ubytes(snap.CompressedSize))
-		}
-
-		v.UI.Infof("Deleting %d snapshot(s) (--force specified).", len(toDelete))
 	}
 
 	// Delete snapshots (both local and remote)
@@ -703,11 +668,10 @@ func (v *Vaultik) confirmAndExecutePurge(
 		snapshotID := snap.ID.String()
 		log.Info("Deleting snapshot", "id", snapshotID)
 
-		// Stop before the remote delete, so a snapshot still in the local
-		// index keeps its metadata on the destination store.
 		err := v.deleteSnapshotFromLocalDB(snapshotID)
 		if err != nil {
-			return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
+			log.Error("Failed to delete from local database",
+				"snapshot_id", snapshotID, "error", err)
 		}
 
 		_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))
@@ -728,9 +692,9 @@ func (v *Vaultik) confirmAndExecutePurge(
 	}
 
 	if !quiet {
-		v.UI.Infof("Deleted %d snapshot(s).", len(toDelete))
+		v.stdoutf("Deleted %d snapshot(s)\n", len(toDelete))
-		v.UI.Noticef("Run '%s' to clean up unreferenced remote blobs.",
+		v.printlnStdout(
-			pruneCommandHint)
+			"\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.")
 	}
 
 	return nil
@@ -966,10 +930,8 @@ func (v *Vaultik) formatVerifyResult(
 		v.printVerifySummary(result, failure)
 	}
 
-	// Not errSnapshotVerifyFailed: the cli layer prints this after its own
-	// "Verification failed:".
 	if failure != "" {
-		return fmt.Errorf("%w: %s", errProblemsInStorage, failure)
+		return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, failure)
 	}
 
 	return nil
@@ -1105,7 +1067,7 @@ func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
 			log.Info("Removing stale local snapshot record", "snapshot_id", id)
 
 			if !opts.JSON {
-				v.UI.Infof("Removing stale local record: %s.", v.UI.Snapshot(id))
+				v.stdoutf("Removing stale local record: %s\n", id)
 			}
 
 			err = v.deleteSnapshotFromLocalDB(id)
@@ -1128,9 +1090,9 @@ func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
 	}
 
 	if removed == 0 {
-		v.UI.Infof("No stale local snapshots found.")
+		v.printlnStdout("No stale local snapshots found.")
 	} else {
-		v.UI.Infof("Removed %d stale local snapshot record(s).", removed)
+		v.stdoutf("Removed %d stale local snapshot record(s).\n", removed)
 	}
 
 	return nil
@@ -1255,11 +1217,9 @@ const snapshotRemoveCommandHint = "vaultik snapshot remove"
 // destination store. Blobs are NOT touched: removing a snapshot's
 // blobs requires reading every remaining remote manifest (the remote
 // may hold snapshots this host doesn't know about), which is what
-// `vaultik prune` is for — when metadata was removed from the
+// `vaultik prune` is for — the command prints the prune invocation to
-// destination store, the text output names the prune invocation to run
+// run as a follow-up. When the remote is unreachable the command still
-// as a follow-up. When removing the metadata fails, a snapshot in the
+// completes the local-DB removal and warns.
-// local index is still removed from it, with a warning; a snapshot
-// only on the destination store is an error.
 func (v *Vaultik) RemoveSnapshot(
 	snapshotID string, opts *RemoveOptions,
 ) (*RemoveResult, error) {
@@ -1294,10 +1254,8 @@ func (v *Vaultik) RemoveSnapshot(
 		}
 	}
 
-	result.RemoteRemoved, err = v.removeSnapshotRemote(
+	if !opts.LocalOnly {
-		snapshotID, inLocalIndex, opts)
+		result.RemoteRemoved = v.removeSnapshotRemote(snapshotID, opts)
-	if err != nil {
-		return result, err
 	}
 
 	if v.SnapshotManager != nil {
@@ -1313,15 +1271,14 @@ func (v *Vaultik) RemoveSnapshot(
 	}
 
 	if inLocalIndex {
-		v.UI.Infof("Removed snapshot '%s' from local database.",
+		v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID)
-			v.UI.Snapshot(snapshotID))
 	}
 
 	if result.RemoteRemoved {
-		v.UI.Infof("Removed snapshot metadata from remote storage.")
+		v.printlnStdout("Removed snapshot metadata from remote storage")
-		v.UI.Noticef("The removed snapshot's blobs remain on the remote. "+
+		v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. "+
 			"Run '%s' to delete any blobs no longer referenced by any "+
-			"remaining remote snapshot.", pruneCommandHint)
+			"remaining remote snapshot.\n", pruneCommandHint)
 	}
 
 	return result, nil
@@ -1377,23 +1334,21 @@ func (v *Vaultik) removeSnapshotDryRun(
 		return result, v.outputRemoveJSON(result)
 	}
 
-	v.UI.Infof("Would remove snapshot: %s.", v.UI.Snapshot(snapshotID))
+	v.stdoutf("Would remove snapshot: %s\n", snapshotID)
 
 	if !opts.LocalOnly {
-		v.UI.Infof("Would also remove snapshot metadata from " +
+		v.printlnStdout("Would also remove snapshot metadata from " +
-			"remote storage (blobs untouched).")
+			"remote storage (blobs untouched)")
 	}
 
-	v.UI.Noticef("Dry run: no changes made.")
+	v.printlnStdout("[Dry run - no changes made]")
 
 	return result, nil
 }
 
 // confirmRemoveSnapshot prompts for confirmation before removing a
 // snapshot. Returns false (treating EOF/read errors as "no") unless the
-// user answers "y". The prompt and its answer bypass the UI: they are an
+// user answers "y".
-// interactive exchange the operator must see, so --quiet does not
-// silence them.
 func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool {
 	if opts.LocalOnly {
 		v.stdoutf("Remove snapshot '%s' from local database "+
@@ -1422,33 +1377,19 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
 }
 
 // removeSnapshotRemote strips the snapshot's metadata from the
-// destination store. It returns true only when the delete finished
+// destination store, warning and proceeding on failure: the local-DB
-// without an error and removed at least one object; under LocalOnly it
+// removal has already happened, so the user is told the remote half
-// does nothing. When the snapshot is only on the destination store, a
+// didn't finish and to run `vaultik snapshot remove` for the snapshot
-// failure is returned as an error, because that removal is all the
+// again once the destination store is reachable (`vaultik prune` never
-// command does. When the snapshot was in the local index, its removal
+// removes snapshot metadata). Returns true when the remote removal
-// there has already happened, so a failure is only a warning telling the
+// succeeded and deleted something.
-// user to run `vaultik snapshot remove` for the snapshot again once the
+func (v *Vaultik) removeSnapshotRemote(snapshotID string, opts *RemoveOptions) bool {
-// destination store is reachable (`vaultik prune` never removes snapshot
-// metadata).
-func (v *Vaultik) removeSnapshotRemote(
-	snapshotID string, inLocalIndex bool, opts *RemoveOptions,
-) (bool, error) {
-	if opts.LocalOnly {
-		return false, nil
-	}
-
 	log.Info("Removing snapshot metadata from remote storage",
 		"snapshot_id", snapshotID)
 
 	remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
 
 	deleted, err := v.deleteRemoteSnapshotByKey(remoteKey)
-	if err != nil && !inLocalIndex {
-		return false, fmt.Errorf(
-			"removing snapshot metadata from destination store: %w", err)
-	}
-
 	if err != nil {
 		log.Warn("Could not remove snapshot metadata from remote storage; "+
 			"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
@@ -1463,10 +1404,10 @@ func (v *Vaultik) removeSnapshotRemote(
 				err, snapshotRemoveCommandHint, snapshotID)
 		}
 
-		return false, nil
+		return false
 	}
 
-	return deleted > 0, nil
+	return deleted > 0
 }
 
 // RemoveAllSnapshots removes every snapshot known to the local
@@ -1513,7 +1454,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
 
 	if len(localSnaps) == 0 && len(orphanRemoteKeys) == 0 {
 		if !opts.JSON {
-			v.UI.Infof("No snapshots found.")
+			v.printlnStdout("No snapshots found")
 		}
 
 		return &RemoveResult{}, nil
@@ -1618,27 +1559,27 @@ func (v *Vaultik) handleRemoveAllDryRun(
 	}
 
 	if !opts.JSON {
-		v.UI.Infof("Would remove %d local snapshot(s):", len(localSnaps))
+		v.stdoutf("Would remove %d local snapshot(s):\n", len(localSnaps))
 
 		for _, id := range localSnaps {
-			v.UI.Detailf("%s", v.UI.Snapshot(id))
+			v.stdoutf("  %s\n", id)
 		}
 
 		if !opts.LocalOnly {
 			if len(orphanRemoteKeys) > 0 {
-				v.UI.Infof("Would also remove %d orphan remote snapshot key(s):",
+				v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n",
 					len(orphanRemoteKeys))
 
 				for _, key := range orphanRemoteKeys {
-					v.UI.Detailf("%s", key)
+					v.stdoutf("  %s\n", key)
 				}
 			} else {
-				v.UI.Infof("Would also remove snapshot metadata from " +
+				v.printlnStdout("Would also remove snapshot metadata from " +
-					"remote storage (blobs untouched).")
+					"remote storage (blobs untouched)")
 			}
 		}
 
-		v.UI.Noticef("Dry run: no changes made.")
+		v.printlnStdout("[Dry run - no changes made]")
 	}
 
 	if opts.JSON {
@@ -1650,8 +1591,8 @@ func (v *Vaultik) handleRemoveAllDryRun(
 
 // executeRemoveAll deletes every local snapshot and, unless LocalOnly
 // is set, every corresponding remote metadata directory plus any
-// orphan remote keys. Blobs are not touched. It returns an error when
+// orphan remote keys. Blobs are not touched; the printed prune-command
-// any snapshot could not be removed.
+// hint is the next step.
 func (v *Vaultik) executeRemoveAll(
 	localSnaps, orphanRemoteKeys []string, opts *RemoveOptions,
 ) (*RemoveResult, error) {
@@ -1665,11 +1606,18 @@ func (v *Vaultik) executeRemoveAll(
 
 	result := &RemoveResult{}
 
-	failures := v.removeAllLocalSnapshots(localSnaps, opts, result)
+	remoteErrors := v.removeAllLocalSnapshots(localSnaps, opts, result)
 
 	if !opts.LocalOnly {
-		failures += v.removeAllOrphanRemotes(orphanRemoteKeys, result)
+		remoteErrors += v.removeAllOrphanRemotes(orphanRemoteKeys, result)
-		result.RemoteRemoved = failures == 0
+
+		if remoteErrors == 0 {
+			result.RemoteRemoved = true
+		} else if v.UI != nil {
+			v.UI.Warningf("Some remote metadata deletions failed. Run '%s' "+
+				"once the remote is healthy to clean up unreferenced blobs.",
+				pruneCommandHint)
+		}
 	}
 
 	if v.SnapshotManager != nil {
@@ -1679,18 +1627,17 @@ func (v *Vaultik) executeRemoveAll(
 		}
 	}
 
-	if failures > 0 {
-		return result, fmt.Errorf("%d %w", failures, errSnapshotRemoveFailed)
-	}
-
 	if opts.JSON {
 		return result, v.outputRemoveJSON(result)
 	}
 
-	v.UI.Infof("Removed %d snapshot(s).", len(result.SnapshotsRemoved))
+	v.stdoutf("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved))
 
 	if !opts.LocalOnly && result.RemoteRemoved {
-		v.UI.Infof("Removed snapshot metadata from remote storage.")
+		v.printlnStdout("Removed snapshot metadata from remote storage")
+		v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. "+
+			"Run '%s' to delete any blobs no longer referenced by any "+
+			"remaining remote snapshot.\n", pruneCommandHint)
 	}
 
 	return result, nil
@@ -1733,13 +1680,12 @@ func (v *Vaultik) deleteIncompleteSnapshot(snapshotIDStr string, id any) bool {
 }
 
 // removeAllLocalSnapshots removes each known local snapshot (and, unless
-// --local-only, its remote metadata), returning the number of snapshots
+// --local-only, its remote metadata), returning the number of remote
-// it could not remove. A snapshot left in the local index keeps its
+// deletion failures.
-// remote metadata too, since it is not an orphan remote key.
 func (v *Vaultik) removeAllLocalSnapshots(
 	localSnaps []string, opts *RemoveOptions, result *RemoveResult,
 ) int {
-	failures := 0
+	remoteErrors := 0
 
 	for _, snapshotID := range localSnaps {
 		log.Info("Removing snapshot", "snapshot_id", snapshotID)
@@ -1749,8 +1695,6 @@ func (v *Vaultik) removeAllLocalSnapshots(
 			log.Error("Failed to remove from local database",
 				"snapshot_id", snapshotID, "error", err)
 
-			failures++
-
 			continue
 		}
 
@@ -1761,14 +1705,14 @@ func (v *Vaultik) removeAllLocalSnapshots(
 				log.Warn("Failed to remove snapshot metadata from remote",
 					"snapshot_id", snapshotID, "error", err)
 
-				failures++
+				remoteErrors++
 			}
 		}
 
 		result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID)
 	}
 
-	return failures
+	return remoteErrors
 }
 
 // removeAllOrphanRemotes deletes remote snapshot metadata that has no
@@ -1947,17 +1891,13 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
 	snapshotsRemain := countDiff(snapshotCountBefore, &result.SnapshotsDeleted)
 
 	v.UI.Completef("Pruned local index database.")
-	v.UI.Detailf("Incomplete snapshots: %s removed "+
+	v.UI.Detailf("Incomplete snapshots: %s removed (%s remain).",
-		"(total snapshots left in the local index: %s).",
 		countText(&result.SnapshotsDeleted), countText(snapshotsRemain))
-	v.UI.Detailf("Orphaned files: %s removed "+
+	v.UI.Detailf("Orphaned files: %s removed (%s remain).",
-		"(total files left in the local index: %s).",
 		countText(result.FilesDeleted), countText(fileCountAfter))
-	v.UI.Detailf("Orphaned chunks: %s removed "+
+	v.UI.Detailf("Orphaned chunks: %s removed (%s remain).",
-		"(total chunks left in the local index: %s).",
 		countText(result.ChunksDeleted), countText(chunkCountAfter))
-	v.UI.Detailf("Orphaned blobs: %s removed "+
+	v.UI.Detailf("Orphaned blobs: %s removed (%s remain).",
-		"(total blobs left in the local index: %s).",
 		countText(result.BlobsDeleted), countText(blobCountAfter))
 
 	return result, nil

internal/vaultik/snapshot_deleted_count_test.go

@@ -1,249 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/ui"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
-// backup counts a file as deleted only when the previous snapshot of
-// the same name had it and it is gone, and never counts an excluded
-// file.
-
-// deletedCountEnv is a backup setup over the source directory src whose
-// user-facing output is kept in out. Each test configures its snapshots
-// with setSnapshot.
-type deletedCountEnv struct {
-	v   *vaultik.Vaultik
-	out *bytes.Buffer
-	src string
-}
-
-// newDeletedCountEnv writes files, keyed by path relative to src.
-func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
-	t.Helper()
-
-	ctx := context.Background()
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	srcDir := filepath.Join(tempDir, "src")
-
-	for rel, content := range files {
-		path := filepath.Join(srcDir, rel)
-		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
-		require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
-	}
-
-	cfg := faultTestConfig()
-	cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
-	cfg.Snapshots = map[string]config.SnapshotConfig{}
-
-	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, cfg.IndexPath)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	repos := database.NewRepositories(db)
-	out := &bytes.Buffer{}
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-	v.UI = ui.NewWithColor(out, false)
-
-	return &deletedCountEnv{v: v, out: out, src: srcDir}
-}
-
-// setSnapshot configures the snapshot name to back up the path rel
-// beneath src, with the given excludes.
-func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
-	e.v.Config.Snapshots[name] = config.SnapshotConfig{
-		Paths:   []string{filepath.Join(e.src, rel)},
-		Exclude: exclude,
-	}
-}
-
-// backUp runs a backup of the named snapshot and returns its output.
-func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
-	t.Helper()
-
-	e.out.Reset()
-	require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-		Snapshots: []string{name},
-	}))
-
-	return e.out.String()
-}
-
-// A file this snapshot excludes is not counted as deleted: neither one
-// that another snapshot backs up, nor one that the previous snapshot of
-// this name had before the exclude was added.
-func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"top.txt":       []byte("top"),
-		"cache/tmp.bin": []byte("cache"),
-		"photos/a.jpg":  []byte("photo a"),
-		"photos/b.jpg":  []byte("photo b"),
-		"photos/c.jpg":  []byte("photo c"),
-	})
-	env.setSnapshot("photos", "photos")
-	env.setSnapshot("home", "", "/photos")
-
-	env.backUp(t, "photos")
-	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
-
-	env.setSnapshot("home", "", "/photos", "/cache")
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-}
-
-// A file inside a directory that an exclude added after a backup
-// matches is not counted as deleted, though the pattern matches only the
-// directory and the directory is gone from disk.
-func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"top.txt":                         []byte("top"),
-		"user/.local/share/Trash/old.txt": []byte("old"),
-	})
-	env.setSnapshot("home", "")
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
-
-	require.NoError(t, os.RemoveAll(
-		filepath.Join(env.src, "user/.local/share/Trash")))
-
-	env.setSnapshot("home", "", "*/.local/share/Trash")
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-}
-
-// A file whose directory was replaced by a regular file of the same name
-// is counted as deleted.
-func TestBackupCountsFileInDirectoryReplacedByFileAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"dir/inside.txt": []byte("inside"),
-	})
-	env.setSnapshot("home", "")
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 1, 0))
-
-	dir := filepath.Join(env.src, "dir")
-	require.NoError(t, os.RemoveAll(dir))
-	require.NoError(t, os.WriteFile(dir, []byte("now a file"), 0o600))
-
-	assert.Contains(t, env.backUp(t, "home"),
-		"Files: 1 examined, 1 backed up, 0 unchanged, 1 deleted.")
-}
-
-// Under --skip-errors a file inside a directory that the walk cannot
-// open is not counted as deleted: it is still on disk.
-func TestBackupDoesNotCountFileInUnopenableDirectoryAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"open.txt":          []byte("open"),
-		"locked/inside.txt": []byte("inside"),
-	})
-	env.setSnapshot("work", "")
-
-	assert.Contains(t, env.backUp(t, "work"), filesLine(2, 2, 0))
-
-	// A directory's mode does not lock out root, who runs the tests, so
-	// opening the directory is made to fail instead.
-	env.v.Fs = &openFailFs{
-		path: filepath.Join(env.src, "locked"),
-		err:  os.ErrPermission,
-	}
-
-	env.out.Reset()
-	require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-		SkipErrors: true,
-		Snapshots:  []string{"work"},
-	}))
-	assert.Contains(t, env.out.String(), filesLine(1, 0, 1))
-}
-
-// A backup counts only files that the previous snapshot of its own name
-// had, not one that a later snapshot of another name over the same path
-// had and that is since removed.
-func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"kept.txt": []byte("kept"),
-	})
-	env.setSnapshot("home", "")
-	env.setSnapshot("other", "")
-
-	env.backUp(t, "home")
-
-	// Snapshot start times are kept in whole seconds: wait so that other's
-	// snapshot is strictly the most recent.
-	time.Sleep(time.Second)
-
-	extra := filepath.Join(env.src, "extra.txt")
-	require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
-	env.backUp(t, "other")
-	require.NoError(t, os.Remove(extra))
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-}
-
-// A file removed from disk is counted as deleted by the next backup of
-// the snapshot that had it, and not by any later backup of that
-// snapshot, nor by the first backup of another name over the same path.
-func TestBackupCountsDeletedFileOnce(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	aContent := bytes.Repeat([]byte("a"), 1000)
-	bContent := bytes.Repeat([]byte("b"), 2000)
-	keptContent := bytes.Repeat([]byte("k"), 500)
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"a.txt":    aContent,
-		"b.txt":    bContent,
-		"kept.txt": keptContent,
-	})
-	env.setSnapshot("home", "")
-	env.setSnapshot("other", "")
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
-
-	require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
-	require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
-
-	summary := env.backUp(t, "home")
-	assert.Contains(t, summary,
-		"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
-	assert.Contains(t, summary,
-		fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
-			env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
-			env.v.UI.Size(int64(len(aContent)+len(bContent)))))
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-	assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
-}

internal/vaultik/snapshot_list.go

@@ -61,9 +61,7 @@ const tabPadding = 3
 // attempt is made to recover or fabricate the human ID.
 //
 // Snapshots in the local index with no counterpart on the destination
-// store are reported as drift below the table, unless the local index
+// store are reported as drift below the table.
-// is bound to a different destination than the configured one: that is
-// reported instead.
 //
 // If remote listing fails (unmounted volume, permission denied,
 // network), we degrade to local-only with a warning. List never fails
@@ -105,26 +103,7 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
 		return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
 	})
 
-	stored, configured, err := v.storageBinding()
-	if err != nil {
-		return err
-	}
-
-	// A local index bound to another destination describes that one, so
-	// its records are expected to be missing here, and prune refuses to
-	// run. The binding is reported instead of advising prune.
-	boundElsewhere := stored != "" && stored != configured
-
 	if jsonOutput {
-		// stdout holds only the document, so this goes to the logger.
-		if boundElsewhere {
-			log.Warn("Local index is bound to a different backup "+
-				"destination; to proceed, revert storage_url (or the s3.* "+
-				"endpoint, bucket and prefix) to the bound destination, "+
-				"or run 'vaultik database delete'",
-				"bound_to", stored, "configured", configured)
-		}
-
 		if remoteErr == nil {
 			v.reportJSONListingLimits(listing)
 		}
@@ -146,13 +125,8 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
 		}
 	}
 
-	if boundElsewhere {
-		v.UI.Warningf("%v\n%s", errStorageBindingMismatch,
-			buildBindingMismatchMessage(stored, configured))
-	}
-
 	if remoteErr == nil {
-		v.reportListDrift(snapshots, listing, boundElsewhere)
+		v.reportListDrift(snapshots, listing)
 	}
 
 	return nil
@@ -458,9 +432,7 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo {
 // reportListDrift prints the reconciliation notes the merged table
 // cannot express on its own: local records with no counterpart on the
 // destination store, plus counts of remote snapshots that were
-// unreadable or omitted. Local records are not reported when
+// unreadable or omitted.
-// boundElsewhere is set: ListSnapshots has already said why they are
-// missing.
 //
 // This is what remains of the old reportRemoteDrift, and it no longer
 // touches the destination store. Its remote-only half collapsed into
@@ -469,7 +441,6 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo {
 // so `snapshot list` lists the destination exactly once per invocation.
 func (v *Vaultik) reportListDrift(
 	snapshots []SnapshotInfo, listing *remoteSnapshotListing,
-	boundElsewhere bool,
 ) {
 	var localOnly []string
 
@@ -479,7 +450,7 @@ func (v *Vaultik) reportListDrift(
 		}
 	}
 
-	if len(localOnly) > 0 && !boundElsewhere {
+	if len(localOnly) > 0 {
 		v.UI.Warningf("%d local snapshot record(s) not found in backup "+
 			"destination store:", len(localOnly))
 

internal/vaultik/snapshot_list_test.go

@@ -421,74 +421,6 @@ func TestListSnapshots_LocalOnlyReportedAsDrift(t *testing.T) {
 	assert.NotContains(t, out, "snapshot cleanup")
 }
 
-// TestListSnapshots_BoundElsewhereDoesNotAdvisePrune covers a changed
-// storage_url: the local index is still bound to the old destination,
-// so its records are missing from the new one. prune refuses to run in
-// that state, so the listing must give the choices prune gives instead
-// of advising it.
-func TestListSnapshots_BoundElsewhereDoesNotAdvisePrune(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	const (
-		oldURL = "file:///mnt/backups/old"
-		newURL = "file:///mnt/backups/new"
-	)
-
-	env := newListEnv(t)
-	env.v.Config.StorageURL = newURL
-	require.NoError(t, env.v.Repositories.LocalMeta.Set(context.Background(),
-		database.LocalMetaKeyStorageURL, oldURL))
-	env.addLocal(t, listLocalID, time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC))
-
-	err := env.v.ListSnapshots(false)
-	require.NoError(t, err)
-
-	out := env.stdout.String()
-
-	assert.Contains(t, out, listLocalID)
-	assert.Contains(t, out,
-		"local index is bound to a different backup destination")
-	assert.Contains(t, out, oldURL)
-	assert.Contains(t, out, newURL)
-	assert.Contains(t, out, "revert storage_url")
-	assert.Contains(t, out, "vaultik database delete")
-	assert.NotContains(t, out, "vaultik prune")
-}
-
-// TestListSnapshots_JSONReportsBoundElsewhere is the --json side of the
-// test above. Every local row comes out with remote_present false, which
-// a script reads as stale local records unless it is told, on stderr,
-// that the local index is bound to a different destination.
-//
-//nolint:paralleltest // captureProcessStderr replaces os.Stderr
-func TestListSnapshots_JSONReportsBoundElsewhere(t *testing.T) {
-	const (
-		oldURL = "file:///mnt/backups/old"
-		newURL = "file:///mnt/backups/new"
-	)
-
-	env := newListEnv(t)
-	env.v.Config.StorageURL = newURL
-	require.NoError(t, env.v.Repositories.LocalMeta.Set(context.Background(),
-		database.LocalMetaKeyStorageURL, oldURL))
-	env.addLocal(t, listLocalID, time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC))
-
-	stderr := captureProcessStderr(t, func() {
-		require.NoError(t, env.v.ListSnapshots(true))
-	})
-
-	rows := decodeListJSON(t, env.stdout.String())
-	require.Len(t, rows, 1)
-	assert.Equal(t, listLocalID, rows[0].ID)
-
-	assert.Contains(t, stderr, "bound to a different backup destination")
-	assert.Contains(t, stderr, oldURL)
-	assert.Contains(t, stderr, newURL)
-	assert.Contains(t, stderr, "revert storage_url")
-	assert.Contains(t, stderr, "vaultik database delete")
-}
-
 // TestListSnapshots_UnreachableRemoteDegrades covers the promise in the
 // doc comment: an unreachable destination is a warning plus local-only
 // output, never a failure.
@@ -551,14 +483,13 @@ func TestListSnapshots_UnreadableManifestDoesNotHideOthers(t *testing.T) {
 //
 //nolint:tagliatelle // snake_case is the established output format
 type listJSONRow struct {
-	ID               string `json:"id"`
+	ID             string `json:"id"`
-	RemoteKey        string `json:"remote_key"`
+	RemoteKey      string `json:"remote_key"`
-	Timestamp        string `json:"timestamp"`
+	Timestamp      string `json:"timestamp"`
-	CompressedSize   int64  `json:"compressed_size"`
+	CompressedSize int64  `json:"compressed_size"`
-	UncompressedSize int64  `json:"uncompressed_size"`
+	NewChunkSize   int64  `json:"new_chunk_size"`
-	NewChunkSize     int64  `json:"new_chunk_size"`
+	LocallyTracked bool   `json:"locally_tracked"`
-	LocallyTracked   bool   `json:"locally_tracked"`
+	RemotePresent  *bool  `json:"remote_present"`
-	RemotePresent    *bool  `json:"remote_present"`
 }
 
 // decodeListJSON parses the command's stdout, which must contain

internal/vaultik/storage_bind.go

@@ -45,9 +45,17 @@ func (v *Vaultik) EnsureStorageBinding() error {
 		return nil
 	}
 
-	stored, configured, err := v.storageBinding()
+	configured := v.Config.StorageURL
+	if configured == "" {
+		// The s3.* form: bind on its endpoint, bucket and prefix,
+		// written the way storage_url would name the same destination.
+		s3 := v.Config.S3
+		configured = "s3://" + s3.Bucket + "/" + s3.Prefix + "?endpoint=" + s3.Endpoint
+	}
+
+	stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
 	if err != nil {
-		return err
+		return fmt.Errorf("reading local storage binding: %w", err)
 	}
 
 	if stored == "" {
@@ -70,27 +78,6 @@ func (v *Vaultik) EnsureStorageBinding() error {
 		errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
 }
 
-// storageBinding returns the destination the local index is bound to
-// ("" when no binding is recorded yet), then the configured destination
-// written the same way. It records nothing, so a read-only command can
-// call it.
-func (v *Vaultik) storageBinding() (string, string, error) {
-	configured := v.Config.StorageURL
-	if configured == "" {
-		// The s3.* form: bind on its endpoint, bucket and prefix,
-		// written the way storage_url would name the same destination.
-		s3 := v.Config.S3
-		configured = "s3://" + s3.Bucket + "/" + s3.Prefix + "?endpoint=" + s3.Endpoint
-	}
-
-	stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
-	if err != nil {
-		return "", "", fmt.Errorf("reading local storage binding: %w", err)
-	}
-
-	return stored, configured, nil
-}
-
 // buildBindingMismatchMessage assembles the multi-line explanation
 // shown when the local index is bound to a different destination than
 // the currently-configured one (the first line lives in the

internal/vaultik/symlinked_destination_test.go

@@ -1,147 +0,0 @@
-package vaultik_test
-
-import (
-	"context"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// TestPruneDeletesNothingThroughSymlinkedDirectory checks that prune keeps
-// every blob and the local snapshot record when the metadata/ or blobs/
-// directory of a file:// destination is a symlink to a directory elsewhere
-// (https://git.eeqj.de/sneak/vaultik/issues/296). A backup writes through
-// such a link, so prune has to list through it too.
-func TestPruneDeletesNothingThroughSymlinkedDirectory(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	for _, dir := range []string{"metadata", "blobs"} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			ctx := context.Background()
-			storeDir := filepath.Join(t.TempDir(), "store")
-			v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
-			blobFiles := blobFilesIn(t, storeDir)
-			moveBehindSymlink(t, storeDir, dir)
-
-			require.NoError(t, v.Prune(&vaultik.PruneOptions{Force: true}))
-
-			// When blobs/ is the link, these paths run through it.
-			for _, blobFile := range blobFiles {
-				assert.FileExists(t, blobFile, "prune must delete no blob")
-			}
-
-			snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
-			require.NoError(t, err)
-			assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
-		})
-	}
-}
-
-// TestPruneFailsOnSymlinkedDirectoryWithMissingTarget checks that prune
-// fails, deleting no blob and keeping the local snapshot record, when the
-// metadata/ or blobs/ directory of a file:// destination is a symlink
-// whose target is missing, as a link to a volume that is not mounted is.
-func TestPruneFailsOnSymlinkedDirectoryWithMissingTarget(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	for _, dir := range []string{"metadata", "blobs"} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			ctx := context.Background()
-			storeDir := filepath.Join(t.TempDir(), "store")
-			v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
-			blobFiles := blobFilesIn(t, storeDir)
-			targetPath := moveBehindSymlink(t, storeDir, dir)
-
-			// Unmount the volume the link points to, and mount it again
-			// once prune has run.
-			unmountedPath := targetPath + "-unmounted"
-			require.NoError(t, os.Rename(targetPath, unmountedPath))
-
-			err := v.Prune(&vaultik.PruneOptions{Force: true})
-			require.ErrorIs(t, err, fs.ErrNotExist)
-
-			require.NoError(t, os.Rename(unmountedPath, targetPath))
-
-			// When blobs/ is the link, these paths run through it.
-			for _, blobFile := range blobFiles {
-				assert.FileExists(t, blobFile, "prune must delete no blob")
-			}
-
-			snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
-			require.NoError(t, err)
-			assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
-		})
-	}
-}
-
-// TestRemoveSnapshotWarnsOnSymlinkedMetadataWithMissingTarget checks that
-// snapshot remove warns that the metadata could not be removed from the
-// destination when metadata/ is a symlink whose target is missing, as it
-// does when the destination directory is missing, instead of finding no
-// metadata to remove.
-func TestRemoveSnapshotWarnsOnSymlinkedMetadataWithMissingTarget(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, repos, out := backUpToFileDestination(ctx, t, storeDir)
-	targetPath := moveBehindSymlink(t, storeDir, "metadata")
-	require.NoError(t, os.Rename(targetPath, targetPath+"-unmounted"))
-
-	result, err := v.RemoveSnapshot(localSnapshotID(ctx, t, repos, "first"),
-		&vaultik.RemoveOptions{Force: true})
-	require.NoError(t, err)
-
-	assert.False(t, result.RemoteRemoved)
-	assert.Contains(t, out.String(),
-		"Could not remove snapshot metadata from remote")
-	assert.NotContains(t, out.String(),
-		"Removed snapshot metadata from remote storage")
-}
-
-// blobFilesIn returns the paths of the blob files under storeDir/blobs, and
-// fails the test when there are none.
-func blobFilesIn(t *testing.T, storeDir string) []string {
-	t.Helper()
-
-	var blobFiles []string
-
-	err := filepath.WalkDir(filepath.Join(storeDir, "blobs"),
-		func(path string, entry fs.DirEntry, err error) error {
-			if err == nil && !entry.IsDir() {
-				blobFiles = append(blobFiles, path)
-			}
-
-			return err
-		})
-	require.NoError(t, err)
-	require.NotEmpty(t, blobFiles)
-
-	return blobFiles
-}
-
-// moveBehindSymlink moves storeDir/dir to a directory elsewhere, puts a
-// symlink to it in its place, and returns where it went.
-func moveBehindSymlink(t *testing.T, storeDir, dir string) string {
-	t.Helper()
-
-	linkPath := filepath.Join(storeDir, dir)
-	targetPath := filepath.Join(t.TempDir(), dir)
-	require.NoError(t, os.Rename(linkPath, targetPath))
-	require.NoError(t, os.Symlink(targetPath, linkPath))
-
-	return targetPath
-}

internal/vaultik/uncompressed_size_test.go

@@ -1,85 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// TestUncompressedSizeOfChangedFileStaysFixed covers
-// https://git.eeqj.de/sneak/vaultik/issues/360. A backup that changes a
-// file leaves the earlier snapshot's UNCOMPRESSED SIZE as it was when that
-// snapshot's backup completed. The file's row in the local index is shared
-// by both snapshots and rewritten by the second backup.
-func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-	changedPath := filepath.Join(dataDir, "changed.bin")
-
-	ctx := context.Background()
-	files := writeFaultSourceTree(t, fs, dataDir)
-	cfg := changedFileConfig(dataDir, dbPath)
-
-	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
-
-	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	repos := database.NewRepositories(db)
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-
-	require.NoError(t, backUp(v, "first"))
-
-	// With nothing to compare against, sizeOfNewChunks totals every
-	// distinct chunk of the files.
-	firstID := localSnapshotID(ctx, t, repos, "first")
-	firstSize := sizeOfNewChunks(t, files, nil)
-	require.Equal(t, firstSize, listedUncompressedSizes(t, v)[firstID],
-		"the first snapshot must count the chunks of every file it holds")
-
-	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
-	require.NoError(t, backUp(v, "second"))
-
-	secondID := localSnapshotID(ctx, t, repos, "second")
-	sizes := listedUncompressedSizes(t, v)
-	assert.Equal(t, firstSize, sizes[firstID],
-		"a later backup must not change the first snapshot's figure")
-	assert.Equal(t, sizeOfNewChunks(t, files, nil), sizes[secondID],
-		"the second snapshot must count the changed file's chunks")
-}
-
-// listedUncompressedSizes runs `snapshot list --json` and returns each
-// listed snapshot's uncompressed_size by snapshot ID.
-func listedUncompressedSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
-	t.Helper()
-
-	var out bytes.Buffer
-
-	v.Stdout = &out
-	require.NoError(t, v.ListSnapshots(true))
-
-	sizes := make(map[string]int64)
-	for _, row := range decodeListJSON(t, out.String()) {
-		sizes[row.ID] = row.UncompressedSize
-	}
-
-	return sizes
-}

internal/vaultik/verify.go

@@ -17,7 +17,6 @@ import (
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
-	"sneak.berlin/go/vaultik/internal/storage"
 )
 
 // Sentinel errors for snapshot verification failures.
@@ -33,9 +32,7 @@ var (
 	errManifestExtraBlob   = errors.New("manifest contains blob not in database")
 	errManifestMissingBlob = errors.New(
 		"manifest omits blob present in database")
-	errBlobSizeMismatch  = errors.New("blob size mismatch")
+	errBlobSizeMismatch = errors.New("blob size mismatch")
-	errBlobsNotInStorage = errors.New(
-		"blobs missing from storage or stored with the wrong size")
 )
 
 // verifyStatusFailed is the JSON status value for a failed verification.
@@ -152,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
 	}
 
 	result.Status = "ok"
+	result.Verified = len(dbBlobs)
 
 	if opts.JSON {
 		return v.outputVerifyJSON(result)
@@ -262,8 +260,6 @@ func (v *Vaultik) downloadVerifiedSnapshotDB(
 			fmt.Errorf("failed to download database: %w", err))
 	}
 
-	dbReader = newCloseOnCancelReader(v.ctx, dbReader)
-
 	defer func() { _ = dbReader.Close() }()
 
 	tdb, err := v.decryptAndLoadDatabase(dbReader, identities)
@@ -308,7 +304,7 @@ func (v *Vaultik) runVerificationSteps(
 		v.stdoutf("Checking blob existence in remote storage...\n")
 	}
 
-	err = v.verifyBlobExistenceFromDB(dbBlobs, result)
+	err = v.verifyBlobExistenceFromDB(dbBlobs)
 	if err != nil {
 		return v.deepVerifyFailure(result, opts, err.Error(), err)
 	}
@@ -319,8 +315,7 @@ func (v *Vaultik) runVerificationSteps(
 			len(dbBlobs), ubytes(totalSize))
 	}
 
-	err = v.performDeepVerificationFromDB(
+	err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identities)
-		dbBlobs, tdb.db.Conn(), opts, result, identities)
 	if err != nil {
 		return v.deepVerifyFailure(result, opts, err.Error(), err)
 	}
@@ -672,15 +667,9 @@ func (v *Vaultik) verifyManifestAgainstDatabase(
 	return nil
 }
 
-// verifyBlobExistenceFromDB checks that every blob from the database is
+// verifyBlobExistenceFromDB checks that all blobs from database exist in S3
-// in storage with the size the database records. It checks them all,
+func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
-// counting the missing and wrong-size blobs in result, and fails if it
+	log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
-// found any. Any storage error other than not found stops the check and
-// fails with that error.
-func (v *Vaultik) verifyBlobExistenceFromDB(
-	blobs []snapshot.BlobInfo, result *VerifyResult,
-) error {
-	log.Info("Verifying blob existence in storage", "blob_count", len(blobs))
 
 	for i, blob := range blobs {
 		// The hash is read from the snapshot database, which is not
@@ -692,23 +681,17 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
 		// Construct blob path
 		blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
 
+		// Check blob exists
 		stat, err := v.Storage.Stat(v.ctx, blobPath)
-		switch {
+		if err != nil {
-		case errors.Is(err, storage.ErrNotFound):
+			return fmt.Errorf("blob %s missing from storage: %w", blob.Hash, err)
-			log.Warn("Blob missing from storage", "hash", blob.Hash)
+		}
 
-			result.Missing++
+		// Verify size matches
-			result.MissingSize += blob.CompressedSize
+		if stat.Size != blob.CompressedSize {
-		case err != nil:
+			return fmt.Errorf(
-			return fmt.Errorf("could not check blob %s in storage: %w",
+				"%w: blob %s: S3 has %d bytes, database has %d bytes",
-				blob.Hash, err)
+				errBlobSizeMismatch, blob.Hash, stat.Size, blob.CompressedSize)
-		case stat.Size != blob.CompressedSize:
-			log.Warn("Blob has the wrong size in storage",
-				"hash", blob.Hash,
-				"storage_size", stat.Size,
-				"database_size", blob.CompressedSize)
-
-			result.Mismatched++
 		}
 
 		// Progress update every 100 blobs
@@ -722,22 +705,16 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
 		}
 	}
 
-	if result.Missing > 0 || result.Mismatched > 0 {
-		return fmt.Errorf("%w: %d missing, %d with the wrong size",
-			errBlobsNotInStorage, result.Missing, result.Mismatched)
-	}
-
 	log.Info("✓ All blobs exist in storage")
 
 	return nil
 }
 
 // performDeepVerificationFromDB downloads and verifies the content of
-// each blob using the database as source, stopping at the first blob that
+// each blob using the database as source.
-// fails. Each blob that passes is counted in result.Verified.
 func (v *Vaultik) performDeepVerificationFromDB(
 	blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
-	result *VerifyResult, identities []age.Identity,
+	identities []age.Identity,
 ) error {
 	// Calculate total bytes for ETA
 	var totalBytesExpected int64
@@ -760,7 +737,6 @@ func (v *Vaultik) performDeepVerificationFromDB(
 			return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
 		}
 
-		result.Verified++
 		bytesProcessed += blobInfo.CompressedSize
 		elapsed := time.Since(startTime)
 		remaining := len(blobs) - (i + 1)