Count a file as deleted only if the previous snapshot of its name had it (closes #304)

A backup counted as deleted every file in the local index under the
source path that the walk did not find. That took in files the
snapshot excludes but another snapshot backs up, and files removed long
ago, which every later backup, and the first backup of another name
over the same path, reported as deleted again.

The backup now counts only the files that the most recent completed
snapshot of the same name lists, the walk did not find, and are gone
from disk. A file that is excluded, or inside an excluded directory, is
not counted, nor is one the walk could not reach under --skip-errors.

The snapshots row stores no deleted figure; only the backup's output
changes.

Model: opus-5-5

ARCHITECTURE.md

@@ -279,8 +279,8 @@ Key methods:
 - `Scan(ctx, path, snapshotID)` → Main scan operation
 
 Scan phases:
-1. **Phase 0**: Detect deleted files from previous snapshots
+1. **Phase 1**: Walk directory, identify files needing processing
-2. **Phase 1**: Walk directory, identify files needing processing
+2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists, the walk did not find, and are gone from disk; a file that is excluded, or inside an excluded directory, is not counted
 3. **Phase 2**: Process files (chunk → pack → upload)
 
 #### SnapshotManager
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
     │       │
     │       ├─► scanner.Scan(ctx, path, snapshotID)
     │       │       │
-    │       │       ├─► Phase 0: detectDeletedFiles()
-    │       │       │
     │       │       ├─► Phase 1: scanPhase()
     │       │       │       Walk directory
     │       │       │       Check file metadata changes
     │       │       │       Build list of files to process
     │       │       │
+    │       │       ├─► Phase 1b: detectDeletedFiles()
+    │       │       │
     │       │       └─► Phase 2: processPhase()
     │       │               For each file:
     │       │                   chunker.ChunkReaderStreaming()

README.md

@@ -712,8 +712,10 @@ the `NO_COLOR` environment variable is unset (https://no-color.org/).
   reports and tables (`version`, `info`, `remote info`, `snapshot
   verify`, `snapshot list`) but never the machine-consumed `config get`
   value or the `--json` documents, which a script depends on. The
-  `database delete` confirmation prompt is also written this way and
+  confirmation prompts of `database delete`, `prune`, `snapshot purge`
-  always shown: it is an interactive exchange the operator must see.
+  and `snapshot remove`, each with what it asks to delete, are also
+  written this way and always shown: they are an interactive exchange
+  the operator must see.
 
 `internal/ui` writes to stdout; it is the output the user asked for. The
 exceptions are the startup banner and the error a failed command ends

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/spf13/cobra v1.10.1
 	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1
+	go.uber.org/dig v1.19.0
 	go.uber.org/fx v1.24.0
 	golang.org/x/sync v0.18.0
 	golang.org/x/sys v0.38.0
@@ -256,7 +257,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
 	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
-	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect

internal/cli/app.go

@@ -8,12 +8,12 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
 	"github.com/adrg/xdg"
 	"github.com/spf13/cobra"
+	"go.uber.org/dig"
 	"go.uber.org/fx"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
@@ -146,16 +146,12 @@ func (e *startupError) Error() string {
 //	could not build arguments for function "Y" (file:line): failed to build U:
 //	received non-nil error from function "Z" (file:line): <real error>
 //
-// Users care about the real error, not the DI plumbing. We strip everything
+// Users care about the real error, not the DI plumbing. dig.RootCause,
-// up through the last "): " (which is always the close-paren of an fx
+// from the library fx is built on, unwraps those layers and returns the
-// function-location annotation followed by the wrapped error).
+// real error whole. Cutting the text at "): " instead would also cut a
+// real error that contains it, such as the blob_size_limit one.
 func cleanStartupError(err error) error {
-	msg := err.Error()
+	return &startupError{msg: dig.RootCause(err).Error()}
-	if idx := strings.LastIndex(msg, "): "); idx >= 0 {
-		msg = msg[idx+3:]
-	}
-
-	return &startupError{msg: msg}
 }
 
 // RunApp starts the fx application, blocks until it is asked to stop, and

internal/cli/app_test.go

@@ -2,58 +2,65 @@ package cli //nolint:testpackage // needs access to unexported cleanStartupError
 
 import (
 	"errors"
+	"os"
 	"path/filepath"
 	"testing"
 
+	"go.uber.org/fx"
+	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/pidlock"
 )
 
+// TestCleanStartupError starts a real fx app whose config has a
+// blob_size_limit below the minimum. fx puts annotations naming the
+// functions it was building in front of the config error; only those
+// may be removed, although the config error itself contains "): ".
 func TestCleanStartupError(t *testing.T) {
 	t.Parallel()
 
-	tests := []struct {
+	path := filepath.Join(t.TempDir(), "config.yml")
-		name string
+
-		in   string
+	err := os.WriteFile(path, []byte("snapshots:\n"+
-		want string
+		"  test:\n"+
-	}{
+		"    paths: [/tmp/vaultik-test-source]\n"+
-		{
+		"storage_url: file:///tmp/vaultik-test-storage\n"+
-			name: "real fx error chain",
+		"chunk_size: 10MB\n"+
-			in: `could not build arguments for function ` +
+		"blob_size_limit: 30MB\n"), 0o600)
-				`"sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 ` +
+	if err != nil {
-				`(/Users/user/dev/vaultik/internal/cli/snapshot.go:71): ` +
+		t.Fatalf("write config: %v", err)
-				`failed to build *vaultik.Vaultik: ` +
-				`could not build arguments for function ` +
-				`"sneak.berlin/go/vaultik/internal/vaultik".New ` +
-				`(/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): ` +
-				`failed to build storage.Storer: ` +
-				`received non-nil error from function ` +
-				`"sneak.berlin/go/vaultik/internal/storage".NewStorer ` +
-				`(/Users/user/dev/vaultik/internal/storage/module.go:23): ` +
-				`creating base path: mkdir /Volumes/BACKUPS: permission denied`,
-			want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`,
-		},
-		{
-			name: "no fx wrapping",
-			in:   "plain error",
-			want: "plain error",
-		},
-		{
-			name: "single fx wrapping",
-			in:   `received non-nil error from function "foo" (file.go:1): underlying problem`,
-			want: "underlying problem",
-		},
 	}
 
-	for _, tt := range tests {
+	app := fx.New(
-		t.Run(tt.name, func(t *testing.T) {
+		fx.NopLogger,
+		fx.Supply(config.Path(path)),
+		config.Module,
+		fx.Invoke(func(*config.Config) {}),
+	)
+
+	err = app.Start(t.Context())
+	if err == nil {
+		t.Fatal("app started; want a blob_size_limit error")
+	}
+
+	want := "failed to load config: invalid config: blob_size_limit must " +
+		"be at least the largest chunk the chunker can emit (chunk_size " +
+		"times the FastCDC size spread): need at least 40000000 bytes"
+
+	got := cleanStartupError(err).Error()
+	if got != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}
+
+// TestCleanStartupErrorNotFromFx checks that an error fx did not wrap is
+// shown unchanged.
+func TestCleanStartupErrorNotFromFx(t *testing.T) {
 	t.Parallel()
 
-			//nolint:err113 // test constructs errors from table input
+	//nolint:err113 // a one-off error standing in for a startup failure
-			got := cleanStartupError(errors.New(tt.in)).Error()
+	got := cleanStartupError(errors.New("plain error")).Error()
-			if got != tt.want {
+	if got != "plain error" {
-				t.Errorf("got %q, want %q", got, tt.want)
+		t.Errorf("got %q, want %q", got, "plain error")
-			}
-		})
 	}
 }
 

internal/cli/entry_prune_quiet_test.go

@@ -0,0 +1,123 @@
+package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// flagCron is used to build the argument vectors below.
+const flagCron = "--cron"
+
+// plantUnreferencedBlob writes a blob that no snapshot references to the
+// destination store at storeDir, so that a prune deletes it. Returns its
+// path.
+func plantUnreferencedBlob(t *testing.T, storeDir string) string {
+	t.Helper()
+
+	hash := strings.Repeat("b", 64)
+	path := filepath.Join(storeDir, "blobs", hash[:2], hash[2:4], hash)
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750))
+	require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600))
+
+	return path
+}
+
+// TestEntrySnapshotCreatePruneIsSilent checks that a successful
+// `snapshot create --prune` under --cron or --quiet prints nothing on
+// stdout or stderr, whether or not the prune finds a blob to delete.
+//
+//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
+func TestEntrySnapshotCreatePruneIsSilent(t *testing.T) {
+	for _, testCase := range []struct {
+		name      string
+		args      []string
+		plantBlob bool
+	}{
+		{
+			name: "cron, no blob to delete",
+			args: []string{cmdSnapshot, cmdCreate, flagCron, flagPrune},
+		},
+		{
+			name:      "cron, blob to delete",
+			args:      []string{cmdSnapshot, cmdCreate, flagCron, flagPrune},
+			plantBlob: true,
+		},
+		{
+			name: "cron with keep-newer-than",
+			args: []string{
+				cmdSnapshot, cmdCreate, flagCron, flagPrune,
+				flagKeepNewerThan, "4w",
+			},
+			plantBlob: true,
+		},
+		{
+			name:      "quiet",
+			args:      []string{flagQuiet, cmdSnapshot, cmdCreate, flagPrune},
+			plantBlob: true,
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			configPath, storeDir := writeBackupConfig(t)
+
+			var blob string
+			if testCase.plantBlob {
+				blob = plantUnreferencedBlob(t, storeDir)
+			}
+
+			code, stdout, stderr := runEntry(t,
+				append([]string{flagConfig, configPath}, testCase.args...)...)
+
+			assert.Equal(t, 0, code)
+			assert.Empty(t, stdout)
+			assert.Empty(t, stderr)
+
+			if testCase.plantBlob {
+				assert.NoFileExists(t, blob, "the prune must delete the blob")
+			}
+		})
+	}
+}
+
+// TestEntryPruneQuietIsSilent checks that a successful `prune --force`
+// under --quiet prints nothing on stdout or stderr, both when there is
+// nothing to remove and when it removes a stale local snapshot record
+// and an unreferenced blob.
+//
+//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
+func TestEntryPruneQuietIsSilent(t *testing.T) {
+	for _, testCase := range []struct {
+		name               string
+		staleRecordAndBlob bool
+	}{
+		{name: "nothing to remove"},
+		{name: "stale record and blob to remove", staleRecordAndBlob: true},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			configPath := writeHermeticPruneConfig(t, testCase.staleRecordAndBlob)
+
+			var blob string
+			if testCase.staleRecordAndBlob {
+				// writeHermeticPruneConfig puts the store beside the config.
+				blob = plantUnreferencedBlob(t,
+					filepath.Join(filepath.Dir(configPath), "store"))
+			}
+
+			code, stdout, stderr := runEntry(t,
+				flagConfig, configPath, flagQuiet, cmdPrune, flagForce)
+
+			assert.Equal(t, 0, code)
+			assert.Empty(t, stdout)
+			assert.Empty(t, stderr)
+
+			if testCase.staleRecordAndBlob {
+				assert.NoFileExists(t, blob, "the prune must delete the blob")
+			}
+		})
+	}
+}

internal/database/files.go

@@ -267,6 +267,45 @@ func (r *FileRepository) ListUnderPath(
 	return files, rows.Err()
 }
 
+// ListUnderPathInSnapshot returns the files that ListUnderPath returns
+// for path and that the snapshot lists, in no particular order.
+func (r *FileRepository) ListUnderPathInSnapshot(
+	ctx context.Context, path, snapshotID string,
+) ([]*File, error) {
+	path = strings.TrimRight(path, "/")
+	dirPrefix := path + "/"
+
+	query := `
+		SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
+			f.mode, f.uid, f.gid, f.link_target
+		FROM files f
+		JOIN snapshot_files sf ON sf.file_id = f.id
+		WHERE sf.snapshot_id = ?
+		  AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
+	`
+
+	rows, err := r.db.conn.QueryContext(ctx, query,
+		snapshotID, path, dirPrefix, dirPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("querying files: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var files []*File
+
+	for rows.Next() {
+		file, err := r.scanFileRows(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scanning file: %w", err)
+		}
+
+		files = append(files, file)
+	}
+
+	return files, rows.Err()
+}
+
 // ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
 // path starts with prefix and that list at least one chunk held by no
 // blob whose upload has completed (uploaded_ts set). A new snapshot

internal/database/files_test.go

@@ -212,6 +212,65 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
 	}
 }
 
+// ListUnderPathInSnapshot leaves out files outside path and files the
+// snapshot does not list, though another snapshot lists them.
+func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
+	t.Parallel()
+
+	db, cleanup := setupTestDB(t)
+	defer cleanup()
+
+	ctx := context.Background()
+	files := database.NewFileRepository(db)
+	snapshots := database.NewSnapshotRepository(db)
+
+	listed := map[string][]string{
+		"one": {"/src/a.txt", "/src2/c.txt"},
+		"two": {"/src/b.txt"},
+	}
+
+	for snapshotID, paths := range listed {
+		err := snapshots.Create(ctx, nil, &database.Snapshot{
+			ID:        types.SnapshotID(snapshotID),
+			Hostname:  "host",
+			StartedAt: time.Now(),
+		})
+		if err != nil {
+			t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
+		}
+
+		for _, path := range paths {
+			err := files.Create(ctx, nil, &database.File{
+				Path:  types.FilePath(path),
+				MTime: time.Now().Truncate(time.Second),
+				Mode:  0644,
+			})
+			if err != nil {
+				t.Fatalf("failed to create %s: %v", path, err)
+			}
+
+			err = snapshots.AddFile(ctx, nil, snapshotID, path)
+			if err != nil {
+				t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
+			}
+		}
+	}
+
+	found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
+	if err != nil {
+		t.Fatalf("failed to list files: %v", err)
+	}
+
+	got := make([]string, 0, len(found))
+	for _, f := range found {
+		got = append(got, f.Path.String())
+	}
+
+	if !slices.Equal(got, []string{"/src/a.txt"}) {
+		t.Errorf("got %q, want only /src/a.txt", got)
+	}
+}
+
 func TestFileRepositorySymlink(t *testing.T) {
 	t.Parallel()
 

internal/snapshot/module.go

@@ -16,6 +16,9 @@ type ScannerParams struct {
 	Fs             afero.Fs
 	Exclude        []string // Exclude patterns (combined global + snapshot-specific)
 	SkipErrors     bool     // Skip file read errors (log loudly but continue)
+	// PreviousSnapshotID is the most recent completed snapshot of the
+	// same name, "" for none.
+	PreviousSnapshotID string
 }
 
 // Module exports backup functionality as an fx module.
@@ -55,6 +58,7 @@ func provideScannerFactory(
 			UI:                 params.UI,
 			Exclude:            excludes,
 			SkipErrors:         params.SkipErrors,
+			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/snapshot/scanner.go

@@ -58,6 +58,10 @@ type Scanner struct {
 	compressionLevel int
 	ageRecipient     string
 	snapshotID       string // Current snapshot being processed
+	// previousSnapshotID is the most recent completed snapshot of the
+	// same name, "" for none. Only a file it lists can be counted as
+	// deleted.
+	previousSnapshotID string
 	// currentSourcePath is the source directory being scanned, stored with
 	// each file record.
 	currentSourcePath string
@@ -129,6 +133,9 @@ type ScannerConfig struct {
 	// SkipErrors skips files that cannot be opened or read (log loudly but
 	// continue); packer, database, encryption, and upload errors still abort.
 	SkipErrors bool
+	// PreviousSnapshotID is the most recent completed snapshot of the
+	// same name, "" for none.
+	PreviousSnapshotID string
 }
 
 // ScanResult contains the results of a scan operation. Files and bytes
@@ -207,6 +214,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		compiledExclude:    compiledExclude,
 		progress:           progress,
 		skipErrors:         cfg.SkipErrors,
+		previousSnapshotID: cfg.PreviousSnapshotID,
 		ui:                 uiw,
 		pendingChunkHashes: make(map[string]struct{}),
 	}
@@ -258,8 +266,9 @@ func (s *Scanner) Scan(
 
 	filesToProcess := scanResult.FilesToProcess
 
-	// Phase 1b: Detect deleted files by comparing DB against scanned files
+	// Phase 1b: Detect deleted files by comparing the previous snapshot
-	err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
+	// of this name against scanned files
+	err = s.detectDeletedFiles(ctx, path, existingFiles, result)
 	if err != nil {
 		return nil, fmt.Errorf("detecting deleted files: %w", err)
 	}
@@ -439,9 +448,8 @@ func (s *Scanner) summarizeScanPhase(
 }
 
 // loadKnownFiles loads the known files at and beneath path from the
-// database into a map for fast lookup. Every loaded file the scan does
+// database into a map for fast lookup. This avoids per-file database
-// not find is counted as deleted. This avoids per-file database queries
+// queries during the scan phase.
-// during the scan phase.
 func (s *Scanner) loadKnownFiles(
 	ctx context.Context, path string,
 ) (map[string]*database.File, error) {
@@ -1954,21 +1962,29 @@ func (s *Scanner) queueFileForBatchInsert(
 	})
 }
 
-// detectDeletedFilesFromMap finds files that existed in previous snapshots
+// detectDeletedFiles counts as deleted each file at or beneath path that
-// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
+// the previous snapshot of this name lists, the walk did not find, and
-// database access.
+// is gone from disk. A file that is excluded, or inside an excluded
-func (s *Scanner) detectDeletedFilesFromMap(
+// directory, is not counted: the walk skips it whether or not it is
+// still on disk.
+func (s *Scanner) detectDeletedFiles(
 	ctx context.Context,
-	knownFiles map[string]*database.File,
+	path string,
 	existingFiles map[string]struct{},
 	result *ScanResult,
 ) error {
-	if len(knownFiles) == 0 {
+	if s.previousSnapshotID == "" {
 		return nil
 	}
 
-	// Check each known file against the enumerated set (no filesystem access needed)
+	previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
-	for path, file := range knownFiles {
+		ctx, path, s.previousSnapshotID)
+	if err != nil {
+		return fmt.Errorf("listing files of snapshot %s: %w",
+			s.previousSnapshotID, err)
+	}
+
+	for _, file := range previousFiles {
 		// Check context cancellation periodically
 		select {
 		case <-ctx.Done():
@@ -1976,13 +1992,22 @@ func (s *Scanner) detectDeletedFilesFromMap(
 		default:
 		}
 
-		// Check if the file exists in our enumerated set
+		filePath := file.Path.String()
-		if _, exists := existingFiles[path]; !exists {
+
-			// File has been deleted
+		_, exists := existingFiles[filePath]
+		if exists || s.excludedOrInExcludedDir(filePath, path) {
+			continue
+		}
+
+		// Under --skip-errors the walk also misses what it cannot reach,
+		// such as the files in a directory it cannot open.
+		if !s.goneFromDisk(filePath) {
+			continue
+		}
+
 		result.FilesDeleted++
 		result.BytesDeleted += file.Size
-			log.Debug("Detected deleted file", "path", path, "size", file.Size)
+		log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
-		}
 	}
 
 	if result.FilesDeleted > 0 {
@@ -1993,6 +2018,34 @@ func (s *Scanner) detectDeletedFilesFromMap(
 	return nil
 }
 
+// excludedOrInExcludedDir reports whether filePath, which is at or
+// beneath rootPath, or any directory between rootPath and it, matches an
+// exclude pattern. A pattern such as */.local/share/Trash matches the
+// directory but not the files inside it, which the walk skips too.
+func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
+	for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
+		if s.shouldExclude(p, rootPath) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// goneFromDisk reports whether looking filePath up on disk finds that
+// it does not exist. Like the walk, it does not follow a symlink, so a
+// symlink whose target is gone is still on disk.
+func (s *Scanner) goneFromDisk(filePath string) bool {
+	var err error
+	if lstater, ok := s.fs.(afero.Lstater); ok {
+		_, _, err = lstater.LstatIfPossible(filePath)
+	} else {
+		_, err = s.fs.Stat(filePath)
+	}
+
+	return errors.Is(err, os.ErrNotExist)
+}
+
 // wrapPermissionError augments permission errors with platform-specific
 // remediation instructions. On macOS, TCC-protected directories (Calendars,
 // Reminders, Photos, etc.) return EPERM unless the running application has

internal/vaultik/blobcache.go

@@ -39,9 +39,9 @@ type blobDiskCacheEntry struct {
 // internally and will fall back to LRU eviction if curBytes exceeds
 // maxBytes. Restore passes math.MaxInt64 as maxBytes and drives
 // eviction itself via Delete() through restoreSweeper, which deletes
-// each blob the moment every file that references its chunks has been
+// each blob once no file still to be restored in this run needs it.
-// written. LRU never fires under that configuration; it is kept as a
+// LRU never fires under that configuration; it is kept as a safety net
-// safety net for callers that don't manage eviction themselves.
+// for callers that don't manage eviction themselves.
 //
 // Get/ReadAt/peak-Len counters are debugging instrumentation used by
 // tests to assert that the restore code path uses ReadAt rather than

internal/vaultik/fault_injection_test.go

@@ -672,6 +672,7 @@ func faultScannerFactory(
 			UI:                 params.UI,
 			Exclude:            params.Exclude,
 			SkipErrors:         params.SkipErrors,
+			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/vaultik/prune.go

@@ -204,7 +204,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 			return v.outputPruneBlobsJSON(result)
 		}
 
-		v.printlnStdout("No unreferenced blobs to remove.")
+		v.UI.Infof("No unreferenced blobs to remove.")
 
 		return nil
 	}
@@ -212,12 +212,17 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 	log.Info("Found unreferenced blobs",
 		"count", len(unreferencedBlobs), "total_size", ubytes(totalSize))
 
-	if !opts.JSON {
+	if opts.Force && !opts.JSON {
-		v.stdoutf("Found %d unreferenced blob(s) totaling %s\n",
+		v.UI.Infof("Found %d unreferenced blob(s) totaling %s.",
 			len(unreferencedBlobs), ubytes(totalSize))
 	}
 
 	if !opts.Force && !opts.JSON {
+		// The count, the prompt and its answer bypass the UI: they are an
+		// interactive exchange the operator must see, so --quiet does not
+		// silence them.
+		v.stdoutf("Found %d unreferenced blob(s) totaling %s.\n",
+			len(unreferencedBlobs), ubytes(totalSize))
 		v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
 
 		var confirm string
@@ -244,7 +249,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
 			return err
 		}
 	} else {
-		v.stdoutf("\nDeleted %d blob(s) totaling %s\n",
+		v.UI.Infof("Deleted %d blob(s) totaling %s.",
 			result.BlobsDeleted, ubytes(result.BytesFreed))
 	}
 

internal/vaultik/purge_local_remote_consistency_test.go

@@ -14,6 +14,7 @@ import (
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/types"
+	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
 
@@ -58,13 +59,16 @@ func setupConsistencyTest(
 		require.NoError(t, err)
 	}
 
+	stdout := &bytes.Buffer{}
+
 	v := &vaultik.Vaultik{
 		Storage:      mockStorage,
 		Repositories: repos,
 		DB:           db,
-		Stdout:       &bytes.Buffer{},
+		Stdout:       stdout,
 		Stderr:       &bytes.Buffer{},
 		Stdin:        &bytes.Buffer{},
+		UI:           ui.NewWithColor(stdout, false),
 	}
 	v.SetContext(ctx)
 

internal/vaultik/purge_per_name_test.go

@@ -14,6 +14,7 @@ import (
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/types"
+	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
 
@@ -86,6 +87,7 @@ func setupPurgeTest(
 		Stdout:       stdout,
 		Stderr:       stderr,
 		Stdin:        stdin,
+		UI:           ui.NewWithColor(stdout, false),
 	}
 	v.SetContext(ctx)
 

internal/vaultik/remove_quiet_test.go

@@ -0,0 +1,148 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestPurgeSnapshots_QuietSuppressesReport checks that `snapshot purge
+// --keep-latest --force` under --quiet deletes the older snapshot and
+// prints nothing.
+func TestPurgeSnapshots_QuietSuppressesReport(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	v := setupPurgeTest(t, snapHostname, []string{snapHomeT0, snapHomeT1})
+
+	var out bytes.Buffer
+
+	v.Stdout = &out
+	v.UI = ui.NewWithColor(&out, false)
+	v.UI.SetQuiet(true)
+
+	require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
+		KeepLatest: true,
+		Force:      true,
+	}))
+
+	assert.Equal(t, []string{snapHomeT1}, listRemainingSnapshots(t, v))
+	assert.Empty(t, out.String())
+}
+
+// TestPurgeSnapshots_QuietPromptListsSnapshots checks that `snapshot
+// purge --keep-latest` without --force still lists the snapshots its
+// prompt asks to delete under --quiet.
+func TestPurgeSnapshots_QuietPromptListsSnapshots(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	v := setupPurgeTest(t, snapHostname, []string{snapHomeT0, snapHomeT1})
+
+	var out bytes.Buffer
+
+	v.Stdout = &out
+	v.UI = ui.NewWithColor(&out, false)
+	v.UI.SetQuiet(true)
+
+	// Stdin is empty, so the prompt reads no answer and cancels.
+	require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
+		KeepLatest: true,
+	}))
+
+	assert.Contains(t, out.String(), snapHomeT0)
+	assert.Contains(t, out.String(), "Delete 1 snapshot(s)? [y/N]")
+	assert.Len(t, listRemainingSnapshots(t, v), 2)
+}
+
+// TestPruneBlobs_QuietPromptShowsBlobTotal checks that `prune` without
+// --force still states the count and total size of the blobs its prompt
+// asks to delete under --quiet.
+func TestPruneBlobs_QuietPromptShowsBlobTotal(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newListEnv(t)
+	env.v.UI.SetQuiet(true)
+
+	// The store holds no manifest, so nothing references this blob.
+	addBlob(t, env.store.testStorer, testBlobHashA)
+
+	// Stdin is empty, so the prompt reads no answer and cancels.
+	require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
+
+	assert.Contains(t, env.stdout.String(),
+		"Found 1 unreferenced blob(s) totaling")
+	assert.Contains(t, env.stdout.String(),
+		"Delete 1 unreferenced blob(s)? [y/N]")
+}
+
+// TestRemoveSnapshot_QuietSuppressesReport checks that `snapshot remove`
+// under --quiet prints nothing with --dry-run or with --force, and that
+// the --json document still prints.
+func TestRemoveSnapshot_QuietSuppressesReport(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	newQuietRemover := func() (*vaultik.TestVaultik, *testStorer) {
+		store := newTestStorer()
+		addManifest(t, store, "snapshot-001", nil)
+
+		tv := vaultik.NewForTesting(store)
+		tv.Repositories = newLocalIndex(t, "snapshot-001")
+		tv.UI.SetQuiet(true)
+
+		return tv, store
+	}
+
+	tv, store := newQuietRemover()
+
+	_, err := tv.RemoveSnapshot("snapshot-001",
+		&vaultik.RemoveOptions{DryRun: true})
+	require.NoError(t, err)
+	assert.Empty(t, tv.Stdout.String(),
+		"--dry-run must print nothing under --quiet")
+
+	_, err = tv.RemoveSnapshot("snapshot-001",
+		&vaultik.RemoveOptions{Force: true})
+	require.NoError(t, err)
+	assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
+	assert.Empty(t, tv.Stdout.String(),
+		"--force must print nothing under --quiet")
+
+	tv, _ = newQuietRemover()
+
+	_, err = tv.RemoveSnapshot("snapshot-001",
+		&vaultik.RemoveOptions{JSON: true})
+	require.NoError(t, err)
+
+	var result vaultik.RemoveResult
+
+	require.NoError(t, json.Unmarshal(tv.Stdout.Bytes(), &result),
+		"the --json document must still print under --quiet")
+	assert.True(t, result.RemoteRemoved)
+}
+
+// TestNukeRemote_QuietSuppressesReport checks that `remote nuke --force`
+// under --quiet prints nothing while it removes a snapshot's metadata
+// and its blobs.
+func TestNukeRemote_QuietSuppressesReport(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := filepath.Join(t.TempDir(), "store")
+	v, _, out := backUpToFileDestination(ctx, t, storeDir)
+	v.UI.SetQuiet(true)
+
+	require.NoError(t, v.NukeRemote(true))
+	assert.Empty(t, out.String())
+}

internal/vaultik/remove_snapshot_test.go

@@ -350,7 +350,7 @@ func TestRemoveSnapshot_DryRun(t *testing.T) {
 	assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
 	assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
 
-	assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
+	assert.Contains(t, tv.Stdout.String(), "Dry run: no changes made.")
 }
 
 // TestRemoveSnapshot_UnknownIDIsError checks that an ID naming no
@@ -504,7 +504,7 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) {
 
 	assert.Equal(t, initialCount, store.keyCount())
 
-	assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
+	assert.Contains(t, tv.Stdout.String(), "Dry run: no changes made.")
 }
 
 func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) {

internal/vaultik/restore.go

@@ -323,12 +323,6 @@ func (v *Vaultik) restoreAllFiles(
 		_ = blobCache.Close()
 	}()
 
-	// Per-restore sweep state: every blob_size_limit/100 bytes written,
-	// scan the cache and delete any blob whose remaining file references
-	// are all already restored.
-	sweeper := newRestoreSweeper(v.ctx, repos, blobCache,
-		v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor)
-
 	blobByHash, blobIDToHash, err := v.buildBlobIndexes(repos)
 	if err != nil {
 		return nil, err
@@ -340,6 +334,12 @@ func (v *Vaultik) restoreAllFiles(
 		return nil, fmt.Errorf("building restore plan: %w", err)
 	}
 
+	// Per-restore sweep state: every blob_size_limit/100 bytes written,
+	// scan the cache and delete any blob that no pending file in the
+	// plan references.
+	sweeper := newRestoreSweeper(plan, blobCache,
+		v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor)
+
 	// Ranging over files rather than the map reports them in a stable order.
 	for _, file := range files {
 		if planErr, ok := unplanned[file.ID]; ok {
@@ -427,10 +427,8 @@ func (v *Vaultik) runRestoreLoop(
 			continue
 		}
 
-		// Record the file as restored so the sweeper can free blobs
+		// Drop the file from the plan's indexes so future picks ignore
-		// once all referencing files are done, and drop it from the
+		// it and the sweeper can free blobs no pending file needs.
-		// plan's indexes so future picks ignore it.
-		session.sweeper.fileRestored(fileID.String())
 		plan.finishFile(fileID)
 
 		processed++

internal/vaultik/restore_interrupt_test.go

@@ -74,7 +74,8 @@ func TestRestoreCleansTempDirOnInterrupt(t *testing.T) {
 	buildLocalityFixture(t, fs, dataDir)
 
 	cfg, storer, snapshotID := setupLocalityBackup(
-		context.Background(), t, fs, dataDir, storeDir, dbPath)
+		context.Background(), t, fs, dataDir, storeDir, dbPath,
+		localitySourceBytes)
 
 	// Point the "" temp paths (the blob cache directory and the
 	// snapshot-database directory) at a private directory so the test can

internal/vaultik/restore_locality_test.go

@@ -63,6 +63,10 @@ type localityCopy struct {
 	data []byte
 }
 
+// localitySourceBytes is the size of each source file of the locality
+// fixture.
+const localitySourceBytes = 1024 * 1024
+
 // buildLocalityFixture writes the adversarial source layout described
 // in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the
 // backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
@@ -73,7 +77,6 @@ func buildLocalityFixture(
 	t.Helper()
 
 	const (
-		srcBytes   = 1024 * 1024
 		srcCount   = 15
 		blobsCount = 3
 		perBlob    = srcCount / blobsCount
@@ -83,7 +86,7 @@ func buildLocalityFixture(
 	for i := range srcCount {
 		s := &localitySource{
 			path: fmt.Sprintf("src-%03d.bin", i+1),
-			data: randomBytes(t, srcBytes),
+			data: randomBytes(t, localitySourceBytes),
 		}
 		sources[i] = s
 		require.NoError(t, afero.WriteFile(
@@ -115,19 +118,20 @@ func buildLocalityFixture(
 
 // setupLocalityBackup runs the backup half of the locality test: a
 // snapshot of dataDir into a file storer with sizes tuned so 15 one-chunk
-// files pack into 3 blobs. The index database is closed before returning
+// files of srcBytes pack into 3 blobs. The index database is closed
-// so the restore half runs from remote bytes only.
+// before returning so the restore half runs from remote bytes only.
 func setupLocalityBackup(
 	ctx context.Context, t *testing.T, fs afero.Fs,
-	dataDir, storeDir, dbPath string,
+	dataDir, storeDir, dbPath string, srcBytes int64,
 ) (*config.Config, *storage.FileStorer, string) {
 	t.Helper()
 
-	// chunkSize avg = 4 MiB makes minSize = 1 MiB, so a 1 MiB file
+	// An average chunk size of four times srcBytes makes the minimum
-	// becomes one chunk. maxBlobSize = 5 MiB packs exactly 5 chunks
+	// chunk srcBytes, so each source file becomes one chunk. A blob limit
-	// per blob, yielding 3 blobs from 15 source files.
+	// of five times srcBytes packs exactly 5 chunks per blob, yielding 3
-	chunkSize := int64(4 * 1024 * 1024)
+	// blobs from 15 source files.
-	maxBlobSize := int64(5 * 1024 * 1024)
+	chunkSize := 4 * srcBytes
+	maxBlobSize := 5 * srcBytes
 
 	storer, err := storage.NewFileStorer(storeDir)
 	require.NoError(t, err)
@@ -208,7 +212,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) {
 	ctx := context.Background()
 
 	cfg, storer, snapshotID := setupLocalityBackup(
-		ctx, t, fs, dataDir, storeDir, dbPath)
+		ctx, t, fs, dataDir, storeDir, dbPath, localitySourceBytes)
 
 	// Wrap the storer so we can count downloads per blob key.
 	counter := newCountingStorer(storer)

internal/vaultik/restore_plan.go

@@ -31,7 +31,8 @@ var (
 //     with no further downloads.
 //   - blobFiles: for each blob, the set of pending files referencing
 //     it. Used to short-circuit "when this blob lands, which files
-//     become ready" without a global scan.
+//     become ready" without a global scan, and by the sweeper to tell
+//     whether a cached blob is still needed.
 type restorePlan struct {
 	fileBlobs map[types.FileID]map[string]struct{}
 	blobFiles map[string]map[types.FileID]struct{}
@@ -237,6 +238,12 @@ func (p *restorePlan) filesReferencingBlob(blobHash string) []types.FileID {
 	return out
 }
 
+// pendingFileUsesBlob reports whether any pending file references the
+// named blob, whether or not the blob is cached.
+func (p *restorePlan) pendingFileUsesBlob(blobHash string) bool {
+	return len(p.blobFiles[blobHash]) > 0
+}
+
 // hasPending reports whether any unfinished files remain.
 func (p *restorePlan) hasPending() bool {
 	return len(p.fileBlobs) > 0

internal/vaultik/restore_plan_test.go

@@ -63,15 +63,6 @@ func TestRunRestoreLoopFailsOnAbandonedFiles(t *testing.T) {
 
 	t.Cleanup(func() { _ = cache.Close() })
 
-	v := &Vaultik{ctx: ctx}
-	session := &restoreSession{
-		v:       v,
-		ctx:     ctx,
-		repos:   repos,
-		sweeper: newRestoreSweeper(ctx, repos, cache, 1),
-		result:  &RestoreResult{},
-	}
-
 	// A file that is still pending but whose uncached-blob set is empty
 	// and which was never queued as ready: the loop can neither restore
 	// nor download it. This is the abandonment the guard must catch.
@@ -83,6 +74,15 @@ func TestRunRestoreLoopFailsOnAbandonedFiles(t *testing.T) {
 		cached:    map[string]struct{}{},
 	}
 
+	v := &Vaultik{ctx: ctx}
+	session := &restoreSession{
+		v:       v,
+		ctx:     ctx,
+		repos:   repos,
+		sweeper: newRestoreSweeper(plan, cache, 1),
+		result:  &RestoreResult{},
+	}
+
 	err = v.runRestoreLoop(session, plan, map[types.FileID]*database.File{}, 0)
 	require.ErrorIs(t, err, errRestoreIncomplete)
 }

internal/vaultik/restore_sweeper.go

@@ -1,46 +1,30 @@
 package vaultik
 
-import (
+// restoreSweeper frees cached blobs once no file still to be restored in
-	"context"
+// this run needs them. It works as follows:
-	"fmt"
-
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-)
-
-// restoreSweeper frees cached blobs once all files that reference any of
-// their chunks have been restored. It works as follows:
 //
-//  1. Callers add a file's ID to an in-memory restored set via
+//  1. After each chunk is restored, chunkRestored accumulates a running
-//     fileRestored once the file is fully written to disk.
-//  2. After each chunk is restored, chunkRestored accumulates a running
 //     byte count.
-//  3. When the accumulator crosses a threshold (one hundredth of the
+//  2. When the accumulator crosses a threshold (one hundredth of the
 //     configured blob size — so a sweep runs about a hundred times per
 //     blob's worth of restored bytes), the sweeper iterates every key in
-//     the cache. For each cached blob it asks the DB which files
+//     the cache and deletes each blob that the restore plan no longer
-//     reference any chunk in that blob, then compares that list against
+//     needs.
-//     the in-memory restored set. If any referencing file is missing
-//     from the set the blob is kept; otherwise the cache entry is
-//     deleted.
 //
-// All DB reads happen against the snapshot's temporary metadata DB,
+// The plan drops a file once it is restored or has failed, and holds
-// which is local, indexed, and not under contention — the queries are
+// only the files this run restores, so files outside the requested paths
-// cheap and run at most once per blob per sweep interval.
+// never keep a blob.
 type restoreSweeper struct {
-	ctx        context.Context //nolint:containedctx // ctx bound at construction by design
+	plan       *restorePlan
-	repos      *database.Repositories
 	cache      *blobDiskCache
 	threshold  int64
 	bytesAccum int64
-	restored   map[string]struct{}
 }
 
 // newRestoreSweeper returns a sweeper that triggers eviction every
 // `threshold` bytes restored. Callers should pass blob_size_limit/100.
 func newRestoreSweeper(
-	ctx context.Context,
+	plan *restorePlan,
-	repos *database.Repositories,
 	cache *blobDiskCache,
 	threshold int64,
 ) *restoreSweeper {
@@ -49,21 +33,12 @@ func newRestoreSweeper(
 	}
 
 	return &restoreSweeper{
-		ctx:       ctx,
+		plan:      plan,
-		repos:     repos,
 		cache:     cache,
 		threshold: threshold,
-		restored:  make(map[string]struct{}),
 	}
 }
 
-// fileRestored records a file as fully restored. After this call, any
-// blob whose only remaining references come from files in the restored
-// set may be evicted on the next sweep.
-func (s *restoreSweeper) fileRestored(fileID string) {
-	s.restored[fileID] = struct{}{}
-}
-
 // chunkRestored accounts n bytes against the sweep threshold and runs a
 // sweep if the threshold has been crossed since the last sweep.
 func (s *restoreSweeper) chunkRestored(n int64) {
@@ -76,60 +51,11 @@ func (s *restoreSweeper) chunkRestored(n int64) {
 	s.sweep()
 }
 
-// sweep deletes any cached blob whose chunks are no longer referenced
+// sweep deletes every cached blob that no pending file references.
-// by an unrestored file. Per-blob DB failures are logged and the blob
-// is kept — we'd rather hold a blob longer than risk a re-download.
 func (s *restoreSweeper) sweep() {
 	for _, blobHash := range s.cache.Keys() {
-		needed, err := s.blobStillNeeded(blobHash)
+		if !s.plan.pendingFileUsesBlob(blobHash) {
-		if err != nil {
-			log.Debug("sweeper referencing-files query failed",
-				"blob_hash", blobHash[:16], "error", err)
-
-			continue
-		}
-
-		if !needed {
 			s.cache.Delete(blobHash)
 		}
 	}
 }
-
-// blobStillNeeded returns true if any file that references a chunk in
-// this blob has not yet been restored. On any error the function
-// returns true — keeping the blob is always the safe answer because we
-// can't prove we're done with it.
-func (s *restoreSweeper) blobStillNeeded(blobHash string) (bool, error) {
-	rows, err := s.repos.DB().Conn().QueryContext(s.ctx, `
-		SELECT DISTINCT fc.file_id
-		FROM file_chunks fc
-		JOIN blob_chunks bc ON bc.chunk_hash = fc.chunk_hash
-		JOIN blobs b ON b.id = bc.blob_id
-		WHERE b.blob_hash = ?
-	`, blobHash)
-	if err != nil {
-		return true, fmt.Errorf("querying referencing files: %w", err)
-	}
-
-	defer func() { _ = rows.Close() }()
-
-	for rows.Next() {
-		var fileID string
-
-		err := rows.Scan(&fileID)
-		if err != nil {
-			return true, fmt.Errorf("scanning file_id: %w", err)
-		}
-
-		if _, ok := s.restored[fileID]; !ok {
-			return true, nil
-		}
-	}
-
-	err = rows.Err()
-	if err != nil {
-		return true, err
-	}
-
-	return false, nil
-}

internal/vaultik/restore_sweeper_test.go

@@ -0,0 +1,163 @@
+package vaultik //nolint:testpackage // reads the restore blob cache's peak size
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/ui"
+)
+
+// TestRestoreFreesBlobsNoPendingFileNeeds asserts that the restore cache
+// frees a blob once no file still to be restored in this run needs it.
+// Every original file has a copy with the same content, so every blob
+// holds chunks of copies: a restore that leaves the copies out of its
+// paths, or fails on them, must not keep the blobs for them.
+func TestRestoreFreesBlobsNoPendingFileNeeds(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	// Small files keep the backup and both restores quick.
+	const fileBytes = 16 * 1024
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+
+	dataDir := filepath.Join(tempDir, "source")
+	originalsDir := filepath.Join(dataDir, "originals")
+	copiesDir := filepath.Join(dataDir, "copies")
+	storeDir := filepath.Join(tempDir, "remote")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	originals := writeFilesWithCopies(t, fs, originalsDir, copiesDir, fileBytes)
+
+	ctx := context.Background()
+
+	cfg, storer, snapshotID := setupLocalityBackup(
+		ctx, t, fs, dataDir, storeDir, dbPath, fileBytes)
+
+	t.Run("restore by path", func(t *testing.T) {
+		t.Parallel()
+
+		restoreDir := filepath.Join(tempDir, "by-path")
+
+		peak, err := restorePeakCachedBlobs(ctx, t, fs, cfg, storer,
+			&RestoreOptions{
+				SnapshotID: snapshotID,
+				TargetDir:  restoreDir,
+				Paths:      []string{originalsDir},
+			})
+		require.NoError(t, err)
+
+		requireRestoredFiles(t, fs, restoreDir, originals)
+		assert.LessOrEqualf(t, peak, 1,
+			"peak cached blobs was %d; blobs were kept for unselected files",
+			peak)
+	})
+
+	t.Run("restore with failed files", func(t *testing.T) {
+		t.Parallel()
+
+		restoreDir := filepath.Join(tempDir, "with-failures")
+
+		// A regular file where the copies directory belongs makes every
+		// copy fail to restore.
+		blocker := filepath.Join(restoreDir, copiesDir)
+		require.NoError(t, fs.MkdirAll(filepath.Dir(blocker), 0o755))
+		require.NoError(t, afero.WriteFile(fs, blocker, nil, 0o644))
+
+		peak, err := restorePeakCachedBlobs(ctx, t, fs, cfg, storer,
+			&RestoreOptions{
+				SnapshotID: snapshotID,
+				TargetDir:  restoreDir,
+				SkipErrors: true,
+			})
+		require.ErrorIs(t, err, errFilesFailedRestore)
+
+		requireRestoredFiles(t, fs, restoreDir, originals)
+		assert.LessOrEqualf(t, peak, 1,
+			"peak cached blobs was %d; blobs were kept for failed files", peak)
+	})
+}
+
+// writeFilesWithCopies writes 15 random files of fileBytes into
+// originalsDir and a copy of each into copiesDir. With
+// setupLocalityBackup given the same size, each file is one chunk and the
+// 15 chunks fill 3 blobs. Returns the originals' content by path.
+func writeFilesWithCopies(
+	t *testing.T, fs afero.Fs, originalsDir, copiesDir string, fileBytes int,
+) map[string][]byte {
+	t.Helper()
+
+	const fileCount = 15
+
+	require.NoError(t, fs.MkdirAll(originalsDir, 0o755))
+	require.NoError(t, fs.MkdirAll(copiesDir, 0o755))
+
+	originals := make(map[string][]byte, fileCount)
+
+	for i := range fileCount {
+		name := fmt.Sprintf("file-%02d.bin", i)
+		data := randomBytes(t, fileBytes)
+		original := filepath.Join(originalsDir, name)
+		originals[original] = data
+
+		require.NoError(t, afero.WriteFile(fs, original, data, 0o644))
+		require.NoError(t, afero.WriteFile(
+			fs, filepath.Join(copiesDir, name), data, 0o644))
+	}
+
+	return originals
+}
+
+// requireRestoredFiles requires every file in want to have been restored
+// under restoreDir with its content intact.
+func requireRestoredFiles(
+	t *testing.T, fs afero.Fs, restoreDir string, want map[string][]byte,
+) {
+	t.Helper()
+
+	for path, data := range want {
+		got, err := afero.ReadFile(fs, filepath.Join(restoreDir, path))
+		require.NoErrorf(t, err, "file missing after restore: %s", path)
+		require.Equalf(t, data, got, "byte mismatch for %s", path)
+	}
+}
+
+// restorePeakCachedBlobs runs a restore and returns the most blobs its
+// cache held at once, together with the restore's error.
+func restorePeakCachedBlobs(
+	ctx context.Context, t *testing.T, fs afero.Fs,
+	cfg *config.Config, storer storage.Storer, opts *RestoreOptions,
+) (int, error) {
+	t.Helper()
+
+	var cache *blobDiskCache
+
+	v := &Vaultik{
+		Config:  cfg,
+		Storage: storer,
+		Fs:      fs,
+		Stdout:  io.Discard,
+		Stderr:  io.Discard,
+		UI:      ui.NewWithColor(io.Discard, false),
+		restoreCacheObserver: func(c *blobDiskCache) {
+			cache = c
+		},
+	}
+	v.SetContext(ctx)
+
+	err := v.Restore(opts)
+
+	require.NotNil(t, cache, "restoreCacheObserver must fire during restore")
+
+	return cache.PeakLen(), err
+}

internal/vaultik/snapshot.go

@@ -243,12 +243,18 @@ func (v *Vaultik) createNamedSnapshot(
 		return err
 	}
 
+	previousSnapshotID, err := v.previousSnapshotID(snapName)
+	if err != nil {
+		return err
+	}
+
 	scanner := v.ScannerFactory(snapshot.ScannerParams{
 		EnableProgress:     !opts.Cron,
 		UI:                 v.UI,
 		Fs:                 v.Fs,
 		Exclude:            v.Config.GetExcludes(snapName),
 		SkipErrors:         opts.SkipErrors,
+		PreviousSnapshotID: previousSnapshotID,
 	})
 
 	snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -283,6 +289,24 @@ func (v *Vaultik) createNamedSnapshot(
 	return nil
 }
 
+// previousSnapshotID returns the ID of the most recent completed snapshot
+// named snapName in the local index, or "" when there is none.
+func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
+	snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
+	if err != nil {
+		return "", fmt.Errorf("listing snapshots: %w", err)
+	}
+
+	for _, s := range snapshots {
+		if s.CompletedAt != nil &&
+			parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
+			return s.ID.String(), nil
+		}
+	}
+
+	return "", nil
+}
+
 // resolveSnapshotPaths resolves source directories to absolute paths
 // with symlink resolution.
 func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
@@ -574,7 +598,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
 
 	if len(toDelete) == 0 {
 		if !opts.Quiet {
-			v.printlnStdout("No snapshots to delete")
+			v.UI.Infof("No snapshots to delete.")
 		}
 
 		return nil
@@ -631,19 +655,19 @@ func selectSnapshotsToPurge(
 func (v *Vaultik) confirmAndExecutePurge(
 	toDelete []SnapshotInfo, force, quiet bool,
 ) error {
-	if !quiet {
+	// Confirm unless --force is used. The list, the prompt and its answer
+	// bypass the UI: they are an interactive exchange the operator must
+	// see, so --quiet does not silence them.
+	if !force {
 		v.stdoutf("The following snapshots will be deleted:\n\n")
 
 		for _, snap := range toDelete {
 			v.stdoutf("  %s (%s, %s)\n",
 				snap.ID,
-				snap.Timestamp.Format("2006-01-02 15:04:05"),
+				snap.Timestamp.Format(time.DateTime),
 				ubytes(snap.CompressedSize))
 		}
-	}
 
-	// Confirm unless --force is used
-	if !force {
 		v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete))
 
 		var confirm string
@@ -661,7 +685,16 @@ func (v *Vaultik) confirmAndExecutePurge(
 			return nil
 		}
 	} else if !quiet {
-		v.stdoutf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete))
+		v.UI.Infof("The following snapshots will be deleted:")
+
+		for _, snap := range toDelete {
+			v.UI.Detailf("%s (%s, %s)",
+				v.UI.Snapshot(snap.ID.String()),
+				snap.Timestamp.Format(time.DateTime),
+				ubytes(snap.CompressedSize))
+		}
+
+		v.UI.Infof("Deleting %d snapshot(s) (--force specified).", len(toDelete))
 	}
 
 	// Delete snapshots (both local and remote)
@@ -694,9 +727,9 @@ func (v *Vaultik) confirmAndExecutePurge(
 	}
 
 	if !quiet {
-		v.stdoutf("Deleted %d snapshot(s)\n", len(toDelete))
+		v.UI.Infof("Deleted %d snapshot(s).", len(toDelete))
-		v.printlnStdout(
+		v.UI.Noticef("Run '%s' to clean up unreferenced remote blobs.",
-			"\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.")
+			pruneCommandHint)
 	}
 
 	return nil
@@ -1069,7 +1102,7 @@ func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
 			log.Info("Removing stale local snapshot record", "snapshot_id", id)
 
 			if !opts.JSON {
-				v.stdoutf("Removing stale local record: %s\n", id)
+				v.UI.Infof("Removing stale local record: %s.", v.UI.Snapshot(id))
 			}
 
 			err = v.deleteSnapshotFromLocalDB(id)
@@ -1092,9 +1125,9 @@ func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
 	}
 
 	if removed == 0 {
-		v.printlnStdout("No stale local snapshots found.")
+		v.UI.Infof("No stale local snapshots found.")
 	} else {
-		v.stdoutf("Removed %d stale local snapshot record(s).\n", removed)
+		v.UI.Infof("Removed %d stale local snapshot record(s).", removed)
 	}
 
 	return nil
@@ -1273,14 +1306,15 @@ func (v *Vaultik) RemoveSnapshot(
 	}
 
 	if inLocalIndex {
-		v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID)
+		v.UI.Infof("Removed snapshot '%s' from local database.",
+			v.UI.Snapshot(snapshotID))
 	}
 
 	if result.RemoteRemoved {
-		v.printlnStdout("Removed snapshot metadata from remote storage")
+		v.UI.Infof("Removed snapshot metadata from remote storage.")
-		v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. "+
+		v.UI.Noticef("The removed snapshot's blobs remain on the remote. "+
 			"Run '%s' to delete any blobs no longer referenced by any "+
-			"remaining remote snapshot.\n", pruneCommandHint)
+			"remaining remote snapshot.", pruneCommandHint)
 	}
 
 	return result, nil
@@ -1336,21 +1370,23 @@ func (v *Vaultik) removeSnapshotDryRun(
 		return result, v.outputRemoveJSON(result)
 	}
 
-	v.stdoutf("Would remove snapshot: %s\n", snapshotID)
+	v.UI.Infof("Would remove snapshot: %s.", v.UI.Snapshot(snapshotID))
 
 	if !opts.LocalOnly {
-		v.printlnStdout("Would also remove snapshot metadata from " +
+		v.UI.Infof("Would also remove snapshot metadata from " +
-			"remote storage (blobs untouched)")
+			"remote storage (blobs untouched).")
 	}
 
-	v.printlnStdout("[Dry run - no changes made]")
+	v.UI.Noticef("Dry run: no changes made.")
 
 	return result, nil
 }
 
 // confirmRemoveSnapshot prompts for confirmation before removing a
 // snapshot. Returns false (treating EOF/read errors as "no") unless the
-// user answers "y".
+// user answers "y". The prompt and its answer bypass the UI: they are an
+// interactive exchange the operator must see, so --quiet does not
+// silence them.
 func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool {
 	if opts.LocalOnly {
 		v.stdoutf("Remove snapshot '%s' from local database "+
@@ -1456,7 +1492,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
 
 	if len(localSnaps) == 0 && len(orphanRemoteKeys) == 0 {
 		if !opts.JSON {
-			v.printlnStdout("No snapshots found")
+			v.UI.Infof("No snapshots found.")
 		}
 
 		return &RemoveResult{}, nil
@@ -1561,27 +1597,27 @@ func (v *Vaultik) handleRemoveAllDryRun(
 	}
 
 	if !opts.JSON {
-		v.stdoutf("Would remove %d local snapshot(s):\n", len(localSnaps))
+		v.UI.Infof("Would remove %d local snapshot(s):", len(localSnaps))
 
 		for _, id := range localSnaps {
-			v.stdoutf("  %s\n", id)
+			v.UI.Detailf("%s", v.UI.Snapshot(id))
 		}
 
 		if !opts.LocalOnly {
 			if len(orphanRemoteKeys) > 0 {
-				v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n",
+				v.UI.Infof("Would also remove %d orphan remote snapshot key(s):",
 					len(orphanRemoteKeys))
 
 				for _, key := range orphanRemoteKeys {
-					v.stdoutf("  %s\n", key)
+					v.UI.Detailf("%s", key)
 				}
 			} else {
-				v.printlnStdout("Would also remove snapshot metadata from " +
+				v.UI.Infof("Would also remove snapshot metadata from " +
-					"remote storage (blobs untouched)")
+					"remote storage (blobs untouched).")
 			}
 		}
 
-		v.printlnStdout("[Dry run - no changes made]")
+		v.UI.Noticef("Dry run: no changes made.")
 	}
 
 	if opts.JSON {
@@ -1630,10 +1666,10 @@ func (v *Vaultik) executeRemoveAll(
 		return result, v.outputRemoveJSON(result)
 	}
 
-	v.stdoutf("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved))
+	v.UI.Infof("Removed %d snapshot(s).", len(result.SnapshotsRemoved))
 
 	if !opts.LocalOnly && result.RemoteRemoved {
-		v.printlnStdout("Removed snapshot metadata from remote storage")
+		v.UI.Infof("Removed snapshot metadata from remote storage.")
 	}
 
 	return result, nil

internal/vaultik/snapshot_deleted_count_test.go

@@ -0,0 +1,225 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
+// backup counts a file as deleted only when the previous snapshot of
+// the same name had it and it is gone, and never counts an excluded
+// file.
+
+// deletedCountEnv is a backup setup over the source directory src whose
+// user-facing output is kept in out. Each test configures its snapshots
+// with setSnapshot.
+type deletedCountEnv struct {
+	v   *vaultik.Vaultik
+	out *bytes.Buffer
+	src string
+}
+
+// newDeletedCountEnv writes files, keyed by path relative to src.
+func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
+	t.Helper()
+
+	ctx := context.Background()
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	srcDir := filepath.Join(tempDir, "src")
+
+	for rel, content := range files {
+		path := filepath.Join(srcDir, rel)
+		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
+		require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
+	}
+
+	cfg := faultTestConfig()
+	cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
+	cfg.Snapshots = map[string]config.SnapshotConfig{}
+
+	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, cfg.IndexPath)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = db.Close() })
+
+	repos := database.NewRepositories(db)
+	out := &bytes.Buffer{}
+	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+	v.UI = ui.NewWithColor(out, false)
+
+	return &deletedCountEnv{v: v, out: out, src: srcDir}
+}
+
+// setSnapshot configures the snapshot name to back up the path rel
+// beneath src, with the given excludes.
+func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
+	e.v.Config.Snapshots[name] = config.SnapshotConfig{
+		Paths:   []string{filepath.Join(e.src, rel)},
+		Exclude: exclude,
+	}
+}
+
+// backUp runs a backup of the named snapshot and returns its output.
+func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
+	t.Helper()
+
+	e.out.Reset()
+	require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+		Snapshots: []string{name},
+	}))
+
+	return e.out.String()
+}
+
+// A file this snapshot excludes is not counted as deleted: neither one
+// that another snapshot backs up, nor one that the previous snapshot of
+// this name had before the exclude was added.
+func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"top.txt":       []byte("top"),
+		"cache/tmp.bin": []byte("cache"),
+		"photos/a.jpg":  []byte("photo a"),
+		"photos/b.jpg":  []byte("photo b"),
+		"photos/c.jpg":  []byte("photo c"),
+	})
+	env.setSnapshot("photos", "photos")
+	env.setSnapshot("home", "", "/photos")
+
+	env.backUp(t, "photos")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
+
+	env.setSnapshot("home", "", "/photos", "/cache")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A file inside a directory that an exclude added after a backup
+// matches is not counted as deleted, though the pattern matches only the
+// directory.
+func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"top.txt":                         []byte("top"),
+		"user/.local/share/Trash/old.txt": []byte("old"),
+	})
+	env.setSnapshot("home", "")
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
+
+	env.setSnapshot("home", "", "*/.local/share/Trash")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// Under --skip-errors a file inside a directory that the walk cannot
+// open is not counted as deleted: it is still on disk.
+func TestBackupDoesNotCountFileInUnopenableDirectoryAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"open.txt":          []byte("open"),
+		"locked/inside.txt": []byte("inside"),
+	})
+	env.setSnapshot("work", "")
+
+	assert.Contains(t, env.backUp(t, "work"), filesLine(2, 2, 0))
+
+	// A directory's mode does not lock out root, who runs the tests, so
+	// opening the directory is made to fail instead.
+	env.v.Fs = &openFailFs{
+		path: filepath.Join(env.src, "locked"),
+		err:  os.ErrPermission,
+	}
+
+	env.out.Reset()
+	require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+		SkipErrors: true,
+		Snapshots:  []string{"work"},
+	}))
+	assert.Contains(t, env.out.String(), filesLine(1, 0, 1))
+}
+
+// A backup counts only files that the previous snapshot of its own name
+// had, not one that a later snapshot of another name over the same path
+// had and that is since removed.
+func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"kept.txt": []byte("kept"),
+	})
+	env.setSnapshot("home", "")
+	env.setSnapshot("other", "")
+
+	env.backUp(t, "home")
+
+	// Snapshot start times are kept in whole seconds: wait so that other's
+	// snapshot is strictly the most recent.
+	time.Sleep(time.Second)
+
+	extra := filepath.Join(env.src, "extra.txt")
+	require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
+	env.backUp(t, "other")
+	require.NoError(t, os.Remove(extra))
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A file removed from disk is counted as deleted by the next backup of
+// the snapshot that had it, and not by any later backup of that
+// snapshot, nor by the first backup of another name over the same path.
+func TestBackupCountsDeletedFileOnce(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	aContent := bytes.Repeat([]byte("a"), 1000)
+	bContent := bytes.Repeat([]byte("b"), 2000)
+	keptContent := bytes.Repeat([]byte("k"), 500)
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"a.txt":    aContent,
+		"b.txt":    bContent,
+		"kept.txt": keptContent,
+	})
+	env.setSnapshot("home", "")
+	env.setSnapshot("other", "")
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
+
+	require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
+	require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
+
+	summary := env.backUp(t, "home")
+	assert.Contains(t, summary,
+		"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
+	assert.Contains(t, summary,
+		fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
+			env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
+			env.v.UI.Size(int64(len(aContent)+len(bContent)))))
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+	assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
+}