Name each prefix once in restore, verify and purge errors (closes #327)

A missing blob during restore, a failed shallow verify and an invalid
purge duration each printed their prefix twice. FetchBlob now returns
the storage error as is, since restore and deep verify name the blob
themselves; purge returns the duration parser's error unchanged; and a
failed shallow verify wraps "problems found in storage" instead of
"verification failed", which the command already prints. Deep verify
words a storage error other than not found as a blob it could not
check, and its log line no longer says S3.

Already fixed for #314: the
deep verify size message naming S3 and other errors called missing;
tests now check both.
Not tested: the log line.

Model: opus-5-5

ARCHITECTURE.md

@@ -279,8 +279,8 @@ Key methods:
 - `Scan(ctx, path, snapshotID)` → Main scan operation
 
 Scan phases:
-1. **Phase 1**: Walk directory, identify files needing processing
-2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists, the walk did not find, and are gone from disk; a file that is excluded, or inside an excluded directory, is not counted
+1. **Phase 0**: Detect deleted files from previous snapshots
+2. **Phase 1**: Walk directory, identify files needing processing
 3. **Phase 2**: Process files (chunk → pack → upload)
 
 #### SnapshotManager
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
     │       │
     │       ├─► scanner.Scan(ctx, path, snapshotID)
     │       │       │
+    │       │       ├─► Phase 0: detectDeletedFiles()
+    │       │       │
     │       │       ├─► Phase 1: scanPhase()
     │       │       │       Walk directory
     │       │       │       Check file metadata changes
     │       │       │       Build list of files to process
     │       │       │
-    │       │       ├─► Phase 1b: detectDeletedFiles()
-    │       │       │
     │       │       └─► Phase 2: processPhase()
     │       │               For each file:
     │       │                   chunker.ChunkReaderStreaming()

internal/database/files.go

@@ -267,45 +267,6 @@ func (r *FileRepository) ListUnderPath(
 	return files, rows.Err()
 }
 
-// ListUnderPathInSnapshot returns the files that ListUnderPath returns
-// for path and that the snapshot lists, in no particular order.
-func (r *FileRepository) ListUnderPathInSnapshot(
-	ctx context.Context, path, snapshotID string,
-) ([]*File, error) {
-	path = strings.TrimRight(path, "/")
-	dirPrefix := path + "/"
-
-	query := `
-		SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
-			f.mode, f.uid, f.gid, f.link_target
-		FROM files f
-		JOIN snapshot_files sf ON sf.file_id = f.id
-		WHERE sf.snapshot_id = ?
-		  AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
-	`
-
-	rows, err := r.db.conn.QueryContext(ctx, query,
-		snapshotID, path, dirPrefix, dirPrefix)
-	if err != nil {
-		return nil, fmt.Errorf("querying files: %w", err)
-	}
-
-	defer func() { _ = rows.Close() }()
-
-	var files []*File
-
-	for rows.Next() {
-		file, err := r.scanFileRows(rows)
-		if err != nil {
-			return nil, fmt.Errorf("scanning file: %w", err)
-		}
-
-		files = append(files, file)
-	}
-
-	return files, rows.Err()
-}
-
 // ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
 // path starts with prefix and that list at least one chunk held by no
 // blob whose upload has completed (uploaded_ts set). A new snapshot

internal/database/files_test.go

@@ -212,65 +212,6 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
 	}
 }
 
-// ListUnderPathInSnapshot leaves out files outside path and files the
-// snapshot does not list, though another snapshot lists them.
-func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
-	t.Parallel()
-
-	db, cleanup := setupTestDB(t)
-	defer cleanup()
-
-	ctx := context.Background()
-	files := database.NewFileRepository(db)
-	snapshots := database.NewSnapshotRepository(db)
-
-	listed := map[string][]string{
-		"one": {"/src/a.txt", "/src2/c.txt"},
-		"two": {"/src/b.txt"},
-	}
-
-	for snapshotID, paths := range listed {
-		err := snapshots.Create(ctx, nil, &database.Snapshot{
-			ID:        types.SnapshotID(snapshotID),
-			Hostname:  "host",
-			StartedAt: time.Now(),
-		})
-		if err != nil {
-			t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
-		}
-
-		for _, path := range paths {
-			err := files.Create(ctx, nil, &database.File{
-				Path:  types.FilePath(path),
-				MTime: time.Now().Truncate(time.Second),
-				Mode:  0644,
-			})
-			if err != nil {
-				t.Fatalf("failed to create %s: %v", path, err)
-			}
-
-			err = snapshots.AddFile(ctx, nil, snapshotID, path)
-			if err != nil {
-				t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
-			}
-		}
-	}
-
-	found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
-	if err != nil {
-		t.Fatalf("failed to list files: %v", err)
-	}
-
-	got := make([]string, 0, len(found))
-	for _, f := range found {
-		got = append(got, f.Path.String())
-	}
-
-	if !slices.Equal(got, []string{"/src/a.txt"}) {
-		t.Errorf("got %q, want only /src/a.txt", got)
-	}
-}
-
 func TestFileRepositorySymlink(t *testing.T) {
 	t.Parallel()
 

internal/snapshot/module.go

@@ -16,9 +16,6 @@ type ScannerParams struct {
 	Fs             afero.Fs
 	Exclude        []string // Exclude patterns (combined global + snapshot-specific)
 	SkipErrors     bool     // Skip file read errors (log loudly but continue)
-	// PreviousSnapshotID is the most recent completed snapshot of the
-	// same name, "" for none.
-	PreviousSnapshotID string
 }
 
 // Module exports backup functionality as an fx module.
@@ -47,18 +44,17 @@ func provideScannerFactory(
 		}
 
 		return NewScanner(ScannerConfig{
-			FS:                 params.Fs,
-			ChunkSize:          cfg.ChunkSize.Int64(),
-			Repositories:       repos,
-			Storage:            storer,
-			MaxBlobSize:        cfg.BlobSizeLimit.Int64(),
-			CompressionLevel:   cfg.CompressionLevel,
-			AgeRecipients:      cfg.AgeRecipients,
-			EnableProgress:     params.EnableProgress,
-			UI:                 params.UI,
-			Exclude:            excludes,
-			SkipErrors:         params.SkipErrors,
-			PreviousSnapshotID: params.PreviousSnapshotID,
+			FS:               params.Fs,
+			ChunkSize:        cfg.ChunkSize.Int64(),
+			Repositories:     repos,
+			Storage:          storer,
+			MaxBlobSize:      cfg.BlobSizeLimit.Int64(),
+			CompressionLevel: cfg.CompressionLevel,
+			AgeRecipients:    cfg.AgeRecipients,
+			EnableProgress:   params.EnableProgress,
+			UI:               params.UI,
+			Exclude:          excludes,
+			SkipErrors:       params.SkipErrors,
 		})
 	}
 }

internal/snapshot/scanner.go

@@ -58,10 +58,6 @@ type Scanner struct {
 	compressionLevel int
 	ageRecipient     string
 	snapshotID       string // Current snapshot being processed
-	// previousSnapshotID is the most recent completed snapshot of the
-	// same name, "" for none. Only a file it lists can be counted as
-	// deleted.
-	previousSnapshotID string
 	// currentSourcePath is the source directory being scanned, stored with
 	// each file record.
 	currentSourcePath string
@@ -133,9 +129,6 @@ type ScannerConfig struct {
 	// SkipErrors skips files that cannot be opened or read (log loudly but
 	// continue); packer, database, encryption, and upload errors still abort.
 	SkipErrors bool
-	// PreviousSnapshotID is the most recent completed snapshot of the
-	// same name, "" for none.
-	PreviousSnapshotID string
 }
 
 // ScanResult contains the results of a scan operation. Files and bytes
@@ -214,7 +207,6 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		compiledExclude:    compiledExclude,
 		progress:           progress,
 		skipErrors:         cfg.SkipErrors,
-		previousSnapshotID: cfg.PreviousSnapshotID,
 		ui:                 uiw,
 		pendingChunkHashes: make(map[string]struct{}),
 	}
@@ -266,9 +258,8 @@ func (s *Scanner) Scan(
 
 	filesToProcess := scanResult.FilesToProcess
 
-	// Phase 1b: Detect deleted files by comparing the previous snapshot
-	// of this name against scanned files
-	err = s.detectDeletedFiles(ctx, path, existingFiles, result)
+	// Phase 1b: Detect deleted files by comparing DB against scanned files
+	err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
 	if err != nil {
 		return nil, fmt.Errorf("detecting deleted files: %w", err)
 	}
@@ -448,8 +439,9 @@ func (s *Scanner) summarizeScanPhase(
 }
 
 // loadKnownFiles loads the known files at and beneath path from the
-// database into a map for fast lookup. This avoids per-file database
-// queries during the scan phase.
+// database into a map for fast lookup. Every loaded file the scan does
+// not find is counted as deleted. This avoids per-file database queries
+// during the scan phase.
 func (s *Scanner) loadKnownFiles(
 	ctx context.Context, path string,
 ) (map[string]*database.File, error) {
@@ -1962,29 +1954,21 @@ func (s *Scanner) queueFileForBatchInsert(
 	})
 }
 
-// detectDeletedFiles counts as deleted each file at or beneath path that
-// the previous snapshot of this name lists, the walk did not find, and
-// is gone from disk. A file that is excluded, or inside an excluded
-// directory, is not counted: the walk skips it whether or not it is
-// still on disk.
-func (s *Scanner) detectDeletedFiles(
+// detectDeletedFilesFromMap finds files that existed in previous snapshots
+// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
+// database access.
+func (s *Scanner) detectDeletedFilesFromMap(
 	ctx context.Context,
-	path string,
+	knownFiles map[string]*database.File,
 	existingFiles map[string]struct{},
 	result *ScanResult,
 ) error {
-	if s.previousSnapshotID == "" {
+	if len(knownFiles) == 0 {
 		return nil
 	}
 
-	previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
-		ctx, path, s.previousSnapshotID)
-	if err != nil {
-		return fmt.Errorf("listing files of snapshot %s: %w",
-			s.previousSnapshotID, err)
-	}
-
-	for _, file := range previousFiles {
+	// Check each known file against the enumerated set (no filesystem access needed)
+	for path, file := range knownFiles {
 		// Check context cancellation periodically
 		select {
 		case <-ctx.Done():
@@ -1992,22 +1976,13 @@ func (s *Scanner) detectDeletedFiles(
 		default:
 		}
 
-		filePath := file.Path.String()
-
-		_, exists := existingFiles[filePath]
-		if exists || s.excludedOrInExcludedDir(filePath, path) {
-			continue
+		// Check if the file exists in our enumerated set
+		if _, exists := existingFiles[path]; !exists {
+			// File has been deleted
+			result.FilesDeleted++
+			result.BytesDeleted += file.Size
+			log.Debug("Detected deleted file", "path", path, "size", file.Size)
 		}
-
-		// Under --skip-errors the walk also misses what it cannot reach,
-		// such as the files in a directory it cannot open.
-		if !s.goneFromDisk(filePath) {
-			continue
-		}
-
-		result.FilesDeleted++
-		result.BytesDeleted += file.Size
-		log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
 	}
 
 	if result.FilesDeleted > 0 {
@@ -2018,36 +1993,6 @@ func (s *Scanner) detectDeletedFiles(
 	return nil
 }
 
-// excludedOrInExcludedDir reports whether filePath, which is at or
-// beneath rootPath, or any directory between rootPath and it, matches an
-// exclude pattern. A pattern such as */.local/share/Trash matches the
-// directory but not the files inside it, which the walk skips too.
-func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
-	for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
-		if s.shouldExclude(p, rootPath) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// goneFromDisk reports whether looking filePath up on disk finds that
-// it does not exist. Like the walk, it does not follow a symlink, so a
-// symlink whose target is gone is still on disk. A file whose directory
-// was replaced by a regular file is gone too, though the lookup reports
-// "not a directory".
-func (s *Scanner) goneFromDisk(filePath string) bool {
-	var err error
-	if lstater, ok := s.fs.(afero.Lstater); ok {
-		_, _, err = lstater.LstatIfPossible(filePath)
-	} else {
-		_, err = s.fs.Stat(filePath)
-	}
-
-	return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR)
-}
-
 // wrapPermissionError augments permission errors with platform-specific
 // remediation instructions. On macOS, TCC-protected directories (Calendars,
 // Reminders, Photos, etc.) return EPERM unless the running application has

internal/vaultik/blob_fetch.go

@@ -104,7 +104,8 @@ func (v *Vaultik) FetchAndDecryptBlob(
 
 // FetchBlob downloads a blob and returns a reader for the encrypted data.
 // The reader is closed as soon as ctx is cancelled, which ends a read
-// blocked inside it.
+// blocked inside it. A storage error is returned as is: restore and deep
+// verify name the blob in their own errors.
 func (v *Vaultik) FetchBlob(
 	ctx context.Context, blobHash string,
 ) (io.ReadCloser, error) {
@@ -119,7 +120,7 @@ func (v *Vaultik) FetchBlob(
 
 	rc, err := v.Storage.Get(ctx, blobPath)
 	if err != nil {
-		return nil, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
+		return nil, err
 	}
 
 	return newCloseOnCancelReader(ctx, rc), nil

internal/vaultik/deep_verify_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 
@@ -142,7 +143,8 @@ func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
 
 // TestDeepVerifyJSONCountsWrongSizeBlob checks that under --deep --json a
 // blob of the wrong size is counted in mismatched and fails the run, and
-// that the check goes on past it to count a missing blob after it.
+// that the check goes on past it to count a missing blob after it. The
+// store is on disk, so the message must not call it S3.
 func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -168,11 +170,14 @@ func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
 	require.Equal(t, 1, got.Mismatched)
 	require.Equal(t, 1, got.Missing)
 	require.Zero(t, got.Verified)
+	require.Contains(t, got.ErrorMessage, "with the wrong size")
+	require.NotContains(t, got.ErrorMessage, "S3")
 }
 
 // TestDeepVerifyStopsOnBlobThatCannotBeChecked checks that a blob that is
 // present in storage but cannot be checked is not counted as missing:
-// deep verify stops and fails with the storage error.
+// deep verify stops and fails, saying it could not check the blob, with
+// the storage error.
 func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -187,6 +192,8 @@ func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
 	require.Error(t, err)
 	require.Equal(t, "failed", got.Status)
 	require.Zero(t, got.Missing)
+	require.Contains(t, got.ErrorMessage,
+		"could not check blob "+path.Base(keys[0]))
 	require.Contains(t, got.ErrorMessage, errStatDenied.Error())
 }
 

internal/vaultik/fault_injection_test.go

@@ -661,18 +661,17 @@ func faultScannerFactory(
 ) snapshot.ScannerFactory {
 	return func(params snapshot.ScannerParams) *snapshot.Scanner {
 		return snapshot.NewScanner(snapshot.ScannerConfig{
-			FS:                 params.Fs,
-			Storage:            storer,
-			ChunkSize:          faultChunkSize,
-			MaxBlobSize:        faultMaxBlobSize,
-			CompressionLevel:   cfg.CompressionLevel,
-			AgeRecipients:      cfg.AgeRecipients,
-			Repositories:       repos,
-			EnableProgress:     params.EnableProgress,
-			UI:                 params.UI,
-			Exclude:            params.Exclude,
-			SkipErrors:         params.SkipErrors,
-			PreviousSnapshotID: params.PreviousSnapshotID,
+			FS:               params.Fs,
+			Storage:          storer,
+			ChunkSize:        faultChunkSize,
+			MaxBlobSize:      faultMaxBlobSize,
+			CompressionLevel: cfg.CompressionLevel,
+			AgeRecipients:    cfg.AgeRecipients,
+			Repositories:     repos,
+			EnableProgress:   params.EnableProgress,
+			UI:               params.UI,
+			Exclude:          params.Exclude,
+			SkipErrors:       params.SkipErrors,
 		})
 	}
 }

internal/vaultik/purge_per_name_test.go

@@ -281,7 +281,8 @@ func TestPurgeOlderThan_TooLargeDeletesNothing(t *testing.T) {
 		OlderThan: "1000y",
 		Force:     true,
 	})
-	require.Error(t, err)
+	require.EqualError(t, err,
+		`invalid duration: "1000y" is longer than about 292 years`)
 
 	err = v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
 		OlderThan: "292y",

internal/vaultik/restore_skip_errors_test.go

@@ -231,7 +231,8 @@ func (f trailingByteFile) Close() error {
 }
 
 // TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
-// still ends the restore with an error when SkipErrors is not set.
+// still ends the restore with an error when SkipErrors is not set, and
+// that the error names the blob once.
 func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -247,6 +248,8 @@ func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
 	})
 
 	require.ErrorIs(t, err, storage.ErrNotFound)
+	assert.Equal(t, 1, strings.Count(err.Error(), "downloading blob"),
+		err.Error())
 }
 
 // TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held

internal/vaultik/shallow_verify_test.go

@@ -27,7 +27,7 @@ import (
 // then grows one stored blob so its size no longer matches the manifest.
 // Shallow verify must then fail, count the grown blob as a size mismatch,
 // and drop it from the verified count rather than continuing to report it
-// as checked.
+// as checked, and its error must not repeat the cli layer's prefix.
 func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -96,6 +96,12 @@ func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) {
 	require.Equal(t, 1, bad.Mismatched)
 	require.Equal(t, healthy.BlobCount-1, bad.Verified,
 		"the wrong-sized blob must not be counted as verified")
+
+	// The cli layer prints this error after its own "Verification failed:".
+	err = newVerifier(io.Discard).VerifySnapshotWithOptions(
+		snapshotID, &vaultik.VerifyOptions{})
+	require.EqualError(t, err,
+		"problems found in storage: 1 blobs have the wrong size")
 }
 
 // TestShallowVerifyDetectsMissingDatabase backs up a real snapshot,

internal/vaultik/snapshot.go

@@ -23,6 +23,7 @@ var (
 	errSnapshotNotInConfig  = errors.New("snapshot not found in config")
 	errNoSnapshotsInConfig  = errors.New("no snapshots configured")
 	errSnapshotVerifyFailed = errors.New("verification failed")
+	errProblemsInStorage    = errors.New("problems found in storage")
 	errRemoveAllNeedsForce  = errors.New("--all requires --force")
 	errSnapshotNotFound     = errors.New("snapshot not found")
 	errSnapshotRemoveFailed = errors.New("snapshot(s) could not be removed")
@@ -243,18 +244,12 @@ func (v *Vaultik) createNamedSnapshot(
 		return err
 	}
 
-	previousSnapshotID, err := v.previousSnapshotID(snapName)
-	if err != nil {
-		return err
-	}
-
 	scanner := v.ScannerFactory(snapshot.ScannerParams{
-		EnableProgress:     !opts.Cron,
-		UI:                 v.UI,
-		Fs:                 v.Fs,
-		Exclude:            v.Config.GetExcludes(snapName),
-		SkipErrors:         opts.SkipErrors,
-		PreviousSnapshotID: previousSnapshotID,
+		EnableProgress: !opts.Cron,
+		UI:             v.UI,
+		Fs:             v.Fs,
+		Exclude:        v.Config.GetExcludes(snapName),
+		SkipErrors:     opts.SkipErrors,
 	})
 
 	snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -289,24 +284,6 @@ func (v *Vaultik) createNamedSnapshot(
 	return nil
 }
 
-// previousSnapshotID returns the ID of the most recent completed snapshot
-// named snapName in the local index, or "" when there is none.
-func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
-	snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
-	if err != nil {
-		return "", fmt.Errorf("listing snapshots: %w", err)
-	}
-
-	for _, s := range snapshots {
-		if s.CompletedAt != nil &&
-			parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
-			return s.ID.String(), nil
-		}
-	}
-
-	return "", nil
-}
-
 // resolveSnapshotPaths resolves source directories to absolute paths
 // with symlink resolution.
 func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
@@ -636,7 +613,7 @@ func selectSnapshotsToPurge(
 	case opts.OlderThan != "":
 		duration, err := parseDuration(opts.OlderThan)
 		if err != nil {
-			return nil, fmt.Errorf("invalid duration: %w", err)
+			return nil, err
 		}
 
 		cutoff := time.Now().UTC().Add(-duration)
@@ -965,8 +942,10 @@ func (v *Vaultik) formatVerifyResult(
 		v.printVerifySummary(result, failure)
 	}
 
+	// Not errSnapshotVerifyFailed: the cli layer prints this after its own
+	// "Verification failed:".
 	if failure != "" {
-		return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, failure)
+		return fmt.Errorf("%w: %s", errProblemsInStorage, failure)
 	}
 
 	return nil

internal/vaultik/snapshot_deleted_count_test.go

@@ -1,249 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/ui"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
-// backup counts a file as deleted only when the previous snapshot of
-// the same name had it and it is gone, and never counts an excluded
-// file.
-
-// deletedCountEnv is a backup setup over the source directory src whose
-// user-facing output is kept in out. Each test configures its snapshots
-// with setSnapshot.
-type deletedCountEnv struct {
-	v   *vaultik.Vaultik
-	out *bytes.Buffer
-	src string
-}
-
-// newDeletedCountEnv writes files, keyed by path relative to src.
-func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
-	t.Helper()
-
-	ctx := context.Background()
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	srcDir := filepath.Join(tempDir, "src")
-
-	for rel, content := range files {
-		path := filepath.Join(srcDir, rel)
-		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
-		require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
-	}
-
-	cfg := faultTestConfig()
-	cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
-	cfg.Snapshots = map[string]config.SnapshotConfig{}
-
-	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, cfg.IndexPath)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	repos := database.NewRepositories(db)
-	out := &bytes.Buffer{}
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-	v.UI = ui.NewWithColor(out, false)
-
-	return &deletedCountEnv{v: v, out: out, src: srcDir}
-}
-
-// setSnapshot configures the snapshot name to back up the path rel
-// beneath src, with the given excludes.
-func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
-	e.v.Config.Snapshots[name] = config.SnapshotConfig{
-		Paths:   []string{filepath.Join(e.src, rel)},
-		Exclude: exclude,
-	}
-}
-
-// backUp runs a backup of the named snapshot and returns its output.
-func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
-	t.Helper()
-
-	e.out.Reset()
-	require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-		Snapshots: []string{name},
-	}))
-
-	return e.out.String()
-}
-
-// A file this snapshot excludes is not counted as deleted: neither one
-// that another snapshot backs up, nor one that the previous snapshot of
-// this name had before the exclude was added.
-func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"top.txt":       []byte("top"),
-		"cache/tmp.bin": []byte("cache"),
-		"photos/a.jpg":  []byte("photo a"),
-		"photos/b.jpg":  []byte("photo b"),
-		"photos/c.jpg":  []byte("photo c"),
-	})
-	env.setSnapshot("photos", "photos")
-	env.setSnapshot("home", "", "/photos")
-
-	env.backUp(t, "photos")
-	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
-
-	env.setSnapshot("home", "", "/photos", "/cache")
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-}
-
-// A file inside a directory that an exclude added after a backup
-// matches is not counted as deleted, though the pattern matches only the
-// directory and the directory is gone from disk.
-func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"top.txt":                         []byte("top"),
-		"user/.local/share/Trash/old.txt": []byte("old"),
-	})
-	env.setSnapshot("home", "")
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
-
-	require.NoError(t, os.RemoveAll(
-		filepath.Join(env.src, "user/.local/share/Trash")))
-
-	env.setSnapshot("home", "", "*/.local/share/Trash")
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-}
-
-// A file whose directory was replaced by a regular file of the same name
-// is counted as deleted.
-func TestBackupCountsFileInDirectoryReplacedByFileAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"dir/inside.txt": []byte("inside"),
-	})
-	env.setSnapshot("home", "")
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 1, 0))
-
-	dir := filepath.Join(env.src, "dir")
-	require.NoError(t, os.RemoveAll(dir))
-	require.NoError(t, os.WriteFile(dir, []byte("now a file"), 0o600))
-
-	assert.Contains(t, env.backUp(t, "home"),
-		"Files: 1 examined, 1 backed up, 0 unchanged, 1 deleted.")
-}
-
-// Under --skip-errors a file inside a directory that the walk cannot
-// open is not counted as deleted: it is still on disk.
-func TestBackupDoesNotCountFileInUnopenableDirectoryAsDeleted(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"open.txt":          []byte("open"),
-		"locked/inside.txt": []byte("inside"),
-	})
-	env.setSnapshot("work", "")
-
-	assert.Contains(t, env.backUp(t, "work"), filesLine(2, 2, 0))
-
-	// A directory's mode does not lock out root, who runs the tests, so
-	// opening the directory is made to fail instead.
-	env.v.Fs = &openFailFs{
-		path: filepath.Join(env.src, "locked"),
-		err:  os.ErrPermission,
-	}
-
-	env.out.Reset()
-	require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-		SkipErrors: true,
-		Snapshots:  []string{"work"},
-	}))
-	assert.Contains(t, env.out.String(), filesLine(1, 0, 1))
-}
-
-// A backup counts only files that the previous snapshot of its own name
-// had, not one that a later snapshot of another name over the same path
-// had and that is since removed.
-func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"kept.txt": []byte("kept"),
-	})
-	env.setSnapshot("home", "")
-	env.setSnapshot("other", "")
-
-	env.backUp(t, "home")
-
-	// Snapshot start times are kept in whole seconds: wait so that other's
-	// snapshot is strictly the most recent.
-	time.Sleep(time.Second)
-
-	extra := filepath.Join(env.src, "extra.txt")
-	require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
-	env.backUp(t, "other")
-	require.NoError(t, os.Remove(extra))
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-}
-
-// A file removed from disk is counted as deleted by the next backup of
-// the snapshot that had it, and not by any later backup of that
-// snapshot, nor by the first backup of another name over the same path.
-func TestBackupCountsDeletedFileOnce(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	aContent := bytes.Repeat([]byte("a"), 1000)
-	bContent := bytes.Repeat([]byte("b"), 2000)
-	keptContent := bytes.Repeat([]byte("k"), 500)
-
-	env := newDeletedCountEnv(t, map[string][]byte{
-		"a.txt":    aContent,
-		"b.txt":    bContent,
-		"kept.txt": keptContent,
-	})
-	env.setSnapshot("home", "")
-	env.setSnapshot("other", "")
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
-
-	require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
-	require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
-
-	summary := env.backUp(t, "home")
-	assert.Contains(t, summary,
-		"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
-	assert.Contains(t, summary,
-		fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
-			env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
-			env.v.UI.Size(int64(len(aContent)+len(bContent)))))
-
-	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
-	assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
-}

internal/vaultik/verify.go

@@ -680,7 +680,7 @@ func (v *Vaultik) verifyManifestAgainstDatabase(
 func (v *Vaultik) verifyBlobExistenceFromDB(
 	blobs []snapshot.BlobInfo, result *VerifyResult,
 ) error {
-	log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
+	log.Info("Verifying blob existence in storage", "blob_count", len(blobs))
 
 	for i, blob := range blobs {
 		// The hash is read from the snapshot database, which is not
@@ -700,7 +700,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
 			result.Missing++
 			result.MissingSize += blob.CompressedSize
 		case err != nil:
-			return fmt.Errorf("checking blob %s in storage: %w",
+			return fmt.Errorf("could not check blob %s in storage: %w",
 				blob.Hash, err)
 		case stat.Size != blob.CompressedSize:
 			log.Warn("Blob has the wrong size in storage",