Count a file as deleted only if the previous snapshot of its name had it (closes #304)

A backup counted as deleted every file in the local index under the
source path that the walk did not find. That took in files the
snapshot excludes but another snapshot backs up, and files removed long
ago, which every later backup, and the first backup of another name
over the same path, reported as deleted again.

The backup now looks up the most recent completed snapshot of the same
name and counts only the files it lists that the walk did not find. A
file that is excluded, or inside an excluded directory, is not counted.

The snapshots row stores no deleted figure; only the backup's output
changes.

Model: opus-5-5

ARCHITECTURE.md

@@ -279,8 +279,8 @@ Key methods:
 - `Scan(ctx, path, snapshotID)` → Main scan operation
 
 Scan phases:
-1. **Phase 0**: Detect deleted files from previous snapshots
-2. **Phase 1**: Walk directory, identify files needing processing
+1. **Phase 1**: Walk directory, identify files needing processing
+2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists and the walk did not find; a file that is excluded, or inside an excluded directory, is not counted
 3. **Phase 2**: Process files (chunk → pack → upload)
 
 #### SnapshotManager
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
     │       │
     │       ├─► scanner.Scan(ctx, path, snapshotID)
     │       │       │
-    │       │       ├─► Phase 0: detectDeletedFiles()
-    │       │       │
     │       │       ├─► Phase 1: scanPhase()
     │       │       │       Walk directory
     │       │       │       Check file metadata changes
     │       │       │       Build list of files to process
     │       │       │
+    │       │       ├─► Phase 1b: detectDeletedFiles()
+    │       │       │
     │       │       └─► Phase 2: processPhase()
     │       │               For each file:
     │       │                   chunker.ChunkReaderStreaming()

README.md

@@ -294,8 +294,7 @@ needed.
 of the local index and the backup destination store — with timestamps
 and three sizes per snapshot (compressed remote size; total
 uncompressed chunk size; size of the chunks that snapshot stored that
-were not stored before). Both chunk sizes are recorded when the
-snapshot's backup runs.
+were not stored before, recorded when its backup ran).
 
 Listing the destination store does **not** require the age secret key,
 so it works in vaultik's intended configuration, where the backed-up

docs/DATAMODEL.md

@@ -122,7 +122,6 @@ Tracks backup snapshots.
 - `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE
 - `blob_count` (INTEGER) - Number of blobs this snapshot created
 - `total_size` (INTEGER) - Total size of all files
-- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the distinct chunks the snapshot's files reference, recorded when its backup ran; `snapshot list` shows it as UNCOMPRESSED SIZE
 - `blob_size` (INTEGER) - Total compressed size of all referenced blobs
 - `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
 - `compression_ratio` (REAL) - Compression ratio achieved

internal/database/files.go

@@ -267,6 +267,45 @@ func (r *FileRepository) ListUnderPath(
 	return files, rows.Err()
 }
 
+// ListUnderPathInSnapshot returns the files that ListUnderPath returns
+// for path and that the snapshot lists, in no particular order.
+func (r *FileRepository) ListUnderPathInSnapshot(
+	ctx context.Context, path, snapshotID string,
+) ([]*File, error) {
+	path = strings.TrimRight(path, "/")
+	dirPrefix := path + "/"
+
+	query := `
+		SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
+			f.mode, f.uid, f.gid, f.link_target
+		FROM files f
+		JOIN snapshot_files sf ON sf.file_id = f.id
+		WHERE sf.snapshot_id = ?
+		  AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
+	`
+
+	rows, err := r.db.conn.QueryContext(ctx, query,
+		snapshotID, path, dirPrefix, dirPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("querying files: %w", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var files []*File
+
+	for rows.Next() {
+		file, err := r.scanFileRows(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scanning file: %w", err)
+		}
+
+		files = append(files, file)
+	}
+
+	return files, rows.Err()
+}
+
 // ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
 // path starts with prefix and that list at least one chunk held by no
 // blob whose upload has completed (uploaded_ts set). A new snapshot

internal/database/files_test.go

@@ -212,6 +212,65 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
 	}
 }
 
+// ListUnderPathInSnapshot leaves out files outside path and files the
+// snapshot does not list, though another snapshot lists them.
+func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
+	t.Parallel()
+
+	db, cleanup := setupTestDB(t)
+	defer cleanup()
+
+	ctx := context.Background()
+	files := database.NewFileRepository(db)
+	snapshots := database.NewSnapshotRepository(db)
+
+	listed := map[string][]string{
+		"one": {"/src/a.txt", "/src2/c.txt"},
+		"two": {"/src/b.txt"},
+	}
+
+	for snapshotID, paths := range listed {
+		err := snapshots.Create(ctx, nil, &database.Snapshot{
+			ID:        types.SnapshotID(snapshotID),
+			Hostname:  "host",
+			StartedAt: time.Now(),
+		})
+		if err != nil {
+			t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
+		}
+
+		for _, path := range paths {
+			err := files.Create(ctx, nil, &database.File{
+				Path:  types.FilePath(path),
+				MTime: time.Now().Truncate(time.Second),
+				Mode:  0644,
+			})
+			if err != nil {
+				t.Fatalf("failed to create %s: %v", path, err)
+			}
+
+			err = snapshots.AddFile(ctx, nil, snapshotID, path)
+			if err != nil {
+				t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
+			}
+		}
+	}
+
+	found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
+	if err != nil {
+		t.Fatalf("failed to list files: %v", err)
+	}
+
+	got := make([]string, 0, len(found))
+	for _, f := range found {
+		got = append(got, f.Path.String())
+	}
+
+	if !slices.Equal(got, []string{"/src/a.txt"}) {
+		t.Errorf("got %q, want only /src/a.txt", got)
+	}
+}
+
 func TestFileRepositorySymlink(t *testing.T) {
 	t.Parallel()
 

internal/database/schema/001.sql

@@ -89,7 +89,6 @@ CREATE TABLE IF NOT EXISTS snapshots (
     new_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_count INTEGER NOT NULL DEFAULT 0,
     total_size INTEGER NOT NULL DEFAULT 0,
-    uncompressed_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_size INTEGER NOT NULL DEFAULT 0,
     blob_uncompressed_size INTEGER NOT NULL DEFAULT 0,
     compression_ratio REAL NOT NULL DEFAULT 1.0,

internal/database/snapshots.go

@@ -120,11 +120,7 @@ func (r *SnapshotRepository) UpdateCounts(
 	return nil
 }
 
-// UpdateExtendedStats updates extended statistics for a snapshot. It also
-// records the total plaintext size of the distinct chunks the snapshot's
-// files reference. Only the snapshot's own backup can take that total: a
-// file's row is shared by every snapshot that lists it, and a later
-// backup of a changed file rewrites the row with the new chunks.
+// UpdateExtendedStats updates extended statistics for a snapshot
 func (r *SnapshotRepository) UpdateExtendedStats(
 	ctx context.Context,
 	tx *sql.Tx,
@@ -149,17 +145,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
 		    compression_level = ?,
 		    upload_bytes = ?,
 		    upload_duration_ms = ?,
-		    new_chunk_size = ?,
-		    uncompressed_chunk_size = (
-		        SELECT COALESCE(SUM(c.size), 0)
-		        FROM (
-		            SELECT DISTINCT fc.chunk_hash
-		            FROM snapshot_files sf
-		            JOIN file_chunks fc ON sf.file_id = fc.file_id
-		            WHERE sf.snapshot_id = snapshots.id
-		        ) sc
-		        JOIN chunks c ON sc.chunk_hash = c.chunk_hash
-		    )
+		    new_chunk_size = ?
 		WHERE id = ?
 	`
 
@@ -569,22 +555,30 @@ func (r *SnapshotRepository) GetSnapshotBlobSizes(
 	return compressed, uncompressed, nil
 }
 
-// GetSnapshotUncompressedChunkSize returns the total plaintext size of the
-// distinct chunks the snapshot's files reference, as the snapshot's backup
-// recorded it (see UpdateExtendedStats).
+// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
+// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
 func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
 	ctx context.Context, snapshotID string,
 ) (int64, error) {
-	query := `SELECT uncompressed_chunk_size FROM snapshots WHERE id = ?`
+	query := `
+		SELECT COALESCE(SUM(c.size), 0)
+		FROM (
+			SELECT DISTINCT fc.chunk_hash
+			FROM snapshot_files sf
+			JOIN file_chunks fc ON sf.file_id = fc.file_id
+			WHERE sf.snapshot_id = ?
+		) sc
+		JOIN chunks c ON sc.chunk_hash = c.chunk_hash
+	`
 
-	var uncompressedChunkSize int64
+	var totalSize int64
 
-	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&uncompressedChunkSize)
+	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
 	if err != nil {
 		return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
 	}
 
-	return uncompressedChunkSize, nil
+	return totalSize, nil
 }
 
 // GetSnapshotNewChunkSize returns the total plaintext size of the chunks the

internal/snapshot/module.go

@@ -16,6 +16,9 @@ type ScannerParams struct {
 	Fs             afero.Fs
 	Exclude        []string // Exclude patterns (combined global + snapshot-specific)
 	SkipErrors     bool     // Skip file read errors (log loudly but continue)
+	// PreviousSnapshotID is the most recent completed snapshot of the
+	// same name, "" for none.
+	PreviousSnapshotID string
 }
 
 // Module exports backup functionality as an fx module.
@@ -44,17 +47,18 @@ func provideScannerFactory(
 		}
 
 		return NewScanner(ScannerConfig{
-			FS:               params.Fs,
-			ChunkSize:        cfg.ChunkSize.Int64(),
-			Repositories:     repos,
-			Storage:          storer,
-			MaxBlobSize:      cfg.BlobSizeLimit.Int64(),
-			CompressionLevel: cfg.CompressionLevel,
-			AgeRecipients:    cfg.AgeRecipients,
-			EnableProgress:   params.EnableProgress,
-			UI:               params.UI,
-			Exclude:          excludes,
-			SkipErrors:       params.SkipErrors,
+			FS:                 params.Fs,
+			ChunkSize:          cfg.ChunkSize.Int64(),
+			Repositories:       repos,
+			Storage:            storer,
+			MaxBlobSize:        cfg.BlobSizeLimit.Int64(),
+			CompressionLevel:   cfg.CompressionLevel,
+			AgeRecipients:      cfg.AgeRecipients,
+			EnableProgress:     params.EnableProgress,
+			UI:                 params.UI,
+			Exclude:            excludes,
+			SkipErrors:         params.SkipErrors,
+			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/snapshot/scanner.go

@@ -58,6 +58,10 @@ type Scanner struct {
 	compressionLevel int
 	ageRecipient     string
 	snapshotID       string // Current snapshot being processed
+	// previousSnapshotID is the most recent completed snapshot of the
+	// same name, "" for none. Only a file it lists can be counted as
+	// deleted.
+	previousSnapshotID string
 	// currentSourcePath is the source directory being scanned, stored with
 	// each file record.
 	currentSourcePath string
@@ -129,6 +133,9 @@ type ScannerConfig struct {
 	// SkipErrors skips files that cannot be opened or read (log loudly but
 	// continue); packer, database, encryption, and upload errors still abort.
 	SkipErrors bool
+	// PreviousSnapshotID is the most recent completed snapshot of the
+	// same name, "" for none.
+	PreviousSnapshotID string
 }
 
 // ScanResult contains the results of a scan operation. Files and bytes
@@ -207,6 +214,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		compiledExclude:    compiledExclude,
 		progress:           progress,
 		skipErrors:         cfg.SkipErrors,
+		previousSnapshotID: cfg.PreviousSnapshotID,
 		ui:                 uiw,
 		pendingChunkHashes: make(map[string]struct{}),
 	}
@@ -258,8 +266,9 @@ func (s *Scanner) Scan(
 
 	filesToProcess := scanResult.FilesToProcess
 
-	// Phase 1b: Detect deleted files by comparing DB against scanned files
-	err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
+	// Phase 1b: Detect deleted files by comparing the previous snapshot
+	// of this name against scanned files
+	err = s.detectDeletedFiles(ctx, path, existingFiles, result)
 	if err != nil {
 		return nil, fmt.Errorf("detecting deleted files: %w", err)
 	}
@@ -439,9 +448,8 @@ func (s *Scanner) summarizeScanPhase(
 }
 
 // loadKnownFiles loads the known files at and beneath path from the
-// database into a map for fast lookup. Every loaded file the scan does
-// not find is counted as deleted. This avoids per-file database queries
-// during the scan phase.
+// database into a map for fast lookup. This avoids per-file database
+// queries during the scan phase.
 func (s *Scanner) loadKnownFiles(
 	ctx context.Context, path string,
 ) (map[string]*database.File, error) {
@@ -1954,21 +1962,28 @@ func (s *Scanner) queueFileForBatchInsert(
 	})
 }
 
-// detectDeletedFilesFromMap finds files that existed in previous snapshots
-// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
-// database access.
-func (s *Scanner) detectDeletedFilesFromMap(
+// detectDeletedFiles counts as deleted each file at or beneath path that
+// the previous snapshot of this name lists and the walk did not find.
+// A file that is excluded, or inside an excluded directory, is not
+// counted: the walk skips it whether or not it is still on disk.
+func (s *Scanner) detectDeletedFiles(
 	ctx context.Context,
-	knownFiles map[string]*database.File,
+	path string,
 	existingFiles map[string]struct{},
 	result *ScanResult,
 ) error {
-	if len(knownFiles) == 0 {
+	if s.previousSnapshotID == "" {
 		return nil
 	}
 
-	// Check each known file against the enumerated set (no filesystem access needed)
-	for path, file := range knownFiles {
+	previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
+		ctx, path, s.previousSnapshotID)
+	if err != nil {
+		return fmt.Errorf("listing files of snapshot %s: %w",
+			s.previousSnapshotID, err)
+	}
+
+	for _, file := range previousFiles {
 		// Check context cancellation periodically
 		select {
 		case <-ctx.Done():
@@ -1976,13 +1991,16 @@ func (s *Scanner) detectDeletedFilesFromMap(
 		default:
 		}
 
-		// Check if the file exists in our enumerated set
-		if _, exists := existingFiles[path]; !exists {
-			// File has been deleted
-			result.FilesDeleted++
-			result.BytesDeleted += file.Size
-			log.Debug("Detected deleted file", "path", path, "size", file.Size)
+		filePath := file.Path.String()
+
+		_, exists := existingFiles[filePath]
+		if exists || s.excludedOrInExcludedDir(filePath, path) {
+			continue
 		}
+
+		result.FilesDeleted++
+		result.BytesDeleted += file.Size
+		log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
 	}
 
 	if result.FilesDeleted > 0 {
@@ -1993,6 +2011,20 @@ func (s *Scanner) detectDeletedFilesFromMap(
 	return nil
 }
 
+// excludedOrInExcludedDir reports whether filePath, which is at or
+// beneath rootPath, or any directory between rootPath and it, matches an
+// exclude pattern. A pattern such as */.local/share/Trash matches the
+// directory but not the files inside it, which the walk skips too.
+func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
+	for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
+		if s.shouldExclude(p, rootPath) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // wrapPermissionError augments permission errors with platform-specific
 // remediation instructions. On macOS, TCC-protected directories (Calendars,
 // Reminders, Photos, etc.) return EPERM unless the running application has

internal/vaultik/deep_verify_test.go

@@ -1,9 +1,7 @@
 package vaultik_test
 
 import (
-	"bytes"
 	"context"
-	"crypto/rand"
 	"io"
 	"os"
 	"path/filepath"
@@ -11,9 +9,7 @@ import (
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
@@ -81,174 +77,6 @@ func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
 		"deep verify should fail on a corrupted blob")
 }
 
-// TestDeepVerifyJSONCountsBlobs checks the verified and missing counts
-// that deep verify reports under --json: on a healthy snapshot, with a
-// damaged blob, and with missing blobs. Deep verify stops at the first
-// damaged blob, so the blobs read before it are the ones verified; when
-// blobs are missing it reads no blob contents, but counts every missing
-// blob.
-func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "remote")
-
-	cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
-
-	healthy, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.NoError(t, err)
-	require.Equal(t, "ok", healthy.Status)
-	require.Len(t, keys, healthy.BlobCount)
-	require.Equal(t, healthy.BlobCount, healthy.Verified)
-	require.Zero(t, healthy.Missing)
-
-	// Damage the last blob read, keeping its length, so every blob before
-	// it is read and passes first.
-	lastPath := filepath.Join(storeDir, keys[len(keys)-1])
-	blob, err := afero.ReadFile(fs, lastPath)
-	require.NoError(t, err)
-
-	blob[len(blob)/2] ^= 0xff
-	require.NoError(t, afero.WriteFile(fs, lastPath, blob, 0o600))
-
-	got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Equal(t, healthy.BlobCount-1, got.Verified)
-	require.Zero(t, got.Missing)
-
-	// Delete the first blob and the damaged last one: both must be
-	// counted as missing, not only the first one found.
-	var deletedSize int64
-
-	for _, key := range []string{keys[0], keys[len(keys)-1]} {
-		info, statErr := storer.Stat(ctx, key)
-		require.NoError(t, statErr)
-
-		deletedSize += info.Size
-
-		require.NoError(t, storer.Delete(ctx, key))
-	}
-
-	got, err = deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Equal(t, 2, got.Missing)
-	require.Equal(t, deletedSize, got.MissingSize)
-	require.Zero(t, got.Verified)
-}
-
-// TestDeepVerifyJSONCountsWrongSizeBlob checks that under --deep --json a
-// blob of the wrong size is counted in mismatched and fails the run, and
-// that the check goes on past it to count a missing blob after it.
-func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "remote")
-
-	cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
-
-	// Make the first blob checked longer, and delete the last one.
-	firstPath := filepath.Join(storeDir, keys[0])
-	blob, err := afero.ReadFile(fs, firstPath)
-	require.NoError(t, err)
-
-	blob = append(blob, []byte("extra")...)
-	require.NoError(t, afero.WriteFile(fs, firstPath, blob, 0o600))
-	require.NoError(t, storer.Delete(ctx, keys[len(keys)-1]))
-
-	got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Equal(t, 1, got.Mismatched)
-	require.Equal(t, 1, got.Missing)
-	require.Zero(t, got.Verified)
-}
-
-// TestDeepVerifyStopsOnBlobThatCannotBeChecked checks that a blob that is
-// present in storage but cannot be checked is not counted as missing:
-// deep verify stops and fails with the storage error.
-func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "remote")
-
-	cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
-	failing := statFailingStorer{Storer: storer, failKeys: []string{keys[0]}}
-
-	got, err := deepVerifyJSON(ctx, t, cfg, failing, snapshotID)
-	require.Error(t, err)
-	require.Equal(t, "failed", got.Status)
-	require.Zero(t, got.Missing)
-	require.Contains(t, got.ErrorMessage, errStatDenied.Error())
-}
-
-// backUpSeveralBlobs backs up random data, which does not compress or
-// deduplicate, so it fills at least three blobs in a store on disk at
-// storeDir. It returns the blobs' storage keys sorted, which is hash
-// order, the order deep verify reads them in.
-func backUpSeveralBlobs(
-	ctx context.Context, t *testing.T, storeDir string,
-) (*config.Config, *storage.FileStorer, string, []string) {
-	t.Helper()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "source")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	chunkSize := int64(32 * 1024)
-	maxBlobSize := int64(128 * 1024)
-
-	data := make([]byte, maxBlobSize*4)
-	_, err := rand.Read(data)
-	require.NoError(t, err)
-	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
-	require.NoError(t, afero.WriteFile(fs,
-		filepath.Join(dataDir, "data.bin"), data, 0o644))
-
-	cfg, storer, snapshotID := runFileStorageBackup(
-		ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
-
-	keys := storedBlobKeys(t, storeDir)
-	require.GreaterOrEqual(t, len(keys), 3)
-
-	return cfg, storer, snapshotID, keys
-}
-
-// deepVerifyJSON runs deep verify with --json on snapshotID and returns
-// the decoded result and the error RunDeepVerify returned.
-func deepVerifyJSON(
-	ctx context.Context, t *testing.T,
-	cfg *config.Config, storer storage.Storer, snapshotID string,
-) (vaultik.VerifyResult, error) {
-	t.Helper()
-
-	var out bytes.Buffer
-
-	v := &vaultik.Vaultik{
-		Config:  cfg,
-		Storage: storer,
-		Fs:      afero.NewOsFs(),
-		Stdout:  &out,
-		Stderr:  io.Discard,
-		UI:      ui.NewWithColor(io.Discard, false),
-	}
-	v.SetContext(ctx)
-
-	err := v.RunDeepVerify(snapshotID,
-		&vaultik.VerifyOptions{Deep: true, JSON: true})
-
-	return decodeVerifyResult(t, out.Bytes()), err
-}
-
 // corruptOneBlob flips a middle byte of the first blob file found under
 // blobsDir, leaving the file length unchanged.
 func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {

internal/vaultik/fault_injection_test.go

@@ -661,17 +661,18 @@ func faultScannerFactory(
 ) snapshot.ScannerFactory {
 	return func(params snapshot.ScannerParams) *snapshot.Scanner {
 		return snapshot.NewScanner(snapshot.ScannerConfig{
-			FS:               params.Fs,
-			Storage:          storer,
-			ChunkSize:        faultChunkSize,
-			MaxBlobSize:      faultMaxBlobSize,
-			CompressionLevel: cfg.CompressionLevel,
-			AgeRecipients:    cfg.AgeRecipients,
-			Repositories:     repos,
-			EnableProgress:   params.EnableProgress,
-			UI:               params.UI,
-			Exclude:          params.Exclude,
-			SkipErrors:       params.SkipErrors,
+			FS:                 params.Fs,
+			Storage:            storer,
+			ChunkSize:          faultChunkSize,
+			MaxBlobSize:        faultMaxBlobSize,
+			CompressionLevel:   cfg.CompressionLevel,
+			AgeRecipients:      cfg.AgeRecipients,
+			Repositories:       repos,
+			EnableProgress:     params.EnableProgress,
+			UI:                 params.UI,
+			Exclude:            params.Exclude,
+			SkipErrors:         params.SkipErrors,
+			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/vaultik/snapshot.go

@@ -243,12 +243,18 @@ func (v *Vaultik) createNamedSnapshot(
 		return err
 	}
 
+	previousSnapshotID, err := v.previousSnapshotID(snapName)
+	if err != nil {
+		return err
+	}
+
 	scanner := v.ScannerFactory(snapshot.ScannerParams{
-		EnableProgress: !opts.Cron,
-		UI:             v.UI,
-		Fs:             v.Fs,
-		Exclude:        v.Config.GetExcludes(snapName),
-		SkipErrors:     opts.SkipErrors,
+		EnableProgress:     !opts.Cron,
+		UI:                 v.UI,
+		Fs:                 v.Fs,
+		Exclude:            v.Config.GetExcludes(snapName),
+		SkipErrors:         opts.SkipErrors,
+		PreviousSnapshotID: previousSnapshotID,
 	})
 
 	snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -283,6 +289,24 @@ func (v *Vaultik) createNamedSnapshot(
 	return nil
 }
 
+// previousSnapshotID returns the ID of the most recent completed snapshot
+// named snapName in the local index, or "" when there is none.
+func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
+	snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
+	if err != nil {
+		return "", fmt.Errorf("listing snapshots: %w", err)
+	}
+
+	for _, s := range snapshots {
+		if s.CompletedAt != nil &&
+			parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
+			return s.ID.String(), nil
+		}
+	}
+
+	return "", nil
+}
+
 // resolveSnapshotPaths resolves source directories to absolute paths
 // with symlink resolution.
 func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {

internal/vaultik/snapshot_deleted_count_test.go

@@ -0,0 +1,196 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
+// backup counts a file as deleted only when the previous snapshot of
+// the same name had it and it is gone, and never counts an excluded
+// file.
+
+// deletedCountEnv is a backup setup over the source directory src whose
+// user-facing output is kept in out. Each test configures its snapshots
+// with setSnapshot.
+type deletedCountEnv struct {
+	v   *vaultik.Vaultik
+	out *bytes.Buffer
+	src string
+}
+
+// newDeletedCountEnv writes files, keyed by path relative to src.
+func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
+	t.Helper()
+
+	ctx := context.Background()
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	srcDir := filepath.Join(tempDir, "src")
+
+	for rel, content := range files {
+		path := filepath.Join(srcDir, rel)
+		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
+		require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
+	}
+
+	cfg := faultTestConfig()
+	cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
+	cfg.Snapshots = map[string]config.SnapshotConfig{}
+
+	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, cfg.IndexPath)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = db.Close() })
+
+	repos := database.NewRepositories(db)
+	out := &bytes.Buffer{}
+	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+	v.UI = ui.NewWithColor(out, false)
+
+	return &deletedCountEnv{v: v, out: out, src: srcDir}
+}
+
+// setSnapshot configures the snapshot name to back up the path rel
+// beneath src, with the given excludes.
+func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
+	e.v.Config.Snapshots[name] = config.SnapshotConfig{
+		Paths:   []string{filepath.Join(e.src, rel)},
+		Exclude: exclude,
+	}
+}
+
+// backUp runs a backup of the named snapshot and returns its output.
+func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
+	t.Helper()
+
+	e.out.Reset()
+	require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+		Snapshots: []string{name},
+	}))
+
+	return e.out.String()
+}
+
+// A file this snapshot excludes is not counted as deleted: neither one
+// that another snapshot backs up, nor one that the previous snapshot of
+// this name had before the exclude was added.
+func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"top.txt":       []byte("top"),
+		"cache/tmp.bin": []byte("cache"),
+		"photos/a.jpg":  []byte("photo a"),
+		"photos/b.jpg":  []byte("photo b"),
+		"photos/c.jpg":  []byte("photo c"),
+	})
+	env.setSnapshot("photos", "photos")
+	env.setSnapshot("home", "", "/photos")
+
+	env.backUp(t, "photos")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
+
+	env.setSnapshot("home", "", "/photos", "/cache")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A file inside a directory that an exclude added after a backup
+// matches is not counted as deleted, though the pattern matches only the
+// directory.
+func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"top.txt":                         []byte("top"),
+		"user/.local/share/Trash/old.txt": []byte("old"),
+	})
+	env.setSnapshot("home", "")
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
+
+	env.setSnapshot("home", "", "*/.local/share/Trash")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A backup counts only files that the previous snapshot of its own name
+// had, not one that a later snapshot of another name over the same path
+// had and that is since removed.
+func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"kept.txt": []byte("kept"),
+	})
+	env.setSnapshot("home", "")
+	env.setSnapshot("other", "")
+
+	env.backUp(t, "home")
+
+	// Snapshot start times are kept in whole seconds: wait so that other's
+	// snapshot is strictly the most recent.
+	time.Sleep(time.Second)
+
+	extra := filepath.Join(env.src, "extra.txt")
+	require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
+	env.backUp(t, "other")
+	require.NoError(t, os.Remove(extra))
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A file removed from disk is counted as deleted by the next backup of
+// the snapshot that had it, and not by any later backup of that
+// snapshot, nor by the first backup of another name over the same path.
+func TestBackupCountsDeletedFileOnce(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	aContent := bytes.Repeat([]byte("a"), 1000)
+	bContent := bytes.Repeat([]byte("b"), 2000)
+	keptContent := bytes.Repeat([]byte("k"), 500)
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"a.txt":    aContent,
+		"b.txt":    bContent,
+		"kept.txt": keptContent,
+	})
+	env.setSnapshot("home", "")
+	env.setSnapshot("other", "")
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
+
+	require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
+	require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
+
+	summary := env.backUp(t, "home")
+	assert.Contains(t, summary,
+		"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
+	assert.Contains(t, summary,
+		fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
+			env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
+			env.v.UI.Size(int64(len(aContent)+len(bContent)))))
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+	assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
+}

internal/vaultik/snapshot_list_test.go

@@ -483,14 +483,13 @@ func TestListSnapshots_UnreadableManifestDoesNotHideOthers(t *testing.T) {
 //
 //nolint:tagliatelle // snake_case is the established output format
 type listJSONRow struct {
-	ID               string `json:"id"`
-	RemoteKey        string `json:"remote_key"`
-	Timestamp        string `json:"timestamp"`
-	CompressedSize   int64  `json:"compressed_size"`
-	UncompressedSize int64  `json:"uncompressed_size"`
-	NewChunkSize     int64  `json:"new_chunk_size"`
-	LocallyTracked   bool   `json:"locally_tracked"`
-	RemotePresent    *bool  `json:"remote_present"`
+	ID             string `json:"id"`
+	RemoteKey      string `json:"remote_key"`
+	Timestamp      string `json:"timestamp"`
+	CompressedSize int64  `json:"compressed_size"`
+	NewChunkSize   int64  `json:"new_chunk_size"`
+	LocallyTracked bool   `json:"locally_tracked"`
+	RemotePresent  *bool  `json:"remote_present"`
 }
 
 // decodeListJSON parses the command's stdout, which must contain

internal/vaultik/uncompressed_size_test.go

@@ -1,85 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// TestUncompressedSizeOfChangedFileStaysFixed covers
-// https://git.eeqj.de/sneak/vaultik/issues/360. A backup that changes a
-// file leaves the earlier snapshot's UNCOMPRESSED SIZE as it was when that
-// snapshot's backup completed. The file's row in the local index is shared
-// by both snapshots and rewritten by the second backup.
-func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-	changedPath := filepath.Join(dataDir, "changed.bin")
-
-	ctx := context.Background()
-	files := writeFaultSourceTree(t, fs, dataDir)
-	cfg := changedFileConfig(dataDir, dbPath)
-
-	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
-
-	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	repos := database.NewRepositories(db)
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-
-	require.NoError(t, backUp(v, "first"))
-
-	// With nothing to compare against, sizeOfNewChunks totals every
-	// distinct chunk of the files.
-	firstID := localSnapshotID(ctx, t, repos, "first")
-	firstSize := sizeOfNewChunks(t, files, nil)
-	require.Equal(t, firstSize, listedUncompressedSizes(t, v)[firstID],
-		"the first snapshot must count the chunks of every file it holds")
-
-	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
-	require.NoError(t, backUp(v, "second"))
-
-	secondID := localSnapshotID(ctx, t, repos, "second")
-	sizes := listedUncompressedSizes(t, v)
-	assert.Equal(t, firstSize, sizes[firstID],
-		"a later backup must not change the first snapshot's figure")
-	assert.Equal(t, sizeOfNewChunks(t, files, nil), sizes[secondID],
-		"the second snapshot must count the changed file's chunks")
-}
-
-// listedUncompressedSizes runs `snapshot list --json` and returns each
-// listed snapshot's uncompressed_size by snapshot ID.
-func listedUncompressedSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
-	t.Helper()
-
-	var out bytes.Buffer
-
-	v.Stdout = &out
-	require.NoError(t, v.ListSnapshots(true))
-
-	sizes := make(map[string]int64)
-	for _, row := range decodeListJSON(t, out.String()) {
-		sizes[row.ID] = row.UncompressedSize
-	}
-
-	return sizes
-}

internal/vaultik/verify.go

@@ -17,7 +17,6 @@ import (
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
-	"sneak.berlin/go/vaultik/internal/storage"
 )
 
 // Sentinel errors for snapshot verification failures.
@@ -33,9 +32,7 @@ var (
 	errManifestExtraBlob   = errors.New("manifest contains blob not in database")
 	errManifestMissingBlob = errors.New(
 		"manifest omits blob present in database")
-	errBlobSizeMismatch  = errors.New("blob size mismatch")
-	errBlobsNotInStorage = errors.New(
-		"blobs missing from storage or stored with the wrong size")
+	errBlobSizeMismatch = errors.New("blob size mismatch")
 )
 
 // verifyStatusFailed is the JSON status value for a failed verification.
@@ -152,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
 	}
 
 	result.Status = "ok"
+	result.Verified = len(dbBlobs)
 
 	if opts.JSON {
 		return v.outputVerifyJSON(result)
@@ -308,7 +306,7 @@ func (v *Vaultik) runVerificationSteps(
 		v.stdoutf("Checking blob existence in remote storage...\n")
 	}
 
-	err = v.verifyBlobExistenceFromDB(dbBlobs, result)
+	err = v.verifyBlobExistenceFromDB(dbBlobs)
 	if err != nil {
 		return v.deepVerifyFailure(result, opts, err.Error(), err)
 	}
@@ -319,8 +317,7 @@ func (v *Vaultik) runVerificationSteps(
 			len(dbBlobs), ubytes(totalSize))
 	}
 
-	err = v.performDeepVerificationFromDB(
-		dbBlobs, tdb.db.Conn(), opts, result, identities)
+	err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identities)
 	if err != nil {
 		return v.deepVerifyFailure(result, opts, err.Error(), err)
 	}
@@ -672,14 +669,8 @@ func (v *Vaultik) verifyManifestAgainstDatabase(
 	return nil
 }
 
-// verifyBlobExistenceFromDB checks that every blob from the database is
-// in storage with the size the database records. It checks them all,
-// counting the missing and wrong-size blobs in result, and fails if it
-// found any. Any storage error other than not found stops the check and
-// fails with that error.
-func (v *Vaultik) verifyBlobExistenceFromDB(
-	blobs []snapshot.BlobInfo, result *VerifyResult,
-) error {
+// verifyBlobExistenceFromDB checks that all blobs from database exist in S3
+func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
 	log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
 
 	for i, blob := range blobs {
@@ -692,23 +683,17 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
 		// Construct blob path
 		blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
 
+		// Check blob exists
 		stat, err := v.Storage.Stat(v.ctx, blobPath)
-		switch {
-		case errors.Is(err, storage.ErrNotFound):
-			log.Warn("Blob missing from storage", "hash", blob.Hash)
+		if err != nil {
+			return fmt.Errorf("blob %s missing from storage: %w", blob.Hash, err)
+		}
 
-			result.Missing++
-			result.MissingSize += blob.CompressedSize
-		case err != nil:
-			return fmt.Errorf("checking blob %s in storage: %w",
-				blob.Hash, err)
-		case stat.Size != blob.CompressedSize:
-			log.Warn("Blob has the wrong size in storage",
-				"hash", blob.Hash,
-				"storage_size", stat.Size,
-				"database_size", blob.CompressedSize)
-
-			result.Mismatched++
+		// Verify size matches
+		if stat.Size != blob.CompressedSize {
+			return fmt.Errorf(
+				"%w: blob %s: S3 has %d bytes, database has %d bytes",
+				errBlobSizeMismatch, blob.Hash, stat.Size, blob.CompressedSize)
 		}
 
 		// Progress update every 100 blobs
@@ -722,22 +707,16 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
 		}
 	}
 
-	if result.Missing > 0 || result.Mismatched > 0 {
-		return fmt.Errorf("%w: %d missing, %d with the wrong size",
-			errBlobsNotInStorage, result.Missing, result.Mismatched)
-	}
-
 	log.Info("✓ All blobs exist in storage")
 
 	return nil
 }
 
 // performDeepVerificationFromDB downloads and verifies the content of
-// each blob using the database as source, stopping at the first blob that
-// fails. Each blob that passes is counted in result.Verified.
+// each blob using the database as source.
 func (v *Vaultik) performDeepVerificationFromDB(
 	blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
-	result *VerifyResult, identities []age.Identity,
+	identities []age.Identity,
 ) error {
 	// Calculate total bytes for ETA
 	var totalBytesExpected int64
@@ -760,7 +739,6 @@ func (v *Vaultik) performDeepVerificationFromDB(
 			return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
 		}
 
-		result.Verified++
 		bytesProcessed += blobInfo.CompressedSize
 		elapsed := time.Since(startTime)
 		remaining := len(blobs) - (i + 1)