Record the stored size of a blob a backup finds already stored (closes #300)

When a finished blob already existed at the destination, the upload was
skipped but the blob row, and so the manifest, kept the size of the
local copy. That copy differs from the stored object when
compression_level or the recipients changed since it was written, so
shallow and deep verify failed on a good snapshot. Marking a blob
uploaded now records the size of the object in storage: for an existing
blob, the size Stat reports.

Judgement call: the test adds a recipient rather than changing the
compression level, because an extra age recipient always makes the
local copy larger, while two zstd levels need not compress the test
data differently.

Model: opus-5-5

README.md

@@ -370,9 +370,7 @@ on the destination in one go, use `vaultik remote nuke --force`.
   store — its remote key (or an unambiguous leading part of it) as shown
   by `snapshot list`. See
   [restoring on another machine](#restoring-on-another-machine).
-* Optional path arguments to restore specific files/directories (default: all).
-  A path that matches nothing in the snapshot is named in a warning, and the
-  command exits non-zero after restoring the paths that match
+* Optional path arguments to restore specific files/directories (default: all)
 * Preserves file permissions, timestamps, ownership (ownership requires root),
   symlinks, and empty directories
 * `--verify`: After restoring, verify every file's chunk hashes match

TODO.md

@@ -22,6 +22,16 @@ the tag exists and is exercised; what is left is merging `next` to
 
 # Completed Steps
 
+- 2026-10-13: Recorded the size of the stored object for a blob a backup
+  found already stored
+  ([issue #300](https://git.eeqj.de/sneak/vaultik/issues/300)). When the
+  upload was skipped, the blob row and the manifest kept the size of the
+  local copy, which differs from the stored object's when
+  `compression_level` or the recipients changed since it was written.
+  `snapshot verify` then failed on a good snapshot, shallow and deep,
+  and the listed sizes were wrong. Marking the blob uploaded now records
+  the size of the object in storage.
+
 - 2026-10-13: Made `snapshot remove` fail for a snapshot that does not
   exist ([issue #293](https://git.eeqj.de/sneak/vaultik/issues/293)).
   The command never checked the ID: it reported the snapshot removed

docs/DATAMODEL.md

@@ -90,7 +90,7 @@ Stores information about packed, compressed, and encrypted blob files.
 - `created_ts` (INTEGER NOT NULL) - Creation timestamp
 - `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress)
 - `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before compression
-- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption
+- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption. Once uploaded, the size of the object in storage: when the blob was already stored, under another compression level or other recipients, this differs from the size of the local copy
 - `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded)
 
 **Purpose:** Tracks blob lifecycle from creation through upload. The UUID primary key allows immediate association of chunks with blobs.
@@ -183,7 +183,7 @@ Tracks blob upload metrics.
    - `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after packing
 
 4. **Upload**
-   - `UPDATE blobs SET uploaded_ts = ?` - Mark blob as uploaded
+   - `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as uploaded, with the size of the object in storage (the upload is skipped when the blob is already stored)
    - `INSERT INTO uploads` - Record upload metrics with snapshot_id
    - `INSERT INTO snapshot_blobs` - Associate blob with snapshot
 

internal/cli/snapshot_restore.go

@@ -30,8 +30,6 @@ This command will restore files from the specified snapshot to the
 target directory.
 If no paths are specified, all files are restored.
 If paths are specified, only matching files/directories are restored.
-A path that matches nothing in the snapshot is named in a warning, and
-the command exits non-zero after restoring the paths that match.
 
 The snapshot may be named by its ID or, when restoring on a host with no
 local index, by the remote key that 'snapshot list' prints for a

internal/database/blobs.go

@@ -159,13 +159,16 @@ func (r *BlobRepository) UpdateFinished(
 	return nil
 }
 
-// UpdateUploaded marks a blob as uploaded
+// UpdateUploaded marks a blob as uploaded and records the size of the
+// stored object as its compressed size. That size can differ from the
+// local copy's when an identical blob was already stored with another
+// compression level or other recipients.
 func (r *BlobRepository) UpdateUploaded(
-	ctx context.Context, tx *sql.Tx, id string,
+	ctx context.Context, tx *sql.Tx, id string, storedSize int64,
 ) error {
 	query := `
-		UPDATE blobs 
-		SET uploaded_ts = ?
+		UPDATE blobs
+		SET uploaded_ts = ?, compressed_size = ?
 		WHERE id = ?
 	`
 
@@ -173,9 +176,9 @@ func (r *BlobRepository) UpdateUploaded(
 
 	var err error
 	if tx != nil {
-		_, err = tx.ExecContext(ctx, query, now, id)
+		_, err = tx.ExecContext(ctx, query, now, storedSize, id)
 	} else {
-		_, err = r.db.ExecWithLog(ctx, query, now, id)
+		_, err = r.db.ExecWithLog(ctx, query, now, storedSize, id)
 	}
 
 	if err != nil {

internal/database/blobs_test.go

@@ -122,8 +122,8 @@ func TestBlobRepositoryUpdates(t *testing.T) {
 		t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
 	}
 
-	// Test UpdateUploaded
-	err = repo.UpdateUploaded(ctx, nil, blob.ID.String())
+	// Test UpdateUploaded, with a stored object larger than the local copy
+	err = repo.UpdateUploaded(ctx, nil, blob.ID.String(), 600)
 	if err != nil {
 		t.Fatalf("failed to update blob as uploaded: %v", err)
 	}
@@ -141,6 +141,10 @@ func TestBlobRepositoryUpdates(t *testing.T) {
 	if uploaded.UploadedTS.Before(now.Add(-1 * time.Second)) {
 		t.Error("uploaded timestamp should be around test time")
 	}
+
+	if uploaded.CompressedSize != 600 {
+		t.Errorf("expected the stored size 600, got %d", uploaded.CompressedSize)
+	}
 }
 
 func TestBlobRepositoryDuplicate(t *testing.T) {

internal/snapshot/scanner.go

@@ -1509,7 +1509,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
 			// snapshot references has uploaded_ts set, so deduplication and
 			// interrupted-run repair treat these blobs as trustworthy.
 			err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
-				err := s.repos.Blobs.UpdateUploaded(ctx, tx, b.ID)
+				err := s.repos.Blobs.UpdateUploaded(
+					ctx, tx, b.ID, b.Compressed)
 				if err != nil {
 					return fmt.Errorf("marking blob uploaded: %w", err)
 				}
@@ -1545,7 +1546,7 @@ func (s *Scanner) handleBlobReady(
 	blobPath := fmt.Sprintf("blobs/%s/%s/%s",
 		finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
 
-	blobExists, err := s.uploadBlobIfNeeded(
+	storedSize, blobExists, err := s.uploadBlobIfNeeded(
 		ctx, blobPath, blobWithReader, startTime, result)
 	if err != nil {
 		s.cleanupBlobTempFile(blobWithReader)
@@ -1553,7 +1554,8 @@ func (s *Scanner) handleBlobReady(
 		return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
 	}
 
-	err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime)
+	err = s.recordBlobMetadata(
+		ctx, finishedBlob, storedSize, blobExists, startTime)
 	if err != nil {
 		s.cleanupBlobTempFile(blobWithReader)
 
@@ -1575,20 +1577,22 @@ func (s *Scanner) handleBlobReady(
 }
 
 // uploadBlobIfNeeded uploads the blob to storage if it doesn't already
-// exist, returns whether it existed
+// exist. It returns the size of the object in storage, which for an
+// existing object can differ from the local copy's, and whether it
+// already existed.
 func (s *Scanner) uploadBlobIfNeeded(
 	ctx context.Context,
 	blobPath string,
 	blobWithReader *blob.WithReader,
 	startTime time.Time,
 	result *ScanResult,
-) (bool, error) {
+) (int64, bool, error) {
 	finishedBlob := blobWithReader.FinishedBlob
 
 	// Check if blob already exists (deduplication after restart)
 	destination := s.storage.Info().Location
 
-	_, err := s.storage.Stat(ctx, blobPath)
+	stored, err := s.storage.Stat(ctx, blobPath)
 	if err == nil {
 		log.Info("Blob already exists in storage, skipping upload",
 			"hash", finishedBlob.Hash,
@@ -1597,7 +1601,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 			s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
 			s.ui.Path(destination))
 
-		return true, nil
+		return stored.Size, true, nil
 	}
 
 	s.ui.Beginf("Uploading blob %s (%s) to %s.",
@@ -1611,7 +1615,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 	if err != nil {
 		log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
 
-		return false, fmt.Errorf("uploading blob to storage: %w", err)
+		return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
 	}
 
 	uploadDuration := time.Since(startTime)
@@ -1642,7 +1646,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 		stats.BytesUploaded.Add(finishedBlob.Compressed)
 	}
 
-	return false, nil
+	return finishedBlob.Compressed, false, nil
 }
 
 // makeUploadProgressCallback creates a progress callback for blob uploads.
@@ -1712,6 +1716,7 @@ func (s *Scanner) makeUploadProgressCallback(
 func (s *Scanner) recordBlobMetadata(
 	ctx context.Context,
 	finishedBlob *blob.FinishedBlob,
+	storedSize int64,
 	blobExists bool,
 	startTime time.Time,
 ) error {
@@ -1723,7 +1728,8 @@ func (s *Scanner) recordBlobMetadata(
 	uploadDuration := time.Since(startTime)
 
 	return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
-		err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
+		err := s.repos.Blobs.UpdateUploaded(
+			txCtx, tx, finishedBlob.ID, storedSize)
 		if err != nil {
 			return fmt.Errorf("updating blob upload timestamp: %w", err)
 		}

internal/vaultik/restore.go

@@ -27,7 +27,6 @@ import (
 // Sentinel errors for restore failures.
 var (
 	errFilesFailedRestore    = errors.New("file(s) failed to restore")
-	errPathsNotInSnapshot    = errors.New("requested path(s) not in the snapshot")
 	errFilesFailedVerify     = errors.New("files failed verification")
 	errDecryptionKeyRequired = errors.New(
 		"decryption key required for restore\n\n" +
@@ -150,7 +149,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 	repos := database.NewRepositories(tempDB)
 
 	// Step 2: Get list of files to restore
-	files, unmatched, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
+	files, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
 	if err != nil {
 		return fmt.Errorf("getting files to restore: %w", err)
 	}
@@ -159,7 +158,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 		log.Warn("No files found to restore")
 		v.UI.Warningf("No files found to restore.")
 
-		return v.reportUnmatchedPaths(unmatched)
+		return nil
 	}
 
 	log.Info("Found files to restore", "count", len(files))
@@ -199,16 +198,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 		v.UI.Duration(result.Duration),
 	)
 
-	return v.finishRestore(repos, files, unmatched, opts, result)
+	return v.finishRestore(repos, files, opts, result)
 }
 
 // finishRestore emits the post-restore warnings, runs optional
-// verification, and converts any failed-file count or requested path
-// that matched nothing into an error.
+// verification, and converts any failed-file count into an error.
 func (v *Vaultik) finishRestore(
 	repos *database.Repositories,
 	files []*database.File,
-	unmatched []string,
 	opts *RestoreOptions,
 	result *RestoreResult,
 ) error {
@@ -226,8 +223,6 @@ func (v *Vaultik) finishRestore(
 		}
 	}
 
-	unmatchedErr := v.reportUnmatchedPaths(unmatched)
-
 	// Run verification if requested
 	if opts.Verify {
 		err := v.handleRestoreVerification(repos, files, opts, result)
@@ -240,23 +235,7 @@ func (v *Vaultik) finishRestore(
 		return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
 	}
 
-	return unmatchedErr
-}
-
-// reportUnmatchedPaths warns, by name, about each requested path that
-// matched nothing in the snapshot, and returns an error counting them,
-// or nil when there are none.
-func (v *Vaultik) reportUnmatchedPaths(unmatched []string) error {
-	if len(unmatched) == 0 {
-		return nil
-	}
-
-	for _, path := range unmatched {
-		log.Warn("Requested path matches nothing in the snapshot", "path", path)
-		v.UI.Warningf("%s matches nothing in the snapshot.", v.UI.Path(path))
-	}
-
-	return fmt.Errorf("%d %w", len(unmatched), errPathsNotInSnapshot)
+	return nil
 }
 
 // restoreIdentities parses the configured age secret key once into every
@@ -833,34 +812,27 @@ func (v *Vaultik) materializeSnapshotDB(
 	return db, tempDir, nil
 }
 
-// getFilesToRestore returns the list of files to restore based on path
-// filters, and the filters, as given, that matched no file.
+// getFilesToRestore returns the list of files to restore based on path filters
 func (v *Vaultik) getFilesToRestore(
 	ctx context.Context, repos *database.Repositories, pathFilters []string,
-) ([]*database.File, []string, error) {
+) ([]*database.File, error) {
 	// If no filters, get all files
 	if len(pathFilters) == 0 {
-		files, err := repos.Files.ListAll(ctx)
-
-		return files, nil, err
+		return repos.Files.ListAll(ctx)
 	}
 
 	// Get files matching the path filters
-	var (
-		result    []*database.File
-		unmatched []string
-	)
+	var result []*database.File
 
 	seen := make(map[string]bool)
 
 	for _, filter := range pathFilters {
-		files, err := repos.Files.ListUnderPath(ctx, filepath.Clean(filter))
-		if err != nil {
-			return nil, nil, fmt.Errorf("listing files under %s: %w", filter, err)
-		}
+		// Normalize the filter path
+		filter = filepath.Clean(filter)
 
-		if len(files) == 0 {
-			unmatched = append(unmatched, filter)
+		files, err := repos.Files.ListUnderPath(ctx, filter)
+		if err != nil {
+			return nil, fmt.Errorf("listing files under %s: %w", filter, err)
 		}
 
 		for _, file := range files {
@@ -871,7 +843,7 @@ func (v *Vaultik) getFilesToRestore(
 		}
 	}
 
-	return result, unmatched, nil
+	return result, nil
 }
 
 // buildChunkToBlobMap creates a mapping from chunk hash to blob information

internal/vaultik/restore_unmatched_path_test.go

@@ -1,142 +0,0 @@
-package vaultik_test
-
-import (
-	"bytes"
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/ui"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// These tests cover https://git.eeqj.de/sneak/vaultik/issues/309: a path
-// given to restore that matches nothing in the snapshot is named in a
-// warning and makes the restore fail, after the paths that do match are
-// restored.
-
-func TestRestoreNamesUnmatchedPathAndRestoresTheMatchingOne(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	storeDir := filepath.Join(tempDir, "remote")
-	restoreDir := filepath.Join(tempDir, "restored")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	ctx := context.Background()
-	cfg := faultTestConfig()
-	files := writeFaultSourceTree(t, fs, dataDir)
-
-	store, err := storage.NewFileStorer(storeDir)
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-
-	id := fullFaultBackup(ctx, t, fs, store, cfg,
-		database.NewRepositories(db), dataDir, dbPath, "unmatched")
-	require.NoError(t, db.Close())
-
-	matched := filepath.Join(dataDir, "small.txt")
-	unmatched := filepath.Join(dataDir, "typo.txt")
-
-	var out bytes.Buffer
-
-	v := newReaderVaultik(ctx, cfg, store, nil, fs)
-	v.UI = ui.NewWithColor(&out, false)
-
-	err = v.Restore(&vaultik.RestoreOptions{
-		SnapshotID: id,
-		TargetDir:  restoreDir,
-		Paths:      []string{matched, unmatched},
-	})
-
-	require.Error(t, err, "a path that matches nothing must fail the restore")
-	assert.Contains(t, out.String(), unmatched,
-		"the path that matches nothing must be named")
-	assert.NotContains(t, out.String(), matched,
-		"the path that matches must not be reported")
-
-	got, err := afero.ReadFile(fs, filepath.Join(restoreDir, matched))
-	require.NoError(t, err, "the path that matches must still be restored")
-	assert.Equal(t, files[matched], got)
-}
-
-// When no requested path matches, nothing is restored and each path is
-// named. One path is relative. The other goes through the symlink that
-// is the snapshot's configured path, while the entries are stored under
-// the directory the symlink points to.
-func TestRestoreNamesEveryPathWhenNothingMatches(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	const snapshotName = "linked"
-
-	fs := afero.NewOsFs()
-	tempDir := t.TempDir()
-	dataDir := filepath.Join(tempDir, "src")
-	linkDir := filepath.Join(tempDir, "srclink")
-	storeDir := filepath.Join(tempDir, "remote")
-	restoreDir := filepath.Join(tempDir, "restored")
-	dbPath := filepath.Join(tempDir, "index.sqlite")
-
-	ctx := context.Background()
-	writeFaultSourceTree(t, fs, dataDir)
-	require.NoError(t, os.Symlink(dataDir, linkDir))
-
-	cfg := faultTestConfig()
-	cfg.IndexPath = dbPath
-	cfg.ChunkSize = config.Size(faultChunkSize)
-	cfg.Snapshots = map[string]config.SnapshotConfig{
-		snapshotName: {Paths: []string{linkDir}},
-	}
-
-	store, err := storage.NewFileStorer(storeDir)
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, dbPath)
-	require.NoError(t, err)
-
-	repos := database.NewRepositories(db)
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-
-	require.NoError(t, backUp(v, snapshotName))
-
-	id := localSnapshotID(ctx, t, repos, snapshotName)
-	require.NoError(t, db.Close())
-
-	throughLink := filepath.Join(linkDir, "sub")
-	relative := strings.TrimPrefix(filepath.Join(dataDir, "small.txt"), "/")
-
-	var out bytes.Buffer
-
-	reader := newReaderVaultik(ctx, cfg, store, nil, fs)
-	reader.UI = ui.NewWithColor(&out, false)
-
-	err = reader.Restore(&vaultik.RestoreOptions{
-		SnapshotID: id,
-		TargetDir:  restoreDir,
-		Paths:      []string{throughLink, relative},
-	})
-
-	require.Error(t, err, "a restore that matches nothing must fail")
-	assert.Contains(t, out.String(), throughLink,
-		"the path through the symlink must be named")
-	assert.Contains(t, out.String(), relative,
-		"the relative path must be named")
-
-	_, err = fs.Stat(restoreDir)
-	assert.True(t, os.IsNotExist(err), "nothing must be restored")
-}

internal/vaultik/reused_blob_verify_test.go

@@ -0,0 +1,83 @@
+package vaultik_test
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+	"testing"
+
+	"filippo.io/age"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestVerifyPassesWhenBackupReusesStoredBlob backs up a file, then backs it
+// up again from an empty local index, as after `database delete`, with an
+// extra age recipient. The second backup builds the same blob, finds it
+// already stored and skips the upload, but its own copy is larger, because
+// age writes a header entry for each recipient. The second snapshot must
+// record the size of the stored object, so shallow and deep verify pass.
+func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+
+	dataDir := filepath.Join(tempDir, "source")
+	storeDir := filepath.Join(tempDir, "remote")
+	secondDBPath := filepath.Join(tempDir, "second.sqlite")
+
+	chunkSize := int64(64 * 1024)
+	maxBlobSize := int64(512 * 1024)
+
+	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
+	require.NoError(t, afero.WriteFile(fs,
+		filepath.Join(dataDir, "data.bin"),
+		bytesPattern("reuse-", int(chunkSize*3)), 0o644))
+
+	ctx := context.Background()
+
+	runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
+		filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
+
+	env := setupDedupBackupEnv(
+		ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
+
+	defer func() { _ = env.db.Close() }()
+
+	extra, err := age.GenerateX25519Identity()
+	require.NoError(t, err)
+
+	env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
+		extra.Recipient().String())
+
+	snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
+		env.cfg.Hostname, "second", dataDir, secondDBPath)
+	require.Positive(t, result.BlobsCreated)
+	require.Zero(t, result.BlobsUploaded,
+		"the second backup should find every blob already stored")
+
+	require.NoError(t, env.db.Close())
+
+	verifier := &vaultik.Vaultik{
+		Config:  env.cfg,
+		Storage: env.storer,
+		Fs:      fs,
+		Stdout:  io.Discard,
+		Stderr:  io.Discard,
+		UI:      ui.NewWithColor(io.Discard, false),
+	}
+	verifier.SetContext(ctx)
+
+	require.NoError(t,
+		verifier.VerifySnapshotWithOptions(snapshotID, &vaultik.VerifyOptions{}),
+		"shallow verify should pass")
+	require.NoError(t,
+		verifier.VerifySnapshotWithOptions(snapshotID,
+			&vaultik.VerifyOptions{Deep: true}),
+		"deep verify should pass")
+}