Record the stored size of a blob a backup finds already stored (closes #300)

When a finished blob already existed at the destination, the upload was
skipped but the blob row, and so the manifest, kept the size of the
local copy. That copy can differ from the stored object when
compression_level or the recipients changed since it was written, and
then shallow and deep verify failed on a good snapshot. Marking a blob
uploaded now records the size of the object in storage: for an existing
blob, the size Stat reports.

Judgement call: the test adds a recipient rather than changing the
compression level, because an extra age recipient always makes the
local copy larger, while two zstd levels need not compress the test
data differently.

Model: opus-5-5

docs/DATAMODEL.md

@@ -90,7 +90,7 @@ Stores information about packed, compressed, and encrypted blob files.
 - `created_ts` (INTEGER NOT NULL) - Creation timestamp
 - `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress)
 - `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before compression
-- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption
+- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption. Once uploaded, the size of the object in storage: when the blob was already stored, under another compression level or other recipients, this can differ from the size of the local copy
 - `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded)
 
 **Purpose:** Tracks blob lifecycle from creation through upload. The UUID primary key allows immediate association of chunks with blobs.
@@ -183,7 +183,7 @@ Tracks blob upload metrics.
    - `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after packing
 
 4. **Upload**
-   - `UPDATE blobs SET uploaded_ts = ?` - Mark blob as uploaded
+   - `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as uploaded, with the size of the object in storage (the upload is skipped when the blob is already stored)
    - `INSERT INTO uploads` - Record upload metrics with snapshot_id
    - `INSERT INTO snapshot_blobs` - Associate blob with snapshot
 

internal/database/blobs.go

@@ -159,13 +159,16 @@ func (r *BlobRepository) UpdateFinished(
 	return nil
 }
 
-// UpdateUploaded marks a blob as uploaded
+// UpdateUploaded marks a blob as uploaded and records the size of the
+// stored object as its compressed size. That size can differ from the
+// local copy's when an identical blob was already stored with another
+// compression level or other recipients.
 func (r *BlobRepository) UpdateUploaded(
-	ctx context.Context, tx *sql.Tx, id string,
+	ctx context.Context, tx *sql.Tx, id string, storedSize int64,
 ) error {
 	query := `
-		UPDATE blobs 
-		SET uploaded_ts = ?
+		UPDATE blobs
+		SET uploaded_ts = ?, compressed_size = ?
 		WHERE id = ?
 	`
 
@@ -173,9 +176,9 @@ func (r *BlobRepository) UpdateUploaded(
 
 	var err error
 	if tx != nil {
-		_, err = tx.ExecContext(ctx, query, now, id)
+		_, err = tx.ExecContext(ctx, query, now, storedSize, id)
 	} else {
-		_, err = r.db.ExecWithLog(ctx, query, now, id)
+		_, err = r.db.ExecWithLog(ctx, query, now, storedSize, id)
 	}
 
 	if err != nil {

internal/database/blobs_test.go

@@ -122,8 +122,8 @@ func TestBlobRepositoryUpdates(t *testing.T) {
 		t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
 	}
 
-	// Test UpdateUploaded
-	err = repo.UpdateUploaded(ctx, nil, blob.ID.String())
+	// Test UpdateUploaded, with a stored object larger than the local copy
+	err = repo.UpdateUploaded(ctx, nil, blob.ID.String(), 600)
 	if err != nil {
 		t.Fatalf("failed to update blob as uploaded: %v", err)
 	}
@@ -141,6 +141,10 @@ func TestBlobRepositoryUpdates(t *testing.T) {
 	if uploaded.UploadedTS.Before(now.Add(-1 * time.Second)) {
 		t.Error("uploaded timestamp should be around test time")
 	}
+
+	if uploaded.CompressedSize != 600 {
+		t.Errorf("expected the stored size 600, got %d", uploaded.CompressedSize)
+	}
 }
 
 func TestBlobRepositoryDuplicate(t *testing.T) {

internal/log/tty_handler.go

@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 	"unicode"
-
-	"github.com/dustin/go-humanize"
 )
 
 // groupSeparator joins an open group path to an attribute key. This
@@ -306,13 +304,18 @@ func formatDuration(d time.Duration) string {
 	}
 }
 
-// formatBytes formats a byte count with humanize.Bytes, as every command
-// prints one. humanize.Bytes takes no negative count, so one is printed
-// as it is.
+// formatBytes formats bytes in a human-readable way
 func formatBytes(b int64) string {
-	if b < 0 {
+	const unit = 1024
+	if b < unit {
 		return fmt.Sprintf("%d B", b)
 	}
 
-	return humanize.Bytes(uint64(b))
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
 }

internal/log/tty_handler_test.go

@@ -200,8 +200,7 @@ func TestTTYHandlerWithGroupQualifiesKeys(t *testing.T) {
 func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
 	t.Parallel()
 
-	// Printed as "1.0 MB", in powers of 1000 like every other byte count.
-	const oneMegabyte = 1_000_000
+	const oneAndAHalfKiB = 1536
 
 	for name, testCase := range map[string]struct {
 		derive func(*slog.Logger) *slog.Logger
@@ -224,13 +223,13 @@ func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
 			var buf bytes.Buffer
 
 			logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
-			testCase.derive(logger).Info("uploaded", "bytes", oneMegabyte)
+			testCase.derive(logger).Info("uploaded", "bytes", oneAndAHalfKiB)
 
 			line := ansiEscape.ReplaceAllString(buf.String(), "")
 
-			assert.Contains(t, line, testCase.key+"=1.0 MB",
+			assert.Contains(t, line, testCase.key+"=1.5 KB",
 				"a byte count must be human-readable however it is qualified")
-			assert.NotContains(t, line, strconv.Itoa(oneMegabyte),
+			assert.NotContains(t, line, strconv.Itoa(oneAndAHalfKiB),
 				"the raw number must not survive the formatting")
 		})
 	}

internal/snapshot/scanner.go

@@ -1514,7 +1514,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
 			// snapshot references has uploaded_ts set, so deduplication and
 			// interrupted-run repair treat these blobs as trustworthy.
 			err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
-				err := s.repos.Blobs.UpdateUploaded(ctx, tx, b.ID)
+				err := s.repos.Blobs.UpdateUploaded(
+					ctx, tx, b.ID, b.Compressed)
 				if err != nil {
 					return fmt.Errorf("marking blob uploaded: %w", err)
 				}
@@ -1550,7 +1551,7 @@ func (s *Scanner) handleBlobReady(
 	blobPath := fmt.Sprintf("blobs/%s/%s/%s",
 		finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
 
-	blobExists, err := s.uploadBlobIfNeeded(
+	storedSize, blobExists, err := s.uploadBlobIfNeeded(
 		ctx, blobPath, blobWithReader, startTime, result)
 	if err != nil {
 		s.cleanupBlobTempFile(blobWithReader)
@@ -1558,7 +1559,8 @@ func (s *Scanner) handleBlobReady(
 		return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
 	}
 
-	err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime)
+	err = s.recordBlobMetadata(
+		ctx, finishedBlob, storedSize, blobExists, startTime)
 	if err != nil {
 		s.cleanupBlobTempFile(blobWithReader)
 
@@ -1580,20 +1582,22 @@ func (s *Scanner) handleBlobReady(
 }
 
 // uploadBlobIfNeeded uploads the blob to storage if it doesn't already
-// exist, returns whether it existed
+// exist. It returns the size of the object in storage, which for an
+// existing object can differ from the local copy's, and whether it
+// already existed.
 func (s *Scanner) uploadBlobIfNeeded(
 	ctx context.Context,
 	blobPath string,
 	blobWithReader *blob.WithReader,
 	startTime time.Time,
 	result *ScanResult,
-) (bool, error) {
+) (int64, bool, error) {
 	finishedBlob := blobWithReader.FinishedBlob
 
 	// Check if blob already exists (deduplication after restart)
 	destination := s.storage.Info().Location
 
-	_, err := s.storage.Stat(ctx, blobPath)
+	stored, err := s.storage.Stat(ctx, blobPath)
 	if err == nil {
 		log.Info("Blob already exists in storage, skipping upload",
 			"hash", finishedBlob.Hash,
@@ -1602,7 +1606,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 			s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
 			s.ui.Path(destination))
 
-		return true, nil
+		return stored.Size, true, nil
 	}
 
 	s.ui.Beginf("Uploading blob %s (%s) to %s.",
@@ -1616,7 +1620,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 	if err != nil {
 		log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
 
-		return false, fmt.Errorf("uploading blob to storage: %w", err)
+		return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
 	}
 
 	uploadDuration := time.Since(startTime)
@@ -1647,7 +1651,7 @@ func (s *Scanner) uploadBlobIfNeeded(
 		stats.BytesUploaded.Add(finishedBlob.Compressed)
 	}
 
-	return false, nil
+	return finishedBlob.Compressed, false, nil
 }
 
 // makeUploadProgressCallback creates a progress callback for blob uploads.
@@ -1717,6 +1721,7 @@ func (s *Scanner) makeUploadProgressCallback(
 func (s *Scanner) recordBlobMetadata(
 	ctx context.Context,
 	finishedBlob *blob.FinishedBlob,
+	storedSize int64,
 	blobExists bool,
 	startTime time.Time,
 ) error {
@@ -1728,7 +1733,8 @@ func (s *Scanner) recordBlobMetadata(
 	uploadDuration := time.Since(startTime)
 
 	return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
-		err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
+		err := s.repos.Blobs.UpdateUploaded(
+			txCtx, tx, finishedBlob.ID, storedSize)
 		if err != nil {
 			return fmt.Errorf("updating blob upload timestamp: %w", err)
 		}

internal/vaultik/helpers.go

@@ -89,6 +89,22 @@ type SnapshotInfo struct {
 	RemotePresent    *bool            `json:"remote_present"`
 }
 
+// formatBytes formats bytes in a human-readable format
+func formatBytes(bytes int64) string {
+	const unit = 1024
+	if bytes < unit {
+		return fmt.Sprintf("%d B", bytes)
+	}
+
+	div, exp := int64(unit), 0
+	for n := bytes / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+
+	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
+}
+
 // parseSnapshotTimestamp extracts the timestamp from a snapshot ID
 // Format: hostname_snapshotname_2026-01-12T14:41:15Z
 func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {

internal/vaultik/reused_blob_verify_test.go

@@ -0,0 +1,83 @@
+package vaultik_test
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+	"testing"
+
+	"filippo.io/age"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestVerifyPassesWhenBackupReusesStoredBlob backs up a file, then backs it
+// up again from an empty local index, as after `database delete`, with an
+// extra age recipient. The second backup builds the same blob, finds it
+// already stored and skips the upload, but its own copy is larger, because
+// age writes a header entry for each recipient. The second snapshot must
+// record the size of the stored object, so shallow and deep verify pass.
+func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+
+	dataDir := filepath.Join(tempDir, "source")
+	storeDir := filepath.Join(tempDir, "remote")
+	secondDBPath := filepath.Join(tempDir, "second.sqlite")
+
+	chunkSize := int64(64 * 1024)
+	maxBlobSize := int64(512 * 1024)
+
+	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
+	require.NoError(t, afero.WriteFile(fs,
+		filepath.Join(dataDir, "data.bin"),
+		bytesPattern("reuse-", int(chunkSize*3)), 0o644))
+
+	ctx := context.Background()
+
+	runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
+		filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
+
+	env := setupDedupBackupEnv(
+		ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
+
+	defer func() { _ = env.db.Close() }()
+
+	extra, err := age.GenerateX25519Identity()
+	require.NoError(t, err)
+
+	env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
+		extra.Recipient().String())
+
+	snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
+		env.cfg.Hostname, "second", dataDir, secondDBPath)
+	require.Positive(t, result.BlobsCreated)
+	require.Zero(t, result.BlobsUploaded,
+		"the second backup should find every blob already stored")
+
+	require.NoError(t, env.db.Close())
+
+	verifier := &vaultik.Vaultik{
+		Config:  env.cfg,
+		Storage: env.storer,
+		Fs:      fs,
+		Stdout:  io.Discard,
+		Stderr:  io.Discard,
+		UI:      ui.NewWithColor(io.Discard, false),
+	}
+	verifier.SetContext(ctx)
+
+	require.NoError(t,
+		verifier.VerifySnapshotWithOptions(snapshotID, &vaultik.VerifyOptions{}),
+		"shallow verify should pass")
+	require.NoError(t,
+		verifier.VerifySnapshotWithOptions(snapshotID,
+			&vaultik.VerifyOptions{Deep: true}),
+		"deep verify should pass")
+}

internal/vaultik/snapshot.go

@@ -633,7 +633,7 @@ func (v *Vaultik) confirmAndExecutePurge(
 			v.stdoutf("  %s (%s, %s)\n",
 				snap.ID,
 				snap.Timestamp.Format("2006-01-02 15:04:05"),
-				ubytes(snap.CompressedSize))
+				formatBytes(snap.CompressedSize))
 		}
 	}
 

internal/vaultik/snapshot_list.go

@@ -553,8 +553,8 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
 
 		if snap.LocallyTracked {
 			id = snap.ID.String()
-			uncompressed = ubytes(snap.UncompressedSize)
-			newChunks = ubytes(snap.NewChunkSize)
+			uncompressed = formatBytes(snap.UncompressedSize)
+			newChunks = formatBytes(snap.NewChunkSize)
 		} else {
 			id = formatRemoteOnlyID(snap.RemoteKey)
 			uncompressed = remoteOnlyCell
@@ -564,7 +564,7 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
 		_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
 			id,
 			snap.Timestamp.Format("2006-01-02 15:04:05"),
-			ubytes(snap.CompressedSize),
+			formatBytes(snap.CompressedSize),
 			uncompressed,
 			newChunks)
 		if err != nil {

internal/vaultik/snapshot_list_test.go

@@ -212,8 +212,8 @@ const (
 	listConfiguredName = "home"
 	listLocalID        = "testhost_home_2026-03-01T10:00:00Z"
 	listRemoteID       = "otherhost_media_2026-03-02T11:22:33Z"
-	// fiveMegabytes is printed as "5.0 MB" by every command.
-	fiveMegabytes = 5_000_000
+	// fiveMegabytes formats as "5.0 MB" through formatBytes.
+	fiveMegabytes = 5 * 1024 * 1024
 )
 
 // TestListSnapshots_RemoteWithoutSecretKey is the regression guard for
@@ -306,48 +306,6 @@ func TestListSnapshots_RemoteOnlyRowRendering(t *testing.T) {
 		"are not in the local index")
 }
 
-// TestSnapshotListAndPurgePrintSizesLikeRemoteInfo checks that the
-// `snapshot list` table and the purge listing print a byte count the way
-// `remote info` does: in powers of 1000, as the MB label says.
-func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	const wantSize = "5.0 MB" // fiveMegabytes
-
-	env := newListEnv(t)
-
-	// A remote-only snapshot: `snapshot list` and `remote info` both take
-	// its size from the manifest.
-	remoteKey := env.addRemote(t, listRemoteID,
-		time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC))
-
-	// A locally tracked snapshot of the same size for the purge to list.
-	// The purge prints the snapshot's blob size, the last argument here.
-	start := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
-	env.addLocal(t, listLocalID, start)
-	env.addRemote(t, listLocalID, start)
-	require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
-		context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
-
-	require.NoError(t, env.v.RemoteInfo(false))
-	assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
-		wantSize, "remote info")
-
-	env.stdout.Reset()
-	require.NoError(t, env.v.ListSnapshots(false))
-	assert.Contains(t, findTableRow(t, env.stdout.String(),
-		"<remote only:"+remoteKey[:12]+">"), wantSize, "snapshot list")
-
-	// Without --force and with nothing on stdin the purge lists what it
-	// would delete, reads no confirmation, and deletes nothing.
-	env.stdout.Reset()
-	require.NoError(t, env.v.PurgeSnapshotsWithOptions(
-		&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
-	assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
-		wantSize, "snapshot purge")
-}
-
 // findTableRow returns the single output line containing needle.
 func findTableRow(t *testing.T, out, needle string) string {
 	t.Helper()