Record each snapshot's uncompressed size when its backup runs (closes #360)

snapshot list summed the chunks a snapshot reaches through its file rows
in the local index. A file's row is shared by every snapshot that lists
it and rewritten when the file changes, so backing up a changed file
changed every older snapshot's UNCOMPRESSED SIZE.

The backup now takes that total when it records the snapshot's stats,
into a new snapshots column, uncompressed_chunk_size, and snapshot list
reads the column, as it already does for new_chunk_size.

Model: opus-5-5

README.md

@@ -294,7 +294,8 @@ needed.
 of the local index and the backup destination store — with timestamps
 and three sizes per snapshot (compressed remote size; total
 uncompressed chunk size; size of the chunks that snapshot stored that
-were not stored before, recorded when its backup ran).
+were not stored before). Both chunk sizes are recorded when the
+snapshot's backup runs.
 
 Listing the destination store does **not** require the age secret key,
 so it works in vaultik's intended configuration, where the backed-up

docs/DATAMODEL.md

@@ -122,6 +122,7 @@ Tracks backup snapshots.
 - `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE
 - `blob_count` (INTEGER) - Number of blobs this snapshot created
 - `total_size` (INTEGER) - Total size of all files
+- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the distinct chunks the snapshot's files reference, recorded when its backup ran; `snapshot list` shows it as UNCOMPRESSED SIZE
 - `blob_size` (INTEGER) - Total compressed size of all referenced blobs
 - `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
 - `compression_ratio` (REAL) - Compression ratio achieved

internal/database/schema/001.sql

@@ -89,6 +89,7 @@ CREATE TABLE IF NOT EXISTS snapshots (
     new_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_count INTEGER NOT NULL DEFAULT 0,
     total_size INTEGER NOT NULL DEFAULT 0,
+    uncompressed_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_size INTEGER NOT NULL DEFAULT 0,
     blob_uncompressed_size INTEGER NOT NULL DEFAULT 0,
     compression_ratio REAL NOT NULL DEFAULT 1.0,

internal/database/snapshots.go

@@ -120,7 +120,11 @@ func (r *SnapshotRepository) UpdateCounts(
 	return nil
 }
 
-// UpdateExtendedStats updates extended statistics for a snapshot
+// UpdateExtendedStats updates extended statistics for a snapshot. It also
+// records the total plaintext size of the distinct chunks the snapshot's
+// files reference. Only the snapshot's own backup can take that total: a
+// file's row is shared by every snapshot that lists it, and a later
+// backup of a changed file rewrites the row with the new chunks.
 func (r *SnapshotRepository) UpdateExtendedStats(
 	ctx context.Context,
 	tx *sql.Tx,
@@ -145,7 +149,17 @@ func (r *SnapshotRepository) UpdateExtendedStats(
 		    compression_level = ?,
 		    upload_bytes = ?,
 		    upload_duration_ms = ?,
-		    new_chunk_size = ?
+		    new_chunk_size = ?,
+		    uncompressed_chunk_size = (
+		        SELECT COALESCE(SUM(c.size), 0)
+		        FROM (
+		            SELECT DISTINCT fc.chunk_hash
+		            FROM snapshot_files sf
+		            JOIN file_chunks fc ON sf.file_id = fc.file_id
+		            WHERE sf.snapshot_id = snapshots.id
+		        ) sc
+		        JOIN chunks c ON sc.chunk_hash = c.chunk_hash
+		    )
 		WHERE id = ?
 	`
 
@@ -555,30 +569,22 @@ func (r *SnapshotRepository) GetSnapshotBlobSizes(
 	return compressed, uncompressed, nil
 }
 
-// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
-// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
+// GetSnapshotUncompressedChunkSize returns the total plaintext size of the
+// distinct chunks the snapshot's files reference, as the snapshot's backup
+// recorded it (see UpdateExtendedStats).
 func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
 	ctx context.Context, snapshotID string,
 ) (int64, error) {
-	query := `
-		SELECT COALESCE(SUM(c.size), 0)
-		FROM (
-			SELECT DISTINCT fc.chunk_hash
-			FROM snapshot_files sf
-			JOIN file_chunks fc ON sf.file_id = fc.file_id
-			WHERE sf.snapshot_id = ?
-		) sc
-		JOIN chunks c ON sc.chunk_hash = c.chunk_hash
-	`
+	query := `SELECT uncompressed_chunk_size FROM snapshots WHERE id = ?`
 
-	var totalSize int64
+	var uncompressedChunkSize int64
 
-	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
+	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&uncompressedChunkSize)
 	if err != nil {
 		return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
 	}
 
-	return totalSize, nil
+	return uncompressedChunkSize, nil
 }
 
 // GetSnapshotNewChunkSize returns the total plaintext size of the chunks the

internal/vaultik/snapshot_list_test.go

@@ -483,13 +483,14 @@ func TestListSnapshots_UnreadableManifestDoesNotHideOthers(t *testing.T) {
 //
 //nolint:tagliatelle // snake_case is the established output format
 type listJSONRow struct {
-	ID             string `json:"id"`
-	RemoteKey      string `json:"remote_key"`
-	Timestamp      string `json:"timestamp"`
-	CompressedSize int64  `json:"compressed_size"`
-	NewChunkSize   int64  `json:"new_chunk_size"`
-	LocallyTracked bool   `json:"locally_tracked"`
-	RemotePresent  *bool  `json:"remote_present"`
+	ID               string `json:"id"`
+	RemoteKey        string `json:"remote_key"`
+	Timestamp        string `json:"timestamp"`
+	CompressedSize   int64  `json:"compressed_size"`
+	UncompressedSize int64  `json:"uncompressed_size"`
+	NewChunkSize     int64  `json:"new_chunk_size"`
+	LocallyTracked   bool   `json:"locally_tracked"`
+	RemotePresent    *bool  `json:"remote_present"`
 }
 
 // decodeListJSON parses the command's stdout, which must contain

internal/vaultik/uncompressed_size_test.go

@@ -0,0 +1,85 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestUncompressedSizeOfChangedFileStaysFixed covers
+// https://git.eeqj.de/sneak/vaultik/issues/360. A backup that changes a
+// file leaves the earlier snapshot's UNCOMPRESSED SIZE as it was when that
+// snapshot's backup completed. The file's row in the local index is shared
+// by both snapshots and rewritten by the second backup.
+func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	dataDir := filepath.Join(tempDir, "src")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+	changedPath := filepath.Join(dataDir, "changed.bin")
+
+	ctx := context.Background()
+	files := writeFaultSourceTree(t, fs, dataDir)
+	cfg := changedFileConfig(dataDir, dbPath)
+
+	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
+
+	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, dbPath)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = db.Close() })
+
+	repos := database.NewRepositories(db)
+	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+
+	require.NoError(t, backUp(v, "first"))
+
+	// With nothing to compare against, sizeOfNewChunks totals every
+	// distinct chunk of the files.
+	firstID := localSnapshotID(ctx, t, repos, "first")
+	firstSize := sizeOfNewChunks(t, files, nil)
+	require.Equal(t, firstSize, listedUncompressedSizes(t, v)[firstID],
+		"the first snapshot must count the chunks of every file it holds")
+
+	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
+	require.NoError(t, backUp(v, "second"))
+
+	secondID := localSnapshotID(ctx, t, repos, "second")
+	sizes := listedUncompressedSizes(t, v)
+	assert.Equal(t, firstSize, sizes[firstID],
+		"a later backup must not change the first snapshot's figure")
+	assert.Equal(t, sizeOfNewChunks(t, files, nil), sizes[secondID],
+		"the second snapshot must count the changed file's chunks")
+}
+
+// listedUncompressedSizes runs `snapshot list --json` and returns each
+// listed snapshot's uncompressed_size by snapshot ID.
+func listedUncompressedSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
+	t.Helper()
+
+	var out bytes.Buffer
+
+	v.Stdout = &out
+	require.NoError(t, v.ListSnapshots(true))
+
+	sizes := make(map[string]int64)
+	for _, row := range decodeListJSON(t, out.String()) {
+		sizes[row.ID] = row.UncompressedSize
+	}
+
+	return sizes
+}