From 3cb1cf9e9fe401238fa025da57eb2a1f96b8000e Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 13 Oct 2026 14:51:08 +0000 Subject: [PATCH] Record each snapshot's uncompressed size when its backup runs (closes #360) snapshot list summed the chunks a snapshot reaches through its file rows in the local index. A file's row is shared by every snapshot that lists it and rewritten when the file changes, so backing up a changed file changed every older snapshot's UNCOMPRESSED SIZE. The backup now takes that total when it records the snapshot's stats, into a new snapshots column, uncompressed_chunk_size, and snapshot list reads the column, as it already does for new_chunk_size. Model: opus-5-5 --- README.md | 3 +- docs/DATAMODEL.md | 1 + internal/database/schema/001.sql | 1 + internal/database/snapshots.go | 40 +++++----- internal/vaultik/snapshot_list_test.go | 15 ++-- internal/vaultik/uncompressed_size_test.go | 85 ++++++++++++++++++++++ 6 files changed, 120 insertions(+), 25 deletions(-) create mode 100644 internal/vaultik/uncompressed_size_test.go diff --git a/README.md b/README.md index f4c5f61..da522ea 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,8 @@ needed. of the local index and the backup destination store — with timestamps and three sizes per snapshot (compressed remote size; total uncompressed chunk size; size of the chunks that snapshot stored that -were not stored before, recorded when its backup ran). +were not stored before). Both chunk sizes are recorded when the +snapshot's backup runs. Listing the destination store does **not** require the age secret key, so it works in vaultik's intended configuration, where the backed-up diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index b3191ac..47ae087 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -122,6 +122,7 @@ Tracks backup snapshots. - `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE - `blob_count` (INTEGER) - Number of blobs this snapshot created - `total_size` (INTEGER) - Total size of all files +- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the distinct chunks the snapshot's files reference, recorded when its backup ran; `snapshot list` shows it as UNCOMPRESSED SIZE - `blob_size` (INTEGER) - Total compressed size of all referenced blobs - `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs - `compression_ratio` (REAL) - Compression ratio achieved diff --git a/internal/database/schema/001.sql b/internal/database/schema/001.sql index 0b022c2..9645b56 100644 --- a/internal/database/schema/001.sql +++ b/internal/database/schema/001.sql @@ -89,6 +89,7 @@ CREATE TABLE IF NOT EXISTS snapshots ( new_chunk_size INTEGER NOT NULL DEFAULT 0, blob_count INTEGER NOT NULL DEFAULT 0, total_size INTEGER NOT NULL DEFAULT 0, + uncompressed_chunk_size INTEGER NOT NULL DEFAULT 0, blob_size INTEGER NOT NULL DEFAULT 0, blob_uncompressed_size INTEGER NOT NULL DEFAULT 0, compression_ratio REAL NOT NULL DEFAULT 1.0, diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index c9c609a..d94dede 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -120,7 +120,11 @@ func (r *SnapshotRepository) UpdateCounts( return nil } -// UpdateExtendedStats updates extended statistics for a snapshot +// UpdateExtendedStats updates extended statistics for a snapshot. It also +// records the total plaintext size of the distinct chunks the snapshot's +// files reference. Only the snapshot's own backup can take that total: a +// file's row is shared by every snapshot that lists it, and a later +// backup of a changed file rewrites the row with the new chunks. func (r *SnapshotRepository) UpdateExtendedStats( ctx context.Context, tx *sql.Tx, @@ -145,7 +149,17 @@ func (r *SnapshotRepository) UpdateExtendedStats( compression_level = ?, upload_bytes = ?, upload_duration_ms = ?, - new_chunk_size = ? + new_chunk_size = ?, + uncompressed_chunk_size = ( + SELECT COALESCE(SUM(c.size), 0) + FROM ( + SELECT DISTINCT fc.chunk_hash + FROM snapshot_files sf + JOIN file_chunks fc ON sf.file_id = fc.file_id + WHERE sf.snapshot_id = snapshots.id + ) sc + JOIN chunks c ON sc.chunk_hash = c.chunk_hash + ) WHERE id = ? ` @@ -570,30 +584,22 @@ func (r *SnapshotRepository) GetSnapshotBlobSizes( return compressed, uncompressed, nil } -// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique -// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks). +// GetSnapshotUncompressedChunkSize returns the total plaintext size of the +// distinct chunks the snapshot's files reference, as the snapshot's backup +// recorded it (see UpdateExtendedStats). func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize( ctx context.Context, snapshotID string, ) (int64, error) { - query := ` - SELECT COALESCE(SUM(c.size), 0) - FROM ( - SELECT DISTINCT fc.chunk_hash - FROM snapshot_files sf - JOIN file_chunks fc ON sf.file_id = fc.file_id - WHERE sf.snapshot_id = ? - ) sc - JOIN chunks c ON sc.chunk_hash = c.chunk_hash - ` + query := `SELECT uncompressed_chunk_size FROM snapshots WHERE id = ?` - var totalSize int64 + var uncompressedChunkSize int64 - err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) + err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&uncompressedChunkSize) if err != nil { return 0, fmt.Errorf("querying uncompressed chunk size: %w", err) } - return totalSize, nil + return uncompressedChunkSize, nil } // GetSnapshotNewChunkSize returns the total plaintext size of the chunks the diff --git a/internal/vaultik/snapshot_list_test.go b/internal/vaultik/snapshot_list_test.go index 3beb371..2dd88e4 100644 --- a/internal/vaultik/snapshot_list_test.go +++ b/internal/vaultik/snapshot_list_test.go @@ -483,13 +483,14 @@ func TestListSnapshots_UnreadableManifestDoesNotHideOthers(t *testing.T) { // //nolint:tagliatelle // snake_case is the established output format type listJSONRow struct { - ID string `json:"id"` - RemoteKey string `json:"remote_key"` - Timestamp string `json:"timestamp"` - CompressedSize int64 `json:"compressed_size"` - NewChunkSize int64 `json:"new_chunk_size"` - LocallyTracked bool `json:"locally_tracked"` - RemotePresent *bool `json:"remote_present"` + ID string `json:"id"` + RemoteKey string `json:"remote_key"` + Timestamp string `json:"timestamp"` + CompressedSize int64 `json:"compressed_size"` + UncompressedSize int64 `json:"uncompressed_size"` + NewChunkSize int64 `json:"new_chunk_size"` + LocallyTracked bool `json:"locally_tracked"` + RemotePresent *bool `json:"remote_present"` } // decodeListJSON parses the command's stdout, which must contain diff --git a/internal/vaultik/uncompressed_size_test.go b/internal/vaultik/uncompressed_size_test.go new file mode 100644 index 0000000..f9b77c1 --- /dev/null +++ b/internal/vaultik/uncompressed_size_test.go @@ -0,0 +1,85 @@ +package vaultik_test + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/database" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// TestUncompressedSizeOfChangedFileStaysFixed covers +// https://git.eeqj.de/sneak/vaultik/issues/360. A backup that changes a +// file leaves the earlier snapshot's UNCOMPRESSED SIZE as it was when that +// snapshot's backup completed. The file's row in the local index is shared +// by both snapshots and rewritten by the second backup. +func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "src") + dbPath := filepath.Join(tempDir, "index.sqlite") + changedPath := filepath.Join(dataDir, "changed.bin") + + ctx := context.Background() + files := writeFaultSourceTree(t, fs, dataDir) + cfg := changedFileConfig(dataDir, dbPath) + + appendRandomBytes(t, fs, files, changedPath, appendedFileSize) + + store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote")) + require.NoError(t, err) + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + repos := database.NewRepositories(db) + v := newBackupVaultik(ctx, cfg, store, repos, db, fs) + + require.NoError(t, backUp(v, "first")) + + // With nothing to compare against, sizeOfNewChunks totals every + // distinct chunk of the files. + firstID := localSnapshotID(ctx, t, repos, "first") + firstSize := sizeOfNewChunks(t, files, nil) + require.Equal(t, firstSize, listedUncompressedSizes(t, v)[firstID], + "the first snapshot must count the chunks of every file it holds") + + appendRandomBytes(t, fs, files, changedPath, faultChunkSize) + require.NoError(t, backUp(v, "second")) + + secondID := localSnapshotID(ctx, t, repos, "second") + sizes := listedUncompressedSizes(t, v) + assert.Equal(t, firstSize, sizes[firstID], + "a later backup must not change the first snapshot's figure") + assert.Equal(t, sizeOfNewChunks(t, files, nil), sizes[secondID], + "the second snapshot must count the changed file's chunks") +} + +// listedUncompressedSizes runs `snapshot list --json` and returns each +// listed snapshot's uncompressed_size by snapshot ID. +func listedUncompressedSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 { + t.Helper() + + var out bytes.Buffer + + v.Stdout = &out + require.NoError(t, v.ListSnapshots(true)) + + sizes := make(map[string]int64) + for _, row := range decodeListJSON(t, out.String()) { + sizes[row.ID] = row.UncompressedSize + } + + return sizes +}