diff --git a/README.md b/README.md index 355322d..f4c5f61 100644 --- a/README.md +++ b/README.md @@ -293,8 +293,8 @@ needed. **`snapshot list`**: Show every snapshot known to this host — the union of the local index and the backup destination store — with timestamps and three sizes per snapshot (compressed remote size; total -uncompressed chunk size; size of chunks newly referenced by that -snapshot). +uncompressed chunk size; size of the chunks that snapshot stored that +were not stored before, recorded when its backup ran). Listing the destination store does **not** require the age secret key, so it works in vaultik's intended configuration, where the backed-up @@ -307,8 +307,8 @@ only the hash, so hostname and snapshot name exist solely in the local index and in the encrypted per-snapshot database. Snapshots found only on the destination store are therefore listed as `>` and show `` in -the uncompressed and "new chunk" columns, which can only be computed -from the local index. Their timestamp and compressed size are real, +the uncompressed and "new chunk" columns, which come only from the +local index. Their timestamp and compressed size are real, read from the manifest. Snapshots in the local index with no counterpart on the destination diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index bcb9d1c..b3191ac 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -119,6 +119,7 @@ Tracks backup snapshots. - `completed_at` (INTEGER) - Completion timestamp (NULL if in progress) - `file_count` (INTEGER) - Number of files in snapshot - `chunk_count` (INTEGER) - Number of chunks this snapshot stored that were not stored before +- `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE - `blob_count` (INTEGER) - Number of blobs this snapshot created - `total_size` (INTEGER) - Total size of all files - `blob_size` (INTEGER) - Total compressed size of all referenced blobs diff --git a/internal/database/schema/001.sql b/internal/database/schema/001.sql index c07d978..0b022c2 100644 --- a/internal/database/schema/001.sql +++ b/internal/database/schema/001.sql @@ -86,6 +86,7 @@ CREATE TABLE IF NOT EXISTS snapshots ( completed_at INTEGER, file_count INTEGER NOT NULL DEFAULT 0, chunk_count INTEGER NOT NULL DEFAULT 0, + new_chunk_size INTEGER NOT NULL DEFAULT 0, blob_count INTEGER NOT NULL DEFAULT 0, total_size INTEGER NOT NULL DEFAULT 0, blob_size INTEGER NOT NULL DEFAULT 0, diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index 09d3b3a..c9c609a 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -129,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats( compressionLevel int, uploadBytes int64, uploadDurationMs int64, + newChunkSize int64, ) error { compressionRatio, err := r.extendedCompressionRatio( ctx, tx, snapshotID, blobUncompressedSize, @@ -143,18 +144,19 @@ func (r *SnapshotRepository) UpdateExtendedStats( compression_ratio = ?, compression_level = ?, upload_bytes = ?, - upload_duration_ms = ? + upload_duration_ms = ?, + new_chunk_size = ? WHERE id = ? ` if tx != nil { _, err = tx.ExecContext(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, - uploadBytes, uploadDurationMs, snapshotID) + uploadBytes, uploadDurationMs, newChunkSize, snapshotID) } else { _, err = r.db.ExecWithLog(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, - uploadBytes, uploadDurationMs, snapshotID) + uploadBytes, uploadDurationMs, newChunkSize, snapshotID) } if err != nil { @@ -594,45 +596,24 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize( return totalSize, nil } -// GetSnapshotNewChunkSize returns the sum of plaintext sizes of chunks that are -// referenced by this snapshot but not by any earlier completed snapshot known to -// the local database. The result is the marginal uncompressed data this snapshot -// added to the dedup pool — i.e., the delta from prior snapshots. +// GetSnapshotNewChunkSize returns the total plaintext size of the chunks the +// snapshot's backup stored that were not stored before, as the backup +// recorded it. It is not derived from the snapshot's files: a file's row is +// shared by every snapshot that lists it and is rewritten when the file +// changes, so it shows only the file's latest chunks. func (r *SnapshotRepository) GetSnapshotNewChunkSize( ctx context.Context, snapshotID string, ) (int64, error) { - query := ` - WITH this_snap_chunks AS ( - SELECT DISTINCT fc.chunk_hash - FROM snapshot_files sf - JOIN file_chunks fc ON sf.file_id = fc.file_id - WHERE sf.snapshot_id = ? - ), - prior_chunks AS ( - SELECT DISTINCT fc.chunk_hash - FROM snapshots s - JOIN snapshot_files sf ON sf.snapshot_id = s.id - JOIN file_chunks fc ON fc.file_id = sf.file_id - WHERE s.completed_at IS NOT NULL - AND s.id != ? - AND s.started_at < (SELECT started_at FROM snapshots WHERE id = ?) - ) - SELECT COALESCE(SUM(c.size), 0) - FROM chunks c - JOIN this_snap_chunks t ON c.chunk_hash = t.chunk_hash - WHERE c.chunk_hash NOT IN (SELECT chunk_hash FROM prior_chunks) - ` + query := `SELECT new_chunk_size FROM snapshots WHERE id = ?` - var totalSize int64 + var newChunkSize int64 - err := r.db.conn.QueryRowContext( - ctx, query, snapshotID, snapshotID, snapshotID, - ).Scan(&totalSize) + err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&newChunkSize) if err != nil { return 0, fmt.Errorf("querying new chunk size: %w", err) } - return totalSize, nil + return newChunkSize, nil } // GetIncompleteSnapshots returns all snapshots that haven't been completed diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index e54ee1a..32441cf 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -135,7 +135,9 @@ type ScannerConfig struct { // are counted per file: BytesScanned is the size of the new and changed // files, BytesSkipped that of the unchanged ones. FilesFailed counts the // new and changed files that phase 2 could not store; FilesScanned -// includes them and BytesScanned does not. +// includes them and BytesScanned does not. ChunksCreated counts the +// chunks that were not already stored, and NewChunkSize is their total +// size. type ScanResult struct { FilesScanned int FilesSkipped int @@ -145,6 +147,7 @@ type ScanResult struct { BytesSkipped int64 BytesDeleted int64 ChunksCreated int + NewChunkSize int64 BlobsCreated int BlobsUploaded int BytesUploaded int64 @@ -1866,11 +1869,12 @@ func (s *Scanner) processFileStreaming( return nil } -// updateChunkStats counts a chunk that was not already stored. The scan -// result's file counts, BytesScanned and BytesSkipped are not touched -// here: the scan phase counts each file once. +// updateChunkStats counts a chunk that was not already stored, and its +// size. The scan result's file counts, BytesScanned and BytesSkipped are +// not touched here: the scan phase counts each file once. func (s *Scanner) updateChunkStats(chunkSize int64, result *ScanResult) { result.ChunksCreated++ + result.NewChunkSize += chunkSize if s.progress != nil { s.progress.GetStats().ChunksCreated.Add(1) diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index cdf641e..a959930 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -199,6 +199,7 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended( stats.CompressionLevel, stats.BytesUploaded, stats.UploadDurationMs, + stats.NewChunkSize, ) }) } @@ -906,6 +907,7 @@ type ExtendedBackupStats struct { BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs CompressionLevel int // Compression level used for this snapshot UploadDurationMs int64 // Total milliseconds spent uploading to S3 + NewChunkSize int64 // Total size of the chunks counted in ChunksCreated } // deleteOtherSnapshots deletes all snapshots except the current one diff --git a/internal/vaultik/new_chunk_size_test.go b/internal/vaultik/new_chunk_size_test.go new file mode 100644 index 0000000..5301334 --- /dev/null +++ b/internal/vaultik/new_chunk_size_test.go @@ -0,0 +1,133 @@ +package vaultik_test + +import ( + "bytes" + "context" + "maps" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/chunker" + "sneak.berlin/go/vaultik/internal/database" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// TestNewChunkSizeOfChangedFileStaysFixed covers +// https://git.eeqj.de/sneak/vaultik/issues/311. A backup whose only change +// is a modified file lists the size of the chunks that change added as its +// NEW CHUNK SIZE, and a later backup that changes the file again leaves +// every earlier snapshot's figure as it was. The file's row in the local +// index is shared by all three snapshots and rewritten by each backup. +func TestNewChunkSizeOfChangedFileStaysFixed(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "src") + dbPath := filepath.Join(tempDir, "index.sqlite") + changedPath := filepath.Join(dataDir, "changed.bin") + + ctx := context.Background() + files := writeFaultSourceTree(t, fs, dataDir) + cfg := changedFileConfig(dataDir, dbPath) + + appendRandomBytes(t, fs, files, changedPath, appendedFileSize) + + store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote")) + require.NoError(t, err) + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + repos := database.NewRepositories(db) + v := newBackupVaultik(ctx, cfg, store, repos, db, fs) + + require.NoError(t, backUp(v, "first")) + + firstFiles := maps.Clone(files) + firstID := localSnapshotID(ctx, t, repos, "first") + + appendRandomBytes(t, fs, files, changedPath, faultChunkSize) + require.NoError(t, backUp(v, "second")) + + secondID := localSnapshotID(ctx, t, repos, "second") + secondAdded := sizeOfNewChunks(t, files, firstFiles) + require.Positive(t, secondAdded) + assert.Equal(t, secondAdded, listedNewChunkSizes(t, v)[secondID], + "the second snapshot must count the chunks the change added") + + appendRandomBytes(t, fs, files, changedPath, faultChunkSize) + require.NoError(t, backUp(v, "third")) + + sizes := listedNewChunkSizes(t, v) + assert.Equal(t, sizeOfNewChunks(t, firstFiles, nil), sizes[firstID], + "a later backup must not change the first snapshot's figure") + assert.Equal(t, secondAdded, sizes[secondID], + "a later backup must not change the second snapshot's figure") +} + +// sizeOfNewChunks returns the total size of the distinct chunks of the +// files in after that no file in before has, cut the way the backup cuts +// them. +func sizeOfNewChunks(t *testing.T, after, before map[string][]byte) int64 { + t.Helper() + + known := make(map[string]bool) + for _, chunk := range chunksOf(t, before) { + known[chunk.Hash] = true + } + + var total int64 + + for _, chunk := range chunksOf(t, after) { + if !known[chunk.Hash] { + known[chunk.Hash] = true + total += chunk.Size + } + } + + return total +} + +// chunksOf returns the chunks of every file in files. +func chunksOf(t *testing.T, files map[string][]byte) []chunker.Chunk { + t.Helper() + + c := chunker.NewChunker(faultChunkSize) + + var all []chunker.Chunk + + for _, content := range files { + chunks, err := c.ChunkReader(bytes.NewReader(content)) + require.NoError(t, err) + + all = append(all, chunks...) + } + + return all +} + +// listedNewChunkSizes runs `snapshot list --json` and returns each listed +// snapshot's new_chunk_size by snapshot ID. +func listedNewChunkSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 { + t.Helper() + + var out bytes.Buffer + + v.Stdout = &out + require.NoError(t, v.ListSnapshots(true)) + + sizes := make(map[string]int64) + for _, row := range decodeListJSON(t, out.String()) { + sizes[row.ID] = row.NewChunkSize + } + + return sizes +} diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 9b767df..9e1759a 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -209,6 +209,7 @@ type snapshotStats struct { totalFiles int totalBytes int64 totalChunks int + newChunkSize int64 totalBlobs int totalBytesSkipped int64 totalFilesSkipped int @@ -341,6 +342,7 @@ func (v *Vaultik) scanAllDirectories( stats.totalFiles += result.FilesScanned stats.totalBytes += result.BytesScanned stats.totalChunks += result.ChunksCreated + stats.newChunkSize += result.NewChunkSize stats.totalBlobs += result.BlobsCreated stats.totalFilesSkipped += result.FilesSkipped stats.totalFilesFailed += result.FilesFailed @@ -403,6 +405,7 @@ func (v *Vaultik) finalizeSnapshotMetadata( BlobUncompressedSize: stats.blobUncompressedSize, CompressionLevel: v.Config.CompressionLevel, UploadDurationMs: stats.uploadDuration.Milliseconds(), + NewChunkSize: stats.newChunkSize, } err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats) diff --git a/internal/vaultik/snapshot_list_test.go b/internal/vaultik/snapshot_list_test.go index 030f545..93cfcc8 100644 --- a/internal/vaultik/snapshot_list_test.go +++ b/internal/vaultik/snapshot_list_test.go @@ -445,6 +445,7 @@ type listJSONRow struct { RemoteKey string `json:"remote_key"` Timestamp string `json:"timestamp"` CompressedSize int64 `json:"compressed_size"` + NewChunkSize int64 `json:"new_chunk_size"` LocallyTracked bool `json:"locally_tracked"` RemotePresent *bool `json:"remote_present"` }