Compare commits

..
Author SHA1 Message Date
sneak a4bee51a6f Record each snapshot's uncompressed size when its backup runs (closes #360)
check / check (push) Waiting to run
snapshot list summed the chunks a snapshot reaches through its file rows
in the local index. A file's row is shared by every snapshot that lists
it and rewritten when the file changes, so backing up a changed file
changed every older snapshot's UNCOMPRESSED SIZE.

The backup now takes that total when it records the snapshot's stats,
into a new snapshots column, uncompressed_chunk_size, and snapshot list
reads the column, as it already does for new_chunk_size.

Model: opus-5-5
2026-10-13 15:51:26 +00:00
clawbot e48bafccef Restore the other files when a chunk is in no blob under --skip-errors (closes #326)
check / check (push) Waiting to run
Restore planning stopped at the first file needing a chunk that no blob
holds, so --skip-errors never saw that file and nothing was restored.
The planner now leaves such a file out of the plan and returns it with
the reason, and each one goes through handleRestoreFileError like any
other failed file: skipped and counted under --skip-errors, fatal
without it. The fatal error now suggests --skip-errors, as other restore
failures do.

The skip-errors tests now share one backup helper; the new case removes
a blob from the snapshot metadata and exports it again.

Model: opus-5-5
2026-10-13 16:53:05 +02:00
clawbot bffd4578fc Record each snapshot's new chunk size when its backup runs (closes #311)
check / check (push) Waiting to run
snapshot list worked out NEW CHUNK SIZE from the snapshot's file rows. A
file's row is shared by every snapshot that lists it and rewritten when
the file changes, so a snapshot whose change was a modified file showed
0 B, and older snapshots' figures moved after later backups.

The scanner now totals the size of the chunks it stores that were not
stored before (the chunks chunk_count counts), the backup writes that
total to a new snapshots column, new_chunk_size, and snapshot list reads
it.

Judgement call: the figure is fixed when the backup runs, so removing an
earlier snapshot no longer raises a later snapshot's figure.

Model: opus-5-5
2026-10-13 16:44:14 +02:00
25 changed files with 571 additions and 237 deletions
+5 -4
View File
@@ -293,8 +293,9 @@ needed.
**`snapshot list`**: Show every snapshot known to this host — the union **`snapshot list`**: Show every snapshot known to this host — the union
of the local index and the backup destination store — with timestamps of the local index and the backup destination store — with timestamps
and three sizes per snapshot (compressed remote size; total and three sizes per snapshot (compressed remote size; total
uncompressed chunk size; size of chunks newly referenced by that uncompressed chunk size; size of the chunks that snapshot stored that
snapshot). were not stored before). Both chunk sizes are recorded when the
snapshot's backup runs.
Listing the destination store does **not** require the age secret key, Listing the destination store does **not** require the age secret key,
so it works in vaultik's intended configuration, where the backed-up so it works in vaultik's intended configuration, where the backed-up
@@ -307,8 +308,8 @@ only the hash, so hostname and snapshot name exist solely in the local
index and in the encrypted per-snapshot database. Snapshots found only index and in the encrypted per-snapshot database. Snapshots found only
on the destination store are therefore listed as on the destination store are therefore listed as
`<remote only:<abbreviated remote key>>` and show `<remote only>` in `<remote only:<abbreviated remote key>>` and show `<remote only>` in
the uncompressed and "new chunk" columns, which can only be computed the uncompressed and "new chunk" columns, which come only from the
from the local index. Their timestamp and compressed size are real, local index. Their timestamp and compressed size are real,
read from the manifest. read from the manifest.
Snapshots in the local index with no counterpart on the destination Snapshots in the local index with no counterpart on the destination
+2
View File
@@ -119,8 +119,10 @@ Tracks backup snapshots.
- `completed_at` (INTEGER) - Completion timestamp (NULL if in progress) - `completed_at` (INTEGER) - Completion timestamp (NULL if in progress)
- `file_count` (INTEGER) - Number of files in snapshot - `file_count` (INTEGER) - Number of files in snapshot
- `chunk_count` (INTEGER) - Number of chunks this snapshot stored that were not stored before - `chunk_count` (INTEGER) - Number of chunks this snapshot stored that were not stored before
- `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE
- `blob_count` (INTEGER) - Number of blobs this snapshot created - `blob_count` (INTEGER) - Number of blobs this snapshot created
- `total_size` (INTEGER) - Total size of all files - `total_size` (INTEGER) - Total size of all files
- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the distinct chunks the snapshot's files reference, recorded when its backup ran; `snapshot list` shows it as UNCOMPRESSED SIZE
- `blob_size` (INTEGER) - Total compressed size of all referenced blobs - `blob_size` (INTEGER) - Total compressed size of all referenced blobs
- `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs - `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
- `compression_ratio` (REAL) - Compression ratio achieved - `compression_ratio` (REAL) - Compression ratio achieved
+6 -1
View File
@@ -58,7 +58,12 @@ func (r *BlobChunkRepository) GetByBlobID(
return nil, fmt.Errorf("querying blob chunks: %w", err) return nil, fmt.Errorf("querying blob chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var blobChunks []*BlobChunk var blobChunks []*BlobChunk
+6 -1
View File
@@ -83,7 +83,12 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
return nil, fmt.Errorf("querying blobs: %w", err) return nil, fmt.Errorf("querying blobs: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
out := make(map[string]*Blob) out := make(map[string]*Blob)
+18 -3
View File
@@ -61,7 +61,12 @@ func (r *ChunkFileRepository) GetByChunkHash(
return nil, fmt.Errorf("querying chunk files: %w", err) return nil, fmt.Errorf("querying chunk files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanChunkFiles(rows) return r.scanChunkFiles(rows)
} }
@@ -82,7 +87,12 @@ func (r *ChunkFileRepository) GetByFilePath(
return nil, fmt.Errorf("querying chunk files: %w", err) return nil, fmt.Errorf("querying chunk files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanChunkFiles(rows) return r.scanChunkFiles(rows)
} }
@@ -102,7 +112,12 @@ func (r *ChunkFileRepository) GetByFileID(
return nil, fmt.Errorf("querying chunk files: %w", err) return nil, fmt.Errorf("querying chunk files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanChunkFiles(rows) return r.scanChunkFiles(rows)
} }
+12 -2
View File
@@ -107,7 +107,12 @@ func (r *ChunkRepository) GetByHashes(
return nil, fmt.Errorf("querying chunks: %w", err) return nil, fmt.Errorf("querying chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var chunks []*Chunk var chunks []*Chunk
@@ -147,7 +152,12 @@ func (r *ChunkRepository) ListUnpacked(
return nil, fmt.Errorf("querying unpacked chunks: %w", err) return nil, fmt.Errorf("querying unpacked chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var chunks []*Chunk var chunks []*Chunk
+6 -1
View File
@@ -38,7 +38,12 @@ func (r *ChunkRepository) list(ctx context.Context, query string) ([]*Chunk, err
return nil, fmt.Errorf("querying chunks: %w", err) return nil, fmt.Errorf("querying chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var chunks []*Chunk var chunks []*Chunk
+12
View File
@@ -0,0 +1,12 @@
package database
import (
"fmt"
"os"
)
// Fatalf prints an error message to stderr and exits with status 1
func Fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
os.Exit(1)
}
+18 -3
View File
@@ -62,7 +62,12 @@ func (r *FileChunkRepository) GetByPath(
return nil, fmt.Errorf("querying file chunks: %w", err) return nil, fmt.Errorf("querying file chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanFileChunks(rows) return r.scanFileChunks(rows)
} }
@@ -83,7 +88,12 @@ func (r *FileChunkRepository) GetByFileID(
return nil, fmt.Errorf("querying file chunks: %w", err) return nil, fmt.Errorf("querying file chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanFileChunks(rows) return r.scanFileChunks(rows)
} }
@@ -107,7 +117,12 @@ func (r *FileChunkRepository) GetByPathTx(
return nil, fmt.Errorf("querying file chunks: %w", err) return nil, fmt.Errorf("querying file chunks: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
fileChunks, err := r.scanFileChunks(rows) fileChunks, err := r.scanFileChunks(rows)
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks)) LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
+24 -4
View File
@@ -175,7 +175,12 @@ func (r *FileRepository) ListModifiedSince(
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var files []*File var files []*File
@@ -251,7 +256,12 @@ func (r *FileRepository) ListUnderPath(
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var files []*File var files []*File
@@ -294,7 +304,12 @@ func (r *FileRepository) ListIDsWithChunksNotInUploadedBlobs(
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var ids []types.FileID var ids []types.FileID
@@ -325,7 +340,12 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var files []*File var files []*File
+9 -11
View File
@@ -106,24 +106,22 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
defer func() { defer func() {
if p := recover(); p != nil { if p := recover(); p != nil {
// The panic is the failure to report, so a rollback error rollbackErr := tx.Rollback()
// is dropped. if rollbackErr != nil {
_ = tx.Rollback() Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p) panic(p)
} else if err != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
} }
}() }()
err = fn(ctx, tx) err = fn(ctx, tx)
if err != nil { if err != nil {
// Once ctx is cancelled, database/sql rolls the transaction
// back itself and Rollback returns sql.ErrTxDone.
rollbackErr := tx.Rollback()
if rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) {
return fmt.Errorf("%w; rolling back transaction: %w",
err, rollbackErr)
}
return err return err
} }
-38
View File
@@ -411,41 +411,3 @@ func TestRepositoriesReadTransaction(t *testing.T) {
t.Error("expected to retrieve file in read transaction") t.Error("expected to retrieve file in read transaction")
} }
} }
// When the context is cancelled during a read transaction's body,
// database/sql rolls the transaction back itself, so WithReadTx's own
// rollback finds it already done. WithReadTx must return the body's error
// alone, without that rollback.
func TestRepositoriesReadTransactionContextCancelled(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
repos := database.NewRepositories(db)
err := repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
cancel()
// database/sql rolls back from a goroutine of its own; wait
// until it has, so the test does not depend on that race.
for {
_, err := tx.ExecContext(context.WithoutCancel(ctx), "SELECT 1")
if errors.Is(err, sql.ErrTxDone) {
return ctx.Err()
}
time.Sleep(time.Millisecond)
}
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got: %v", err)
}
if errors.Is(err, sql.ErrTxDone) {
t.Fatalf("the already-done rollback was reported: %v", err)
}
}
+2
View File
@@ -86,8 +86,10 @@ CREATE TABLE IF NOT EXISTS snapshots (
completed_at INTEGER, completed_at INTEGER,
file_count INTEGER NOT NULL DEFAULT 0, file_count INTEGER NOT NULL DEFAULT 0,
chunk_count INTEGER NOT NULL DEFAULT 0, chunk_count INTEGER NOT NULL DEFAULT 0,
new_chunk_size INTEGER NOT NULL DEFAULT 0,
blob_count INTEGER NOT NULL DEFAULT 0, blob_count INTEGER NOT NULL DEFAULT 0,
total_size INTEGER NOT NULL DEFAULT 0, total_size INTEGER NOT NULL DEFAULT 0,
uncompressed_chunk_size INTEGER NOT NULL DEFAULT 0,
blob_size INTEGER NOT NULL DEFAULT 0, blob_size INTEGER NOT NULL DEFAULT 0,
blob_uncompressed_size INTEGER NOT NULL DEFAULT 0, blob_uncompressed_size INTEGER NOT NULL DEFAULT 0,
compression_ratio REAL NOT NULL DEFAULT 1.0, compression_ratio REAL NOT NULL DEFAULT 1.0,
+66 -54
View File
@@ -120,7 +120,11 @@ func (r *SnapshotRepository) UpdateCounts(
return nil return nil
} }
// UpdateExtendedStats updates extended statistics for a snapshot // UpdateExtendedStats updates extended statistics for a snapshot. It also
// records the total plaintext size of the distinct chunks the snapshot's
// files reference. Only the snapshot's own backup can take that total: a
// file's row is shared by every snapshot that lists it, and a later
// backup of a changed file rewrites the row with the new chunks.
func (r *SnapshotRepository) UpdateExtendedStats( func (r *SnapshotRepository) UpdateExtendedStats(
ctx context.Context, ctx context.Context,
tx *sql.Tx, tx *sql.Tx,
@@ -129,6 +133,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
compressionLevel int, compressionLevel int,
uploadBytes int64, uploadBytes int64,
uploadDurationMs int64, uploadDurationMs int64,
newChunkSize int64,
) error { ) error {
compressionRatio, err := r.extendedCompressionRatio( compressionRatio, err := r.extendedCompressionRatio(
ctx, tx, snapshotID, blobUncompressedSize, ctx, tx, snapshotID, blobUncompressedSize,
@@ -143,18 +148,29 @@ func (r *SnapshotRepository) UpdateExtendedStats(
compression_ratio = ?, compression_ratio = ?,
compression_level = ?, compression_level = ?,
upload_bytes = ?, upload_bytes = ?,
upload_duration_ms = ? upload_duration_ms = ?,
new_chunk_size = ?,
uncompressed_chunk_size = (
SELECT COALESCE(SUM(c.size), 0)
FROM (
SELECT DISTINCT fc.chunk_hash
FROM snapshot_files sf
JOIN file_chunks fc ON sf.file_id = fc.file_id
WHERE sf.snapshot_id = snapshots.id
) sc
JOIN chunks c ON sc.chunk_hash = c.chunk_hash
)
WHERE id = ? WHERE id = ?
` `
if tx != nil { if tx != nil {
_, err = tx.ExecContext(ctx, query, _, err = tx.ExecContext(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel, blobUncompressedSize, compressionRatio, compressionLevel,
uploadBytes, uploadDurationMs, snapshotID) uploadBytes, uploadDurationMs, newChunkSize, snapshotID)
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, _, err = r.db.ExecWithLog(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel, blobUncompressedSize, compressionRatio, compressionLevel,
uploadBytes, uploadDurationMs, snapshotID) uploadBytes, uploadDurationMs, newChunkSize, snapshotID)
} }
if err != nil { if err != nil {
@@ -239,7 +255,12 @@ func (r *SnapshotRepository) GetOnlySnapshot(ctx context.Context) (*Snapshot, er
return nil, fmt.Errorf("querying snapshots: %w", err) return nil, fmt.Errorf("querying snapshots: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
snapshots, err := r.scanSnapshotRows(rows) snapshots, err := r.scanSnapshotRows(rows)
if err != nil { if err != nil {
@@ -274,7 +295,12 @@ func (r *SnapshotRepository) ListRecent(
return nil, fmt.Errorf("querying snapshots: %w", err) return nil, fmt.Errorf("querying snapshots: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanSnapshotRows(rows) return r.scanSnapshotRows(rows)
} }
@@ -489,7 +515,12 @@ func (r *SnapshotRepository) GetBlobHashes(
return nil, fmt.Errorf("querying blob hashes: %w", err) return nil, fmt.Errorf("querying blob hashes: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var blobs []string var blobs []string
@@ -553,71 +584,42 @@ func (r *SnapshotRepository) GetSnapshotBlobSizes(
return compressed, uncompressed, nil return compressed, uncompressed, nil
} }
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique // GetSnapshotUncompressedChunkSize returns the total plaintext size of the
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks). // distinct chunks the snapshot's files reference, as the snapshot's backup
// recorded it (see UpdateExtendedStats).
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize( func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
ctx context.Context, snapshotID string, ctx context.Context, snapshotID string,
) (int64, error) { ) (int64, error) {
query := ` query := `SELECT uncompressed_chunk_size FROM snapshots WHERE id = ?`
SELECT COALESCE(SUM(c.size), 0)
FROM (
SELECT DISTINCT fc.chunk_hash
FROM snapshot_files sf
JOIN file_chunks fc ON sf.file_id = fc.file_id
WHERE sf.snapshot_id = ?
) sc
JOIN chunks c ON sc.chunk_hash = c.chunk_hash
`
var totalSize int64 var uncompressedChunkSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&uncompressedChunkSize)
if err != nil { if err != nil {
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err) return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
} }
return totalSize, nil return uncompressedChunkSize, nil
} }
// GetSnapshotNewChunkSize returns the sum of plaintext sizes of chunks that are // GetSnapshotNewChunkSize returns the total plaintext size of the chunks the
// referenced by this snapshot but not by any earlier completed snapshot known to // snapshot's backup stored that were not stored before, as the backup
// the local database. The result is the marginal uncompressed data this snapshot // recorded it. It is not derived from the snapshot's files: a file's row is
// added to the dedup pool — i.e., the delta from prior snapshots. // shared by every snapshot that lists it and is rewritten when the file
// changes, so it shows only the file's latest chunks.
func (r *SnapshotRepository) GetSnapshotNewChunkSize( func (r *SnapshotRepository) GetSnapshotNewChunkSize(
ctx context.Context, snapshotID string, ctx context.Context, snapshotID string,
) (int64, error) { ) (int64, error) {
query := ` query := `SELECT new_chunk_size FROM snapshots WHERE id = ?`
WITH this_snap_chunks AS (
SELECT DISTINCT fc.chunk_hash
FROM snapshot_files sf
JOIN file_chunks fc ON sf.file_id = fc.file_id
WHERE sf.snapshot_id = ?
),
prior_chunks AS (
SELECT DISTINCT fc.chunk_hash
FROM snapshots s
JOIN snapshot_files sf ON sf.snapshot_id = s.id
JOIN file_chunks fc ON fc.file_id = sf.file_id
WHERE s.completed_at IS NOT NULL
AND s.id != ?
AND s.started_at < (SELECT started_at FROM snapshots WHERE id = ?)
)
SELECT COALESCE(SUM(c.size), 0)
FROM chunks c
JOIN this_snap_chunks t ON c.chunk_hash = t.chunk_hash
WHERE c.chunk_hash NOT IN (SELECT chunk_hash FROM prior_chunks)
`
var totalSize int64 var newChunkSize int64
err := r.db.conn.QueryRowContext( err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&newChunkSize)
ctx, query, snapshotID, snapshotID, snapshotID,
).Scan(&totalSize)
if err != nil { if err != nil {
return 0, fmt.Errorf("querying new chunk size: %w", err) return 0, fmt.Errorf("querying new chunk size: %w", err)
} }
return totalSize, nil return newChunkSize, nil
} }
// GetIncompleteSnapshots returns all snapshots that haven't been completed // GetIncompleteSnapshots returns all snapshots that haven't been completed
@@ -638,7 +640,12 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(
return nil, fmt.Errorf("querying incomplete snapshots: %w", err) return nil, fmt.Errorf("querying incomplete snapshots: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
return r.scanSnapshotRows(rows) return r.scanSnapshotRows(rows)
} }
@@ -661,7 +668,12 @@ func (r *SnapshotRepository) GetIncompleteByHostname(
return nil, fmt.Errorf("querying incomplete snapshots: %w", err) return nil, fmt.Errorf("querying incomplete snapshots: %w", err)
} }
defer func() { _ = rows.Close() }() defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
// Same column set as every other multi-row snapshot query, so the // Same column set as every other multi-row snapshot query, so the
// shared scanner applies — including its timestamp normalization. // shared scanner applies — including its timestamp normalization.
-62
View File
@@ -1,62 +0,0 @@
//nolint:testpackage // exercises the unexported cleanupBlobTempFile
package snapshot
import (
"errors"
"os"
"testing"
"github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/blob"
)
// A blob temp file that cannot be closed or removed must make the cleanup
// return an error, not end the process.
func TestCleanupBlobTempFileReturnsErrors(t *testing.T) {
t.Parallel()
fs := afero.NewOsFs()
s := &Scanner{fs: fs}
tests := []struct {
name string
// prepare runs before the cleanup and makes its close or remove
// fail.
prepare func(tempFile afero.File) error
want error
}{
{
name: "close fails",
prepare: func(tempFile afero.File) error { return tempFile.Close() },
want: os.ErrClosed,
},
{
name: "remove fails",
prepare: func(tempFile afero.File) error {
return fs.Remove(tempFile.Name())
},
want: os.ErrNotExist,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tempFile, err := afero.TempFile(fs, t.TempDir(), "blob-*.tmp")
if err != nil {
t.Fatal(err)
}
err = tt.prepare(tempFile)
if err != nil {
t.Fatal(err)
}
err = s.cleanupBlobTempFile(&blob.WithReader{TempFile: tempFile})
if !errors.Is(err, tt.want) {
t.Fatalf("expected an error wrapping %v, got: %v", tt.want, err)
}
})
}
}
+16 -18
View File
@@ -135,7 +135,9 @@ type ScannerConfig struct {
// are counted per file: BytesScanned is the size of the new and changed // are counted per file: BytesScanned is the size of the new and changed
// files, BytesSkipped that of the unchanged ones. FilesFailed counts the // files, BytesSkipped that of the unchanged ones. FilesFailed counts the
// new and changed files that phase 2 could not store; FilesScanned // new and changed files that phase 2 could not store; FilesScanned
// includes them and BytesScanned does not. // includes them and BytesScanned does not. ChunksCreated counts the
// chunks that were not already stored, and NewChunkSize is their total
// size.
type ScanResult struct { type ScanResult struct {
FilesScanned int FilesScanned int
FilesSkipped int FilesSkipped int
@@ -145,6 +147,7 @@ type ScanResult struct {
BytesSkipped int64 BytesSkipped int64
BytesDeleted int64 BytesDeleted int64
ChunksCreated int ChunksCreated int
NewChunkSize int64
BlobsCreated int BlobsCreated int
BlobsUploaded int BlobsUploaded int
BytesUploaded int64 BytesUploaded int64
@@ -1553,24 +1556,20 @@ func (s *Scanner) handleBlobReady(
blobExists, err := s.uploadBlobIfNeeded( blobExists, err := s.uploadBlobIfNeeded(
ctx, blobPath, blobWithReader, startTime, result) ctx, blobPath, blobWithReader, startTime, result)
if err != nil { if err != nil {
cleanupErr := s.cleanupBlobTempFile(blobWithReader) s.cleanupBlobTempFile(blobWithReader)
return errors.Join( return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err), cleanupErr)
} }
err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime) err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime)
if err != nil { if err != nil {
cleanupErr := s.cleanupBlobTempFile(blobWithReader) s.cleanupBlobTempFile(blobWithReader)
return errors.Join(err, cleanupErr)
}
err = s.cleanupBlobTempFile(blobWithReader)
if err != nil {
return err return err
} }
s.cleanupBlobTempFile(blobWithReader)
// Chunks from this blob are now committed to DB - remove from pending set // Chunks from this blob are now committed to DB - remove from pending set
s.removePendingChunkHashes(blobWithReader.InsertedChunkHashes) s.removePendingChunkHashes(blobWithReader.InsertedChunkHashes)
@@ -1763,22 +1762,20 @@ func (s *Scanner) recordBlobMetadata(
} }
// cleanupBlobTempFile closes and removes the blob's temporary file // cleanupBlobTempFile closes and removes the blob's temporary file
func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.WithReader) error { func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.WithReader) {
if blobWithReader.TempFile != nil { if blobWithReader.TempFile != nil {
tempName := blobWithReader.TempFile.Name() tempName := blobWithReader.TempFile.Name()
err := blobWithReader.TempFile.Close() err := blobWithReader.TempFile.Close()
if err != nil { if err != nil {
return fmt.Errorf("closing blob temp file: %w", err) log.Fatal("Failed to close temp file", "file", tempName, "error", err)
} }
err = s.fs.Remove(tempName) err = s.fs.Remove(tempName)
if err != nil { if err != nil {
return fmt.Errorf("removing blob temp file: %w", err) log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
} }
} }
return nil
} }
// streamingChunkInfo tracks chunk metadata collected during streaming // streamingChunkInfo tracks chunk metadata collected during streaming
@@ -1872,11 +1869,12 @@ func (s *Scanner) processFileStreaming(
return nil return nil
} }
// updateChunkStats counts a chunk that was not already stored. The scan // updateChunkStats counts a chunk that was not already stored, and its
// result's file counts, BytesScanned and BytesSkipped are not touched // size. The scan result's file counts, BytesScanned and BytesSkipped are
// here: the scan phase counts each file once. // not touched here: the scan phase counts each file once.
func (s *Scanner) updateChunkStats(chunkSize int64, result *ScanResult) { func (s *Scanner) updateChunkStats(chunkSize int64, result *ScanResult) {
result.ChunksCreated++ result.ChunksCreated++
result.NewChunkSize += chunkSize
if s.progress != nil { if s.progress != nil {
s.progress.GetStats().ChunksCreated.Add(1) s.progress.GetStats().ChunksCreated.Add(1)
+2
View File
@@ -199,6 +199,7 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
stats.CompressionLevel, stats.CompressionLevel,
stats.BytesUploaded, stats.BytesUploaded,
stats.UploadDurationMs, stats.UploadDurationMs,
stats.NewChunkSize,
) )
}) })
} }
@@ -906,6 +907,7 @@ type ExtendedBackupStats struct {
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
CompressionLevel int // Compression level used for this snapshot CompressionLevel int // Compression level used for this snapshot
UploadDurationMs int64 // Total milliseconds spent uploading to S3 UploadDurationMs int64 // Total milliseconds spent uploading to S3
NewChunkSize int64 // Total size of the chunks counted in ChunksCreated
} }
// deleteOtherSnapshots deletes all snapshots except the current one // deleteOtherSnapshots deletes all snapshots except the current one
+133
View File
@@ -0,0 +1,133 @@
package vaultik_test
import (
"bytes"
"context"
"maps"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// TestNewChunkSizeOfChangedFileStaysFixed covers
// https://git.eeqj.de/sneak/vaultik/issues/311. A backup whose only change
// is a modified file lists the size of the chunks that change added as its
// NEW CHUNK SIZE, and a later backup that changes the file again leaves
// every earlier snapshot's figure as it was. The file's row in the local
// index is shared by all three snapshots and rewritten by each backup.
func TestNewChunkSizeOfChangedFileStaysFixed(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
dbPath := filepath.Join(tempDir, "index.sqlite")
changedPath := filepath.Join(dataDir, "changed.bin")
ctx := context.Background()
files := writeFaultSourceTree(t, fs, dataDir)
cfg := changedFileConfig(dataDir, dbPath)
appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
require.NoError(t, backUp(v, "first"))
firstFiles := maps.Clone(files)
firstID := localSnapshotID(ctx, t, repos, "first")
appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
require.NoError(t, backUp(v, "second"))
secondID := localSnapshotID(ctx, t, repos, "second")
secondAdded := sizeOfNewChunks(t, files, firstFiles)
require.Positive(t, secondAdded)
assert.Equal(t, secondAdded, listedNewChunkSizes(t, v)[secondID],
"the second snapshot must count the chunks the change added")
appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
require.NoError(t, backUp(v, "third"))
sizes := listedNewChunkSizes(t, v)
assert.Equal(t, sizeOfNewChunks(t, firstFiles, nil), sizes[firstID],
"a later backup must not change the first snapshot's figure")
assert.Equal(t, secondAdded, sizes[secondID],
"a later backup must not change the second snapshot's figure")
}
// sizeOfNewChunks returns the total size of the distinct chunks of the
// files in after that no file in before has, cut the way the backup cuts
// them.
func sizeOfNewChunks(t *testing.T, after, before map[string][]byte) int64 {
t.Helper()
known := make(map[string]bool)
for _, chunk := range chunksOf(t, before) {
known[chunk.Hash] = true
}
var total int64
for _, chunk := range chunksOf(t, after) {
if !known[chunk.Hash] {
known[chunk.Hash] = true
total += chunk.Size
}
}
return total
}
// chunksOf returns the chunks of every file in files.
func chunksOf(t *testing.T, files map[string][]byte) []chunker.Chunk {
t.Helper()
c := chunker.NewChunker(faultChunkSize)
var all []chunker.Chunk
for _, content := range files {
chunks, err := c.ChunkReader(bytes.NewReader(content))
require.NoError(t, err)
all = append(all, chunks...)
}
return all
}
// listedNewChunkSizes runs `snapshot list --json` and returns each listed
// snapshot's new_chunk_size by snapshot ID.
func listedNewChunkSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
t.Helper()
var out bytes.Buffer
v.Stdout = &out
require.NoError(t, v.ListSnapshots(true))
sizes := make(map[string]int64)
for _, row := range decodeListJSON(t, out.String()) {
sizes[row.ID] = row.NewChunkSize
}
return sizes
}
+12 -1
View File
@@ -309,11 +309,22 @@ func (v *Vaultik) restoreAllFiles(
return nil, err return nil, err
} }
plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash) plan, unplanned, err := newRestorePlan(
v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
if err != nil { if err != nil {
return nil, fmt.Errorf("building restore plan: %w", err) return nil, fmt.Errorf("building restore plan: %w", err)
} }
// Ranging over files rather than the map reports them in a stable order.
for _, file := range files {
if planErr, ok := unplanned[file.ID]; ok {
err = v.handleRestoreFileError(plan, opts, result, file, file.ID, planErr)
if err != nil {
return nil, err
}
}
}
filesByID, totalBytesExpected := indexRestoreFiles(files) filesByID, totalBytesExpected := indexRestoreFiles(files)
v.UI.Beginf("Restoring %s files (%s) to %s.", v.UI.Beginf("Restoring %s files (%s) to %s.",
@@ -61,8 +61,8 @@ func craftedFile(
} }
// TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash // TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash
// has no blob_chunks row fails restore planning and the chunk write with // has no blob_chunks row is left out of the restore plan with an error,
// an error. // and fails the chunk write with an error.
func TestRestoreShortChunkHashInNoBlob(t *testing.T) { func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
t.Parallel() t.Parallel()
@@ -77,9 +77,11 @@ func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos) chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
require.NoError(t, err) require.NoError(t, err)
_, err = newRestorePlan(ctx, repos, []*database.File{file}, plan, unplanned, err := newRestorePlan(ctx, repos, []*database.File{file},
chunkToBlobMap, map[string]string{}) chunkToBlobMap, map[string]string{})
require.ErrorIs(t, err, errPlanChunkMissing) require.NoError(t, err)
require.ErrorIs(t, unplanned[file.ID], errPlanChunkMissing)
require.False(t, plan.hasPending())
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID) fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
require.NoError(t, err) require.NoError(t, err)
+17 -8
View File
@@ -41,20 +41,25 @@ type restorePlan struct {
// newRestorePlan builds the file→blob index for the given files. Files // newRestorePlan builds the file→blob index for the given files. Files
// whose chunks reference no blobs (symlinks, directories) start in the // whose chunks reference no blobs (symlinks, directories) start in the
// ready queue immediately. // ready queue immediately. A file with a chunk that no blob holds cannot
// be restored: it is left out of the plan and returned, by ID, with the
// reason.
func newRestorePlan( func newRestorePlan(
ctx context.Context, ctx context.Context,
repos *database.Repositories, repos *database.Repositories,
files []*database.File, files []*database.File,
chunkToBlobMap map[string]*database.BlobChunk, chunkToBlobMap map[string]*database.BlobChunk,
blobIDToHash map[string]string, blobIDToHash map[string]string,
) (*restorePlan, error) { ) (*restorePlan, map[types.FileID]error, error) {
p := &restorePlan{ p := &restorePlan{
fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)), fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)),
blobFiles: make(map[string]map[types.FileID]struct{}), blobFiles: make(map[string]map[types.FileID]struct{}),
ready: make([]types.FileID, 0, len(files)), ready: make([]types.FileID, 0, len(files)),
cached: make(map[string]struct{}), cached: make(map[string]struct{}),
} }
unplanned := make(map[types.FileID]error)
nextFile:
for _, f := range files { for _, f := range files {
if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 { if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 {
// No chunks to fetch — restore can run immediately. // No chunks to fetch — restore can run immediately.
@@ -66,7 +71,7 @@ func newRestorePlan(
fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID) fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID)
if err != nil { if err != nil {
return nil, fmt.Errorf("planning %s: %w", f.Path, err) return nil, nil, fmt.Errorf("planning %s: %w", f.Path, err)
} }
blobs := make(map[string]struct{}) blobs := make(map[string]struct{})
@@ -74,14 +79,18 @@ func newRestorePlan(
for _, fc := range fileChunks { for _, fc := range fileChunks {
bc, ok := chunkToBlobMap[fc.ChunkHash.String()] bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
if !ok { if !ok {
return nil, fmt.Errorf("planning %s: %w: %s", unplanned[f.ID] = fmt.Errorf("%w: %s",
f.Path, errPlanChunkMissing, shortHash(fc.ChunkHash.String())) errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
continue nextFile
} }
hash, ok := blobIDToHash[bc.BlobID.String()] hash, ok := blobIDToHash[bc.BlobID.String()]
if !ok { if !ok {
return nil, fmt.Errorf("planning %s: %w: %s", unplanned[f.ID] = fmt.Errorf("%w: %s",
f.Path, errPlanBlobIDMissing, bc.BlobID) errPlanBlobIDMissing, bc.BlobID)
continue nextFile
} }
blobs[hash] = struct{}{} blobs[hash] = struct{}{}
@@ -103,7 +112,7 @@ func newRestorePlan(
} }
} }
return p, nil return p, unplanned, nil
} }
// markBlobCached records that the named blob is now resident in the // markBlobCached records that the named blob is now resident in the
+103 -16
View File
@@ -28,7 +28,8 @@ const (
) )
// missingBlobBackup is a snapshot of single-chunk files spread over two // missingBlobBackup is a snapshot of single-chunk files spread over two
// blobs, with one of those blobs deleted from the store. // blobs, with one of those blobs lost: deleted from the store, or missing
// from the snapshot metadata.
type missingBlobBackup struct { type missingBlobBackup struct {
fs afero.Fs fs afero.Fs
cfg *config.Config cfg *config.Config
@@ -37,20 +38,39 @@ type missingBlobBackup struct {
restoreDir string restoreDir string
// files holds the original content by source path. // files holds the original content by source path.
files map[string][]byte files map[string][]byte
// lost holds the source paths whose chunk is in the deleted blob. // lost holds the source paths whose chunk is in the lost blob.
lost map[string]bool lost map[string]bool
} }
// TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors // TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
// after one blob of a two-blob snapshot was deleted. Every file stored in // after one blob of a two-blob snapshot was deleted from the store.
// that blob must be reported as failed and left absent, every other file
// must be restored intact, and Restore must still return an error.
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) { func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
ctx := context.Background() ctx := context.Background()
backup := backupThenDeleteOneBlob(ctx, t) assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
}
// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
// earlier backup defects left behind.
func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
}
// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
// stored in the lost blob must be reported as failed and left absent,
// every other file must be restored intact, and Restore must still return
// an error.
func assertRestoreSkipsLostFiles(
ctx context.Context, t *testing.T, backup *missingBlobBackup,
) {
t.Helper()
var out bytes.Buffer var out bytes.Buffer
@@ -72,14 +92,14 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
if backup.lost[path] { if backup.lost[path] {
assert.Containsf(t, out.String(), path, assert.Containsf(t, out.String(), path,
"%s needs the deleted blob and must be reported", path) "%s needs the lost blob and must be reported", path)
assert.NoFileExists(t, restored) assert.NoFileExists(t, restored)
continue continue
} }
got, err := afero.ReadFile(backup.fs, restored) got, err := afero.ReadFile(backup.fs, restored)
require.NoErrorf(t, err, "%s does not need the deleted blob", path) require.NoErrorf(t, err, "%s does not need the lost blob", path)
assert.Equalf(t, content, got, "%s restored with wrong content", path) assert.Equalf(t, content, got, "%s restored with wrong content", path)
} }
} }
@@ -103,6 +123,30 @@ func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
require.ErrorIs(t, err, storage.ErrNotFound) require.ErrorIs(t, err, storage.ErrNotFound)
} }
// TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held
// by no blob still ends the restore, before any file is written, when
// SkipErrors is not set.
func TestRestoreChunkInNoBlobAbortsWithoutSkipErrors(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
backup := backupThenDropOneBlobFromMetadata(ctx, t)
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
err := v.Restore(&vaultik.RestoreOptions{
SnapshotID: backup.snapshotID,
TargetDir: backup.restoreDir,
})
require.ErrorContains(t, err, "chunk missing from blob map")
for path := range backup.files {
assert.NoFileExists(t, filepath.Join(backup.restoreDir, path))
}
}
// backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk // backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
// files, then deletes from the store the blob holding the first of them. // files, then deletes from the store the blob holding the first of them.
func backupThenDeleteOneBlob( func backupThenDeleteOneBlob(
@@ -110,6 +154,54 @@ func backupThenDeleteOneBlob(
) *missingBlobBackup { ) *missingBlobBackup {
t.Helper() t.Helper()
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
require.NoError(t, db.Close())
require.NoError(t, backup.storer.Delete(ctx, fmt.Sprintf(
"blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)))
return backup
}
// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
// single-chunk files, then removes the blob holding the first of them from
// the snapshot and exports the snapshot metadata again. The exported
// metadata then lists chunks that no blob holds.
func backupThenDropOneBlobFromMetadata(
ctx context.Context, t *testing.T,
) *missingBlobBackup {
t.Helper()
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
// The uploads row refers to the blob row, so it would keep the export
// from deleting it.
for _, query := range []string{
"DELETE FROM snapshot_blobs WHERE blob_hash = ?",
"DELETE FROM uploads WHERE blob_hash = ?",
} {
_, err := db.Conn().ExecContext(ctx, query, lostBlob)
require.NoError(t, err)
}
sm := newFaultSnapshotManager(
backup.fs, backup.storer, backup.cfg, database.NewRepositories(db))
require.NoError(t,
sm.ExportSnapshotMetadata(ctx, db.Path(), backup.snapshotID))
require.NoError(t, db.Close())
return backup
}
// backupSingleChunkFiles backs up missingBlobFileCount single-chunk files.
// It returns the backup, with lost naming the files stored in the same
// blob as the first of them, the still open local index, and the hash of
// that blob.
func backupSingleChunkFiles(
ctx context.Context, t *testing.T,
) (*missingBlobBackup, *database.DB, string) {
t.Helper()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src") dataDir := filepath.Join(tempDir, "src")
@@ -143,16 +235,11 @@ func backupThenDeleteOneBlob(
blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path) blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
} }
require.NoError(t, db.Close()) lostBlob := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
deleted := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
require.NoError(t, storer.Delete(ctx, fmt.Sprintf(
"blobs/%s/%s/%s", deleted[:2], deleted[2:4], deleted)))
lost := make(map[string]bool) lost := make(map[string]bool)
for path, hash := range blobOfFile { for path, hash := range blobOfFile {
if hash == deleted { if hash == lostBlob {
lost[path] = true lost[path] = true
} }
} }
@@ -168,7 +255,7 @@ func backupThenDeleteOneBlob(
restoreDir: filepath.Join(tempDir, "restored"), restoreDir: filepath.Join(tempDir, "restored"),
files: files, files: files,
lost: lost, lost: lost,
} }, db, lostBlob
} }
// blobHashOfSingleChunkFile returns the hash of the blob holding the one // blobHashOfSingleChunkFile returns the hash of the blob holding the one
+3
View File
@@ -209,6 +209,7 @@ type snapshotStats struct {
totalFiles int totalFiles int
totalBytes int64 totalBytes int64
totalChunks int totalChunks int
newChunkSize int64
totalBlobs int totalBlobs int
totalBytesSkipped int64 totalBytesSkipped int64
totalFilesSkipped int totalFilesSkipped int
@@ -341,6 +342,7 @@ func (v *Vaultik) scanAllDirectories(
stats.totalFiles += result.FilesScanned stats.totalFiles += result.FilesScanned
stats.totalBytes += result.BytesScanned stats.totalBytes += result.BytesScanned
stats.totalChunks += result.ChunksCreated stats.totalChunks += result.ChunksCreated
stats.newChunkSize += result.NewChunkSize
stats.totalBlobs += result.BlobsCreated stats.totalBlobs += result.BlobsCreated
stats.totalFilesSkipped += result.FilesSkipped stats.totalFilesSkipped += result.FilesSkipped
stats.totalFilesFailed += result.FilesFailed stats.totalFilesFailed += result.FilesFailed
@@ -403,6 +405,7 @@ func (v *Vaultik) finalizeSnapshotMetadata(
BlobUncompressedSize: stats.blobUncompressedSize, BlobUncompressedSize: stats.blobUncompressedSize,
CompressionLevel: v.Config.CompressionLevel, CompressionLevel: v.Config.CompressionLevel,
UploadDurationMs: stats.uploadDuration.Milliseconds(), UploadDurationMs: stats.uploadDuration.Milliseconds(),
NewChunkSize: stats.newChunkSize,
} }
err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats) err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats)
+8 -6
View File
@@ -483,12 +483,14 @@ func TestListSnapshots_UnreadableManifestDoesNotHideOthers(t *testing.T) {
// //
//nolint:tagliatelle // snake_case is the established output format //nolint:tagliatelle // snake_case is the established output format
type listJSONRow struct { type listJSONRow struct {
ID string `json:"id"` ID string `json:"id"`
RemoteKey string `json:"remote_key"` RemoteKey string `json:"remote_key"`
Timestamp string `json:"timestamp"` Timestamp string `json:"timestamp"`
CompressedSize int64 `json:"compressed_size"` CompressedSize int64 `json:"compressed_size"`
LocallyTracked bool `json:"locally_tracked"` UncompressedSize int64 `json:"uncompressed_size"`
RemotePresent *bool `json:"remote_present"` NewChunkSize int64 `json:"new_chunk_size"`
LocallyTracked bool `json:"locally_tracked"`
RemotePresent *bool `json:"remote_present"`
} }
// decodeListJSON parses the command's stdout, which must contain // decodeListJSON parses the command's stdout, which must contain
@@ -0,0 +1,85 @@
package vaultik_test
import (
"bytes"
"context"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// TestUncompressedSizeOfChangedFileStaysFixed covers
// https://git.eeqj.de/sneak/vaultik/issues/360. A backup that changes a
// file leaves the earlier snapshot's UNCOMPRESSED SIZE as it was when that
// snapshot's backup completed. The file's row in the local index is shared
// by both snapshots and rewritten by the second backup.
func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
dbPath := filepath.Join(tempDir, "index.sqlite")
changedPath := filepath.Join(dataDir, "changed.bin")
ctx := context.Background()
files := writeFaultSourceTree(t, fs, dataDir)
cfg := changedFileConfig(dataDir, dbPath)
appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
require.NoError(t, backUp(v, "first"))
// With nothing to compare against, sizeOfNewChunks totals every
// distinct chunk of the files.
firstID := localSnapshotID(ctx, t, repos, "first")
firstSize := sizeOfNewChunks(t, files, nil)
require.Equal(t, firstSize, listedUncompressedSizes(t, v)[firstID],
"the first snapshot must count the chunks of every file it holds")
appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
require.NoError(t, backUp(v, "second"))
secondID := localSnapshotID(ctx, t, repos, "second")
sizes := listedUncompressedSizes(t, v)
assert.Equal(t, firstSize, sizes[firstID],
"a later backup must not change the first snapshot's figure")
assert.Equal(t, sizeOfNewChunks(t, files, nil), sizes[secondID],
"the second snapshot must count the changed file's chunks")
}
// listedUncompressedSizes runs `snapshot list --json` and returns each
// listed snapshot's uncompressed_size by snapshot ID.
func listedUncompressedSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
t.Helper()
var out bytes.Buffer
v.Stdout = &out
require.NoError(t, v.ListSnapshots(true))
sizes := make(map[string]int64)
for _, row := range decodeListJSON(t, out.String()) {
sizes[row.ID] = row.UncompressedSize
}
return sizes
}