clawbot
e48bafccef
check / check (push) Waiting to run
Restore planning stopped at the first file needing a chunk that no blob holds, so --skip-errors never saw that file and nothing was restored. The planner now leaves such a file out of the plan and returns it with the reason, and each one goes through handleRestoreFileError like any other failed file: skipped and counted under --skip-errors, fatal without it. The fatal error now suggests --skip-errors, as other restore failures do. The skip-errors tests now share one backup helper; the new case removes a blob from the snapshot metadata and exports it again. Model: opus-5-5
283 lines
8.4 KiB
Go
283 lines
8.4 KiB
Go
package vaultik_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/database"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/ui"
|
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
)
|
|
|
|
// A file no larger than the chunker's minimum chunk size (a quarter of
|
|
// faultChunkSize) is stored as one chunk, so it lives in exactly one blob.
|
|
// More of them than fit in faultMaxBlobSize make the snapshot span two
|
|
// blobs.
|
|
const (
|
|
missingBlobFileBytes = int(faultChunkSize / 4)
|
|
missingBlobFileCount = 20
|
|
)
|
|
|
|
// missingBlobBackup is a snapshot of single-chunk files spread over two
|
|
// blobs, with one of those blobs lost: deleted from the store, or missing
|
|
// from the snapshot metadata.
|
|
type missingBlobBackup struct {
|
|
fs afero.Fs
|
|
cfg *config.Config
|
|
storer storage.Storer
|
|
snapshotID string
|
|
restoreDir string
|
|
// files holds the original content by source path.
|
|
files map[string][]byte
|
|
// lost holds the source paths whose chunk is in the lost blob.
|
|
lost map[string]bool
|
|
}
|
|
|
|
// TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
|
|
// after one blob of a two-blob snapshot was deleted from the store.
|
|
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
|
|
}
|
|
|
|
// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
|
|
// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
|
|
// earlier backup defects left behind.
|
|
func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
|
|
}
|
|
|
|
// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
|
|
// stored in the lost blob must be reported as failed and left absent,
|
|
// every other file must be restored intact, and Restore must still return
|
|
// an error.
|
|
func assertRestoreSkipsLostFiles(
|
|
ctx context.Context, t *testing.T, backup *missingBlobBackup,
|
|
) {
|
|
t.Helper()
|
|
|
|
var out bytes.Buffer
|
|
|
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
|
v.UI = ui.NewWithColor(&out, false)
|
|
|
|
err := v.Restore(&vaultik.RestoreOptions{
|
|
SnapshotID: backup.snapshotID,
|
|
TargetDir: backup.restoreDir,
|
|
SkipErrors: true,
|
|
})
|
|
|
|
require.Error(t, err, "restore must fail when files were skipped")
|
|
assert.Contains(t, err.Error(),
|
|
fmt.Sprintf("%d file(s) failed to restore", len(backup.lost)))
|
|
|
|
for path, content := range backup.files {
|
|
restored := filepath.Join(backup.restoreDir, path)
|
|
|
|
if backup.lost[path] {
|
|
assert.Containsf(t, out.String(), path,
|
|
"%s needs the lost blob and must be reported", path)
|
|
assert.NoFileExists(t, restored)
|
|
|
|
continue
|
|
}
|
|
|
|
got, err := afero.ReadFile(backup.fs, restored)
|
|
require.NoErrorf(t, err, "%s does not need the lost blob", path)
|
|
assert.Equalf(t, content, got, "%s restored with wrong content", path)
|
|
}
|
|
}
|
|
|
|
// TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
|
|
// still ends the restore with an error when SkipErrors is not set.
|
|
func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
backup := backupThenDeleteOneBlob(ctx, t)
|
|
|
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
|
|
|
err := v.Restore(&vaultik.RestoreOptions{
|
|
SnapshotID: backup.snapshotID,
|
|
TargetDir: backup.restoreDir,
|
|
})
|
|
|
|
require.ErrorIs(t, err, storage.ErrNotFound)
|
|
}
|
|
|
|
// TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held
|
|
// by no blob still ends the restore, before any file is written, when
|
|
// SkipErrors is not set.
|
|
func TestRestoreChunkInNoBlobAbortsWithoutSkipErrors(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
backup := backupThenDropOneBlobFromMetadata(ctx, t)
|
|
|
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
|
|
|
err := v.Restore(&vaultik.RestoreOptions{
|
|
SnapshotID: backup.snapshotID,
|
|
TargetDir: backup.restoreDir,
|
|
})
|
|
|
|
require.ErrorContains(t, err, "chunk missing from blob map")
|
|
|
|
for path := range backup.files {
|
|
assert.NoFileExists(t, filepath.Join(backup.restoreDir, path))
|
|
}
|
|
}
|
|
|
|
// backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
|
|
// files, then deletes from the store the blob holding the first of them.
|
|
func backupThenDeleteOneBlob(
|
|
ctx context.Context, t *testing.T,
|
|
) *missingBlobBackup {
|
|
t.Helper()
|
|
|
|
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
|
|
require.NoError(t, db.Close())
|
|
|
|
require.NoError(t, backup.storer.Delete(ctx, fmt.Sprintf(
|
|
"blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)))
|
|
|
|
return backup
|
|
}
|
|
|
|
// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
|
|
// single-chunk files, then removes the blob holding the first of them from
|
|
// the snapshot and exports the snapshot metadata again. The exported
|
|
// metadata then lists chunks that no blob holds.
|
|
func backupThenDropOneBlobFromMetadata(
|
|
ctx context.Context, t *testing.T,
|
|
) *missingBlobBackup {
|
|
t.Helper()
|
|
|
|
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
|
|
|
|
// The uploads row refers to the blob row, so it would keep the export
|
|
// from deleting it.
|
|
for _, query := range []string{
|
|
"DELETE FROM snapshot_blobs WHERE blob_hash = ?",
|
|
"DELETE FROM uploads WHERE blob_hash = ?",
|
|
} {
|
|
_, err := db.Conn().ExecContext(ctx, query, lostBlob)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
sm := newFaultSnapshotManager(
|
|
backup.fs, backup.storer, backup.cfg, database.NewRepositories(db))
|
|
require.NoError(t,
|
|
sm.ExportSnapshotMetadata(ctx, db.Path(), backup.snapshotID))
|
|
require.NoError(t, db.Close())
|
|
|
|
return backup
|
|
}
|
|
|
|
// backupSingleChunkFiles backs up missingBlobFileCount single-chunk files.
|
|
// It returns the backup, with lost naming the files stored in the same
|
|
// blob as the first of them, the still open local index, and the hash of
|
|
// that blob.
|
|
func backupSingleChunkFiles(
|
|
ctx context.Context, t *testing.T,
|
|
) (*missingBlobBackup, *database.DB, string) {
|
|
t.Helper()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
dataDir := filepath.Join(tempDir, "src")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
cfg := faultTestConfig()
|
|
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
|
|
files := make(map[string][]byte, missingBlobFileCount)
|
|
|
|
for i := range missingBlobFileCount {
|
|
path := filepath.Join(dataDir, fmt.Sprintf("file-%02d.bin", i))
|
|
files[path] = bytesPattern(
|
|
fmt.Sprintf("file-%02d-", i), missingBlobFileBytes)
|
|
require.NoError(t, afero.WriteFile(fs, path, files[path], 0o644))
|
|
}
|
|
|
|
storer, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
|
require.NoError(t, err)
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
require.NoError(t, err)
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
id := fullFaultBackup(
|
|
ctx, t, fs, storer, cfg, repos, dataDir, dbPath, "missingblob")
|
|
|
|
blobOfFile := make(map[string]string, len(files))
|
|
for path := range files {
|
|
blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
|
|
}
|
|
|
|
lostBlob := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
|
|
lost := make(map[string]bool)
|
|
|
|
for path, hash := range blobOfFile {
|
|
if hash == lostBlob {
|
|
lost[path] = true
|
|
}
|
|
}
|
|
|
|
require.Less(t, len(lost), len(files),
|
|
"the snapshot must span more than one blob")
|
|
|
|
return &missingBlobBackup{
|
|
fs: fs,
|
|
cfg: cfg,
|
|
storer: storer,
|
|
snapshotID: id,
|
|
restoreDir: filepath.Join(tempDir, "restored"),
|
|
files: files,
|
|
lost: lost,
|
|
}, db, lostBlob
|
|
}
|
|
|
|
// blobHashOfSingleChunkFile returns the hash of the blob holding the one
|
|
// chunk of the file at path, as recorded in the local index.
|
|
func blobHashOfSingleChunkFile(
|
|
ctx context.Context, t *testing.T,
|
|
repos *database.Repositories, path string,
|
|
) string {
|
|
t.Helper()
|
|
|
|
chunks, err := repos.FileChunks.GetByPath(ctx, path)
|
|
require.NoError(t, err)
|
|
require.Lenf(t, chunks, 1, "%s must be a single chunk", path)
|
|
|
|
blobChunk, err := repos.BlobChunks.GetByChunkHash(
|
|
ctx, chunks[0].ChunkHash.String())
|
|
require.NoError(t, err)
|
|
require.NotNilf(t, blobChunk, "chunk of %s is in no blob", path)
|
|
|
|
blob, err := repos.Blobs.GetByID(ctx, blobChunk.BlobID.String())
|
|
require.NoError(t, err)
|
|
|
|
return blob.Hash.String()
|
|
}
|