package vaultik_test import ( "bytes" "context" "fmt" "os" "path/filepath" "strings" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/vaultik" ) // A file no larger than the chunker's minimum chunk size (a quarter of // faultChunkSize) is stored as one chunk, so it lives in exactly one blob. // More of them than fit in faultMaxBlobSize make the snapshot span two // blobs. const ( missingBlobFileBytes = int(faultChunkSize / 4) missingBlobFileCount = 20 ) // missingBlobBackup is a snapshot of single-chunk files spread over two // blobs, with one of those blobs lost: deleted from the store, or missing // from the snapshot metadata. type missingBlobBackup struct { fs afero.Fs cfg *config.Config storer storage.Storer snapshotID string restoreDir string // files holds the original content by source path. files map[string][]byte // lost holds the source paths whose chunk is in the lost blob. lost map[string]bool } // TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors // after one blob of a two-blob snapshot was deleted from the store. func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t)) } // TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with // SkipErrors a snapshot whose metadata lists chunks that no blob holds, as // earlier backup defects left behind. func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t)) } // assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file // stored in the lost blob must be reported as failed and left absent, // every other file must be restored intact, and Restore must still return // an error. func assertRestoreSkipsLostFiles( ctx context.Context, t *testing.T, backup *missingBlobBackup, ) { t.Helper() var out bytes.Buffer v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs) v.UI = ui.NewWithColor(&out, false) err := v.Restore(&vaultik.RestoreOptions{ SnapshotID: backup.snapshotID, TargetDir: backup.restoreDir, SkipErrors: true, }) require.Error(t, err, "restore must fail when files were skipped") assert.Contains(t, err.Error(), fmt.Sprintf("%d file(s) failed to restore", len(backup.lost))) for path, content := range backup.files { restored := filepath.Join(backup.restoreDir, path) if backup.lost[path] { assert.Containsf(t, out.String(), path, "%s needs the lost blob and must be reported", path) assert.NoFileExists(t, restored) continue } got, err := afero.ReadFile(backup.fs, restored) require.NoErrorf(t, err, "%s does not need the lost blob", path) assert.Equalf(t, content, got, "%s restored with wrong content", path) } } // TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with // SkipErrors and Verify after one blob of a two-blob snapshot was // deleted. Verification must check only the files that were written, so // a file of the deleted blob is reported as failed to restore and not // also as failing verification, and Restore must still return an error. func TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() backup := backupThenDeleteOneBlob(ctx, t) var out bytes.Buffer v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs) v.UI = ui.NewWithColor(&out, false) err := v.Restore(&vaultik.RestoreOptions{ SnapshotID: backup.snapshotID, TargetDir: backup.restoreDir, Verify: true, SkipErrors: true, }) require.Error(t, err, "restore must fail when files were skipped") assert.Contains(t, err.Error(), fmt.Sprintf("%d file(s) failed to restore", len(backup.lost))) written := len(backup.files) - len(backup.lost) assert.Contains(t, out.String(), fmt.Sprintf("Verifying %d files", written)) assert.Contains(t, out.String(), fmt.Sprintf("Verified %d files", written)) assert.NotContains(t, out.String(), "Verification failed") } // TestRestoreSkipErrorsVerifyReportsFailuresApart restores with // SkipErrors and Verify after one blob of a two-blob snapshot was // deleted, while one file that does restore is written one byte too long. // The restore warning must list only the files of the deleted blob, the // verification summary only the long file, each path once, and Restore // must return an error. func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() backup := backupThenDeleteOneBlob(ctx, t) var long string for path := range backup.files { if !backup.lost[path] { long = path break } } var out bytes.Buffer fs := trailingByteFs{Fs: backup.fs, suffix: long} v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, fs) v.UI = ui.NewWithColor(&out, false) err := v.Restore(&vaultik.RestoreOptions{ SnapshotID: backup.snapshotID, TargetDir: backup.restoreDir, Verify: true, SkipErrors: true, }) require.Error(t, err, "restore must fail when files failed") _, afterWarning, found := strings.Cut(out.String(), fmt.Sprintf("%d file(s) failed to restore:", len(backup.lost))) require.Truef(t, found, "no restore warning in:\n%s", out.String()) warning, summary, found := strings.Cut(afterWarning, "Verification failed: 1 files did not match") require.Truef(t, found, "no verification summary in:\n%s", out.String()) for path := range backup.lost { assert.Equalf(t, 1, strings.Count(warning, path), "%s must be listed once as failed to restore", path) assert.NotContainsf(t, summary, path, "%s was not restored and must not fail verification", path) } assert.NotContainsf(t, warning, long, "%s was restored and must not be listed as failed to restore", long) assert.Equalf(t, 1, strings.Count(summary, long), "%s must be listed once as failing verification", long) } // trailingByteFs appends one byte, on close, to the file it opens whose // path ends in suffix, so that file is restored one byte too long and // fails verification. type trailingByteFs struct { afero.Fs suffix string } //nolint:ireturn // afero.Fs.OpenFile is defined to return the interface func (fs trailingByteFs) OpenFile( name string, flag int, perm os.FileMode, ) (afero.File, error) { f, err := fs.Fs.OpenFile(name, flag, perm) if err != nil || !strings.HasSuffix(name, fs.suffix) { return f, err } return trailingByteFile{File: f}, nil } type trailingByteFile struct { afero.File } func (f trailingByteFile) Close() error { _, _ = f.Write([]byte{'x'}) return f.File.Close() } // TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob // still ends the restore with an error when SkipErrors is not set, and // that the error names the blob once. func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() backup := backupThenDeleteOneBlob(ctx, t) v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs) err := v.Restore(&vaultik.RestoreOptions{ SnapshotID: backup.snapshotID, TargetDir: backup.restoreDir, }) require.ErrorIs(t, err, storage.ErrNotFound) assert.Equal(t, 1, strings.Count(err.Error(), "downloading blob"), err.Error()) } // TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held // by no blob still ends the restore, before any file is written, when // SkipErrors is not set. func TestRestoreChunkInNoBlobAbortsWithoutSkipErrors(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() ctx := context.Background() backup := backupThenDropOneBlobFromMetadata(ctx, t) v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs) err := v.Restore(&vaultik.RestoreOptions{ SnapshotID: backup.snapshotID, TargetDir: backup.restoreDir, }) require.ErrorContains(t, err, "chunk missing from blob map") for path := range backup.files { assert.NoFileExists(t, filepath.Join(backup.restoreDir, path)) } } // backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk // files, then deletes from the store the blob holding the first of them. func backupThenDeleteOneBlob( ctx context.Context, t *testing.T, ) *missingBlobBackup { t.Helper() backup, db, lostBlob := backupSingleChunkFiles(ctx, t) require.NoError(t, db.Close()) require.NoError(t, backup.storer.Delete(ctx, fmt.Sprintf( "blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob))) return backup } // backupThenDropOneBlobFromMetadata backs up missingBlobFileCount // single-chunk files, then removes the blob holding the first of them from // the snapshot and exports the snapshot metadata again. The exported // metadata then lists chunks that no blob holds. func backupThenDropOneBlobFromMetadata( ctx context.Context, t *testing.T, ) *missingBlobBackup { t.Helper() backup, db, lostBlob := backupSingleChunkFiles(ctx, t) // The uploads row refers to the blob row, so it would keep the export // from deleting it. for _, query := range []string{ "DELETE FROM snapshot_blobs WHERE blob_hash = ?", "DELETE FROM uploads WHERE blob_hash = ?", } { _, err := db.Conn().ExecContext(ctx, query, lostBlob) require.NoError(t, err) } sm := newFaultSnapshotManager( backup.fs, backup.storer, backup.cfg, database.NewRepositories(db)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, db.Path(), backup.snapshotID)) require.NoError(t, db.Close()) return backup } // backupSingleChunkFiles backs up missingBlobFileCount single-chunk files. // It returns the backup, with lost naming the files stored in the same // blob as the first of them, the still open local index, and the hash of // that blob. func backupSingleChunkFiles( ctx context.Context, t *testing.T, ) (*missingBlobBackup, *database.DB, string) { t.Helper() fs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "src") dbPath := filepath.Join(tempDir, "index.sqlite") cfg := faultTestConfig() require.NoError(t, fs.MkdirAll(dataDir, 0o755)) files := make(map[string][]byte, missingBlobFileCount) for i := range missingBlobFileCount { path := filepath.Join(dataDir, fmt.Sprintf("file-%02d.bin", i)) files[path] = bytesPattern( fmt.Sprintf("file-%02d-", i), missingBlobFileBytes) require.NoError(t, afero.WriteFile(fs, path, files[path], 0o644)) } storer, err := storage.NewFileStorer(t.TempDir()) require.NoError(t, err) db, err := database.New(ctx, dbPath) require.NoError(t, err) repos := database.NewRepositories(db) id := fullFaultBackup( ctx, t, fs, storer, cfg, repos, dataDir, dbPath, "missingblob") blobOfFile := make(map[string]string, len(files)) for path := range files { blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path) } lostBlob := blobOfFile[filepath.Join(dataDir, "file-00.bin")] lost := make(map[string]bool) for path, hash := range blobOfFile { if hash == lostBlob { lost[path] = true } } require.Less(t, len(lost), len(files), "the snapshot must span more than one blob") return &missingBlobBackup{ fs: fs, cfg: cfg, storer: storer, snapshotID: id, restoreDir: filepath.Join(tempDir, "restored"), files: files, lost: lost, }, db, lostBlob } // blobHashOfSingleChunkFile returns the hash of the blob holding the one // chunk of the file at path, as recorded in the local index. func blobHashOfSingleChunkFile( ctx context.Context, t *testing.T, repos *database.Repositories, path string, ) string { t.Helper() chunks, err := repos.FileChunks.GetByPath(ctx, path) require.NoError(t, err) require.Lenf(t, chunks, 1, "%s must be a single chunk", path) blobChunk, err := repos.BlobChunks.GetByChunkHash( ctx, chunks[0].ChunkHash.String()) require.NoError(t, err) require.NotNilf(t, blobChunk, "chunk of %s is in no blob", path) blob, err := repos.Blobs.GetByID(ctx, blobChunk.BlobID.String()) require.NoError(t, err) return blob.Hash.String() }