package vaultik_test import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "path" "path/filepath" "slices" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/require" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/vaultik" ) // TestShallowVerifyDetectsWrongBlobSize backs up a real snapshot, runs // shallow verify (which passes and reports exactly the blobs it checked), // then grows one stored blob so its size no longer matches the manifest. // Shallow verify must then fail, count the grown blob as a size mismatch, // and drop it from the verified count rather than continuing to report it // as checked, and its error must not repeat the cli layer's prefix. func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() fs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "source") storeDir := t.TempDir() dbPath := filepath.Join(tempDir, "index.sqlite") chunkSize := int64(32 * 1024) maxBlobSize := int64(128 * 1024) // Enough data to span several blobs, so the mismatch count and the // dropped verified count are both meaningful. require.NoError(t, fs.MkdirAll(dataDir, 0o755)) require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "data.bin"), bytesPattern("shallow-", int(maxBlobSize*4)), 0o644)) ctx := context.Background() cfg, storer, snapshotID := runFileStorageBackup( ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) newVerifier := func(out io.Writer) *vaultik.Vaultik { v := &vaultik.Vaultik{ Config: cfg, Storage: storer, Fs: fs, Stdout: out, Stderr: io.Discard, UI: ui.NewWithColor(io.Discard, false), } v.SetContext(ctx) return v } var out bytes.Buffer require.NoError(t, newVerifier(&out).VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{JSON: true}), "shallow verify should pass on a healthy snapshot") healthy := decodeVerifyResult(t, out.Bytes()) require.Equal(t, "ok", healthy.Status) require.Positive(t, healthy.BlobCount) require.Equal(t, healthy.BlobCount, healthy.Verified, "shallow verify must report exactly the blobs it checked") require.Zero(t, healthy.Mismatched) // Grow one stored blob so its size no longer matches the manifest. growOneBlob(t, fs, filepath.Join(storeDir, "blobs")) out.Reset() err := newVerifier(&out).VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{JSON: true}) require.Error(t, err, "shallow verify must fail when a blob's stored size differs from the manifest") bad := decodeVerifyResult(t, out.Bytes()) require.Equal(t, "failed", bad.Status) require.Equal(t, 1, bad.Mismatched) require.Equal(t, healthy.BlobCount-1, bad.Verified, "the wrong-sized blob must not be counted as verified") // The cli layer prints this error after its own "Verification failed:". err = newVerifier(io.Discard).VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{}) require.EqualError(t, err, "problems found in storage: 1 blobs have the wrong size") } // TestShallowVerifyDetectsMissingDatabase backs up a real snapshot, // confirms shallow verify passes, then deletes the snapshot's encrypted // database. Shallow verify must fail: a snapshot without its database is // not restorable, even when every blob is present. func TestShallowVerifyDetectsMissingDatabase(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() fs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "source") storeDir := t.TempDir() dbPath := filepath.Join(tempDir, "index.sqlite") chunkSize := int64(32 * 1024) maxBlobSize := int64(128 * 1024) require.NoError(t, fs.MkdirAll(dataDir, 0o755)) require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "data.bin"), bytesPattern("shallow-db-", int(maxBlobSize*2)), 0o644)) ctx := context.Background() cfg, storer, snapshotID := runFileStorageBackup( ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) newVerifier := func() *vaultik.Vaultik { v := &vaultik.Vaultik{ Config: cfg, Storage: storer, Fs: fs, Stdout: io.Discard, Stderr: io.Discard, UI: ui.NewWithColor(io.Discard, false), } v.SetContext(ctx) return v } require.NoError(t, newVerifier().VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{}), "shallow verify should pass on a healthy snapshot") // The database lives under the hashed remote key, not the human ID. dbObject := filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID), "db.zst.age") require.NoError(t, os.Remove(dbObject)) require.Error(t, newVerifier().VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{}), "shallow verify must fail when db.zst.age is absent") } // errStatDenied stands in for any storage error other than not-found: a // permission error, an S3 403, a dropped connection. var errStatDenied = errors.New("injected stat failure") // statFailingStorer fails Stat with errStatDenied for the keys in // failKeys and passes every other call to the wrapped Storer. type statFailingStorer struct { storage.Storer failKeys []string } func (s statFailingStorer) Stat( ctx context.Context, key string, ) (*storage.ObjectInfo, error) { if slices.Contains(s.failKeys, key) { return nil, fmt.Errorf("stat %q: %w", key, errStatDenied) } return s.Storer.Stat(ctx, key) } // TestShallowVerifyReportsStatErrorsApartFromMissing backs up a real // snapshot, deletes one blob, and makes storage fail Stat with an error // other than not-found for a second blob and for the encrypted database. // Only the deleted blob may be counted as missing. The second blob and // the database must be reported as could not be checked, with the error // storage returned, and verify must fail, in the text output and under // --json. func TestShallowVerifyReportsStatErrorsApartFromMissing(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() fs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "source") storeDir := t.TempDir() dbPath := filepath.Join(tempDir, "index.sqlite") chunkSize := int64(32 * 1024) maxBlobSize := int64(128 * 1024) // Random content, so the snapshot spans several blobs. require.NoError(t, fs.MkdirAll(dataDir, 0o755)) appendRandomBytes(t, fs, map[string][]byte{}, filepath.Join(dataDir, "data.bin"), maxBlobSize*4) ctx := context.Background() cfg, storer, snapshotID := runFileStorageBackup( ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) blobKeys := storedBlobKeys(t, storeDir) require.GreaterOrEqual(t, len(blobKeys), 2, "expected at least two blobs") missingKey, uncheckedKey := blobKeys[0], blobKeys[1] missingHash, uncheckedHash := path.Base(missingKey), path.Base(uncheckedKey) require.NoError(t, os.Remove(filepath.Join(storeDir, missingKey))) dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age" failing := statFailingStorer{ Storer: storer, failKeys: []string{uncheckedKey, dbKey}, } newVerifier := func(out io.Writer) *vaultik.Vaultik { v := &vaultik.Vaultik{ Config: cfg, Storage: failing, Fs: fs, Stdout: out, Stderr: io.Discard, UI: ui.NewWithColor(io.Discard, false), } v.SetContext(ctx) return v } var out bytes.Buffer err := newVerifier(&out).VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{JSON: true}) require.Error(t, err) result := decodeVerifyResult(t, out.Bytes()) require.Equal(t, "failed", result.Status) require.Equal(t, 1, result.Missing, "only the deleted blob is missing") require.Equal(t, result.BlobCount-2, result.Verified) require.Len(t, result.Unchecked, 1) require.Equal(t, uncheckedHash, result.Unchecked[0].Hash) require.Contains(t, result.Unchecked[0].Error, errStatDenied.Error()) require.False(t, result.DatabaseMissing, "a database that could not be checked is not missing") require.Contains(t, result.DatabaseCheckError, errStatDenied.Error()) out.Reset() err = newVerifier(&out).VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{}) require.Error(t, err) text := out.String() require.Contains(t, text, "Missing: "+missingHash) require.NotContains(t, text, "Missing: "+uncheckedHash) require.Contains(t, text, "Could not check: "+uncheckedHash) require.Contains(t, text, "Encrypted database: could not be checked") require.Contains(t, text, errStatDenied.Error()) require.Contains(t, text, "FAILED") } // TestShallowVerifyQuietFailsOnStatErrorAlone backs up a healthy snapshot, // then makes storage fail Stat, with an error other than not-found, for // one blob alone and then for the encrypted database alone. Each run must // fail verify, and under --quiet, which drops the report, the error // storage returned must still reach stderr. // //nolint:paralleltest // captureProcessStderr replaces os.Stderr func TestShallowVerifyQuietFailsOnStatErrorAlone(t *testing.T) { log.Initialize(log.Config{}) fs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "source") storeDir := t.TempDir() dbPath := filepath.Join(tempDir, "index.sqlite") chunkSize := int64(32 * 1024) maxBlobSize := int64(128 * 1024) require.NoError(t, fs.MkdirAll(dataDir, 0o755)) require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "data.bin"), bytesPattern("shallow-quiet-", int(maxBlobSize*2)), 0o644)) ctx := context.Background() cfg, storer, snapshotID := runFileStorageBackup( ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) newQuietVerifier := func(s storage.Storer) *vaultik.Vaultik { v := &vaultik.Vaultik{ Config: cfg, Storage: s, Fs: fs, Stdout: io.Discard, Stderr: io.Discard, UI: ui.NewWithColor(io.Discard, false), } v.SetContext(ctx) v.UI.SetQuiet(true) return v } require.NoError(t, newQuietVerifier(storer).VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{}), "shallow verify should pass on a healthy snapshot") blobKeys := storedBlobKeys(t, storeDir) require.NotEmpty(t, blobKeys) dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age" for _, failKey := range []string{blobKeys[0], dbKey} { v := newQuietVerifier(statFailingStorer{ Storer: storer, failKeys: []string{failKey}, }) var err error stderr := captureProcessStderr(t, func() { err = v.VerifySnapshotWithOptions( snapshotID, &vaultik.VerifyOptions{}) }) require.Error(t, err, "verify must fail when %s could not be checked", failKey) require.Contains(t, stderr, errStatDenied.Error(), "under --quiet the error for %s must still reach stderr", failKey) } } // storedBlobKeys returns the storage key of every blob file under // storeDir, sorted. func storedBlobKeys(t *testing.T, storeDir string) []string { t.Helper() var keys []string err := filepath.Walk(filepath.Join(storeDir, "blobs"), func(p string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { return nil } rel, err := filepath.Rel(storeDir, p) if err != nil { return err } keys = append(keys, filepath.ToSlash(rel)) return nil }) require.NoError(t, err) return keys } // growOneBlob appends bytes to the first blob file found under blobsDir, // changing its on-disk size so it no longer matches the manifest. func growOneBlob(t *testing.T, fs afero.Fs, blobsDir string) { t.Helper() var blobPath string err := afero.Walk(fs, blobsDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if blobPath == "" && !info.IsDir() { blobPath = path } return nil }) require.NoError(t, err) require.NotEmpty(t, blobPath, "expected at least one blob on disk") data, err := afero.ReadFile(fs, blobPath) require.NoError(t, err) data = append(data, []byte("extra")...) require.NoError(t, afero.WriteFile(fs, blobPath, data, 0o644)) } func decodeVerifyResult(t *testing.T, b []byte) vaultik.VerifyResult { t.Helper() var result vaultik.VerifyResult require.NoError(t, json.Unmarshal(b, &result)) return result }