clawbot
4193ced1f6
check / check (push) Waiting to run
Shallow snapshot verify counted every error from checking a blob or the encrypted database as "missing" and printed the error nowhere, so a permission error or an S3 403 read as data loss. Now only a not-found answer from storage counts as missing. Any other error is reported as could not be checked, with the error storage returned, in the text report and under --json (the new unchecked list and database_check_error field), and the run fails. Under --quiet, which drops the report, each of those errors is logged as a warning on stderr. The database check moved into its own small function because the verify function was at the linter's length limit. Model: opus-5-5
415 lines
12 KiB
Go
415 lines
12 KiB
Go
package vaultik_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/ui"
|
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
)
|
|
|
|
// TestShallowVerifyDetectsWrongBlobSize backs up a real snapshot, runs
|
|
// shallow verify (which passes and reports exactly the blobs it checked),
|
|
// then grows one stored blob so its size no longer matches the manifest.
|
|
// Shallow verify must then fail, count the grown blob as a size mismatch,
|
|
// and drop it from the verified count rather than continuing to report it
|
|
// as checked.
|
|
func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(32 * 1024)
|
|
maxBlobSize := int64(128 * 1024)
|
|
|
|
// Enough data to span several blobs, so the mismatch count and the
|
|
// dropped verified count are both meaningful.
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"),
|
|
bytesPattern("shallow-", int(maxBlobSize*4)), 0o644))
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
newVerifier := func(out io.Writer) *vaultik.Vaultik {
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: fs,
|
|
Stdout: out,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
return v
|
|
}
|
|
|
|
var out bytes.Buffer
|
|
|
|
require.NoError(t,
|
|
newVerifier(&out).VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{JSON: true}),
|
|
"shallow verify should pass on a healthy snapshot")
|
|
|
|
healthy := decodeVerifyResult(t, out.Bytes())
|
|
require.Equal(t, "ok", healthy.Status)
|
|
require.Positive(t, healthy.BlobCount)
|
|
require.Equal(t, healthy.BlobCount, healthy.Verified,
|
|
"shallow verify must report exactly the blobs it checked")
|
|
require.Zero(t, healthy.Mismatched)
|
|
|
|
// Grow one stored blob so its size no longer matches the manifest.
|
|
growOneBlob(t, fs, filepath.Join(storeDir, "blobs"))
|
|
|
|
out.Reset()
|
|
err := newVerifier(&out).VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{JSON: true})
|
|
require.Error(t, err,
|
|
"shallow verify must fail when a blob's stored size differs from the manifest")
|
|
|
|
bad := decodeVerifyResult(t, out.Bytes())
|
|
require.Equal(t, "failed", bad.Status)
|
|
require.Equal(t, 1, bad.Mismatched)
|
|
require.Equal(t, healthy.BlobCount-1, bad.Verified,
|
|
"the wrong-sized blob must not be counted as verified")
|
|
}
|
|
|
|
// TestShallowVerifyDetectsMissingDatabase backs up a real snapshot,
|
|
// confirms shallow verify passes, then deletes the snapshot's encrypted
|
|
// database. Shallow verify must fail: a snapshot without its database is
|
|
// not restorable, even when every blob is present.
|
|
func TestShallowVerifyDetectsMissingDatabase(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(32 * 1024)
|
|
maxBlobSize := int64(128 * 1024)
|
|
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"),
|
|
bytesPattern("shallow-db-", int(maxBlobSize*2)), 0o644))
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
newVerifier := func() *vaultik.Vaultik {
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
return v
|
|
}
|
|
|
|
require.NoError(t,
|
|
newVerifier().VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{}),
|
|
"shallow verify should pass on a healthy snapshot")
|
|
|
|
// The database lives under the hashed remote key, not the human ID.
|
|
dbObject := filepath.Join(storeDir, "metadata",
|
|
snapshot.RemoteSnapshotKey(snapshotID), "db.zst.age")
|
|
require.NoError(t, os.Remove(dbObject))
|
|
|
|
require.Error(t,
|
|
newVerifier().VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{}),
|
|
"shallow verify must fail when db.zst.age is absent")
|
|
}
|
|
|
|
// errStatDenied stands in for any storage error other than not-found: a
|
|
// permission error, an S3 403, a dropped connection.
|
|
var errStatDenied = errors.New("injected stat failure")
|
|
|
|
// statFailingStorer fails Stat with errStatDenied for the keys in
|
|
// failKeys and passes every other call to the wrapped Storer.
|
|
type statFailingStorer struct {
|
|
storage.Storer
|
|
|
|
failKeys []string
|
|
}
|
|
|
|
func (s statFailingStorer) Stat(
|
|
ctx context.Context, key string,
|
|
) (*storage.ObjectInfo, error) {
|
|
if slices.Contains(s.failKeys, key) {
|
|
return nil, fmt.Errorf("stat %q: %w", key, errStatDenied)
|
|
}
|
|
|
|
return s.Storer.Stat(ctx, key)
|
|
}
|
|
|
|
// TestShallowVerifyReportsStatErrorsApartFromMissing backs up a real
|
|
// snapshot, deletes one blob, and makes storage fail Stat with an error
|
|
// other than not-found for a second blob and for the encrypted database.
|
|
// Only the deleted blob may be counted as missing. The second blob and
|
|
// the database must be reported as could not be checked, with the error
|
|
// storage returned, and verify must fail, in the text output and under
|
|
// --json.
|
|
func TestShallowVerifyReportsStatErrorsApartFromMissing(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(32 * 1024)
|
|
maxBlobSize := int64(128 * 1024)
|
|
|
|
// Random content, so the snapshot spans several blobs.
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
appendRandomBytes(t, fs, map[string][]byte{},
|
|
filepath.Join(dataDir, "data.bin"), maxBlobSize*4)
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
blobKeys := storedBlobKeys(t, storeDir)
|
|
require.GreaterOrEqual(t, len(blobKeys), 2, "expected at least two blobs")
|
|
|
|
missingKey, uncheckedKey := blobKeys[0], blobKeys[1]
|
|
missingHash, uncheckedHash := path.Base(missingKey), path.Base(uncheckedKey)
|
|
|
|
require.NoError(t, os.Remove(filepath.Join(storeDir, missingKey)))
|
|
|
|
dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age"
|
|
failing := statFailingStorer{
|
|
Storer: storer,
|
|
failKeys: []string{uncheckedKey, dbKey},
|
|
}
|
|
|
|
newVerifier := func(out io.Writer) *vaultik.Vaultik {
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: failing,
|
|
Fs: fs,
|
|
Stdout: out,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
return v
|
|
}
|
|
|
|
var out bytes.Buffer
|
|
|
|
err := newVerifier(&out).VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{JSON: true})
|
|
require.Error(t, err)
|
|
|
|
result := decodeVerifyResult(t, out.Bytes())
|
|
require.Equal(t, "failed", result.Status)
|
|
require.Equal(t, 1, result.Missing, "only the deleted blob is missing")
|
|
require.Equal(t, result.BlobCount-2, result.Verified)
|
|
require.Len(t, result.Unchecked, 1)
|
|
require.Equal(t, uncheckedHash, result.Unchecked[0].Hash)
|
|
require.Contains(t, result.Unchecked[0].Error, errStatDenied.Error())
|
|
require.False(t, result.DatabaseMissing,
|
|
"a database that could not be checked is not missing")
|
|
require.Contains(t, result.DatabaseCheckError, errStatDenied.Error())
|
|
|
|
out.Reset()
|
|
err = newVerifier(&out).VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{})
|
|
require.Error(t, err)
|
|
|
|
text := out.String()
|
|
require.Contains(t, text, "Missing: "+missingHash)
|
|
require.NotContains(t, text, "Missing: "+uncheckedHash)
|
|
require.Contains(t, text, "Could not check: "+uncheckedHash)
|
|
require.Contains(t, text, "Encrypted database: could not be checked")
|
|
require.Contains(t, text, errStatDenied.Error())
|
|
require.Contains(t, text, "FAILED")
|
|
}
|
|
|
|
// TestShallowVerifyQuietFailsOnStatErrorAlone backs up a healthy snapshot,
|
|
// then makes storage fail Stat, with an error other than not-found, for
|
|
// one blob alone and then for the encrypted database alone. Each run must
|
|
// fail verify, and under --quiet, which drops the report, the error
|
|
// storage returned must still reach stderr.
|
|
//
|
|
//nolint:paralleltest // captureProcessStderr replaces os.Stderr
|
|
func TestShallowVerifyQuietFailsOnStatErrorAlone(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(32 * 1024)
|
|
maxBlobSize := int64(128 * 1024)
|
|
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"),
|
|
bytesPattern("shallow-quiet-", int(maxBlobSize*2)), 0o644))
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
newQuietVerifier := func(s storage.Storer) *vaultik.Vaultik {
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: s,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
v.UI.SetQuiet(true)
|
|
|
|
return v
|
|
}
|
|
|
|
require.NoError(t,
|
|
newQuietVerifier(storer).VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{}),
|
|
"shallow verify should pass on a healthy snapshot")
|
|
|
|
blobKeys := storedBlobKeys(t, storeDir)
|
|
require.NotEmpty(t, blobKeys)
|
|
|
|
dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age"
|
|
|
|
for _, failKey := range []string{blobKeys[0], dbKey} {
|
|
v := newQuietVerifier(statFailingStorer{
|
|
Storer: storer,
|
|
failKeys: []string{failKey},
|
|
})
|
|
|
|
var err error
|
|
|
|
stderr := captureProcessStderr(t, func() {
|
|
err = v.VerifySnapshotWithOptions(
|
|
snapshotID, &vaultik.VerifyOptions{})
|
|
})
|
|
|
|
require.Error(t, err,
|
|
"verify must fail when %s could not be checked", failKey)
|
|
require.Contains(t, stderr, errStatDenied.Error(),
|
|
"under --quiet the error for %s must still reach stderr", failKey)
|
|
}
|
|
}
|
|
|
|
// storedBlobKeys returns the storage key of every blob file under
|
|
// storeDir, sorted.
|
|
func storedBlobKeys(t *testing.T, storeDir string) []string {
|
|
t.Helper()
|
|
|
|
var keys []string
|
|
|
|
err := filepath.Walk(filepath.Join(storeDir, "blobs"),
|
|
func(p string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
rel, err := filepath.Rel(storeDir, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
keys = append(keys, filepath.ToSlash(rel))
|
|
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
return keys
|
|
}
|
|
|
|
// growOneBlob appends bytes to the first blob file found under blobsDir,
|
|
// changing its on-disk size so it no longer matches the manifest.
|
|
func growOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
|
|
t.Helper()
|
|
|
|
var blobPath string
|
|
|
|
err := afero.Walk(fs, blobsDir,
|
|
func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if blobPath == "" && !info.IsDir() {
|
|
blobPath = path
|
|
}
|
|
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, blobPath, "expected at least one blob on disk")
|
|
|
|
data, err := afero.ReadFile(fs, blobPath)
|
|
require.NoError(t, err)
|
|
|
|
data = append(data, []byte("extra")...)
|
|
require.NoError(t, afero.WriteFile(fs, blobPath, data, 0o644))
|
|
}
|
|
|
|
func decodeVerifyResult(t *testing.T, b []byte) vaultik.VerifyResult {
|
|
t.Helper()
|
|
|
|
var result vaultik.VerifyResult
|
|
|
|
require.NoError(t, json.Unmarshal(b, &result))
|
|
|
|
return result
|
|
}
|