clawbot
42eb50b151
check / check (push) Waiting to run
Deep verify set verified only on success and never set missing, so a failed --deep --json run reported zero for both. It now counts each blob whose contents pass, and its existence check covers every blob, counting not-found ones as missing and wrong-size ones as mismatched, as shallow verify does; it fails if it found any. Any other storage error stops the check and fails with that error, so a blob that cannot be checked is never reported as missing. Deep verify still stops at the first damaged blob and reads no contents while any blob is missing or the wrong size, so verified is 0 there. Model: opus-5-5
281 lines
8.7 KiB
Go
281 lines
8.7 KiB
Go
package vaultik_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/ui"
|
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
)
|
|
|
|
// TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob backs up a real
|
|
// snapshot with the on-disk storage backend, runs deep verification on
|
|
// it, then flips a byte inside one stored blob and runs deep
|
|
// verification again. A healthy snapshot must pass; a corrupted blob
|
|
// must fail. The healthy case is the regression guard: deep
|
|
// verification used to hash the encrypted blob bytes and compare them
|
|
// to the blob's ID (the double SHA256 of the plaintext), so it reported
|
|
// every healthy blob as corrupt.
|
|
func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := filepath.Join(tempDir, "remote")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(64 * 1024)
|
|
maxBlobSize := int64(512 * 1024)
|
|
|
|
// One file large enough to span several chunks within a single blob.
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"),
|
|
bytesPattern("deep-", int(chunkSize*3)), 0o644))
|
|
|
|
ctx := context.Background()
|
|
|
|
// runFileStorageBackup writes a real snapshot to storeDir and closes
|
|
// the source index, so verification runs from remote bytes only.
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
newVerifier := func() *vaultik.Vaultik {
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
return v
|
|
}
|
|
|
|
require.NoError(t,
|
|
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
|
"deep verify should pass on a healthy snapshot")
|
|
|
|
// Flip a byte inside one blob without changing its length, so the
|
|
// blob-existence and size checks still pass and verification reaches
|
|
// the blob-content stage.
|
|
corruptOneBlob(t, fs, filepath.Join(storeDir, "blobs"))
|
|
|
|
require.Error(t,
|
|
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
|
"deep verify should fail on a corrupted blob")
|
|
}
|
|
|
|
// TestDeepVerifyJSONCountsBlobs checks the verified and missing counts
|
|
// that deep verify reports under --json: on a healthy snapshot, with a
|
|
// damaged blob, and with missing blobs. Deep verify stops at the first
|
|
// damaged blob, so the blobs read before it are the ones verified; when
|
|
// blobs are missing it reads no blob contents, but counts every missing
|
|
// blob.
|
|
func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
ctx := context.Background()
|
|
storeDir := filepath.Join(t.TempDir(), "remote")
|
|
|
|
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
|
|
|
healthy, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "ok", healthy.Status)
|
|
require.Len(t, keys, healthy.BlobCount)
|
|
require.Equal(t, healthy.BlobCount, healthy.Verified)
|
|
require.Zero(t, healthy.Missing)
|
|
|
|
// Damage the last blob read, keeping its length, so every blob before
|
|
// it is read and passes first.
|
|
lastPath := filepath.Join(storeDir, keys[len(keys)-1])
|
|
blob, err := afero.ReadFile(fs, lastPath)
|
|
require.NoError(t, err)
|
|
|
|
blob[len(blob)/2] ^= 0xff
|
|
require.NoError(t, afero.WriteFile(fs, lastPath, blob, 0o600))
|
|
|
|
got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Equal(t, healthy.BlobCount-1, got.Verified)
|
|
require.Zero(t, got.Missing)
|
|
|
|
// Delete the first blob and the damaged last one: both must be
|
|
// counted as missing, not only the first one found.
|
|
var deletedSize int64
|
|
|
|
for _, key := range []string{keys[0], keys[len(keys)-1]} {
|
|
info, statErr := storer.Stat(ctx, key)
|
|
require.NoError(t, statErr)
|
|
|
|
deletedSize += info.Size
|
|
|
|
require.NoError(t, storer.Delete(ctx, key))
|
|
}
|
|
|
|
got, err = deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Equal(t, 2, got.Missing)
|
|
require.Equal(t, deletedSize, got.MissingSize)
|
|
require.Zero(t, got.Verified)
|
|
}
|
|
|
|
// TestDeepVerifyJSONCountsWrongSizeBlob checks that under --deep --json a
|
|
// blob of the wrong size is counted in mismatched and fails the run, and
|
|
// that the check goes on past it to count a missing blob after it.
|
|
func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
ctx := context.Background()
|
|
storeDir := filepath.Join(t.TempDir(), "remote")
|
|
|
|
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
|
|
|
// Make the first blob checked longer, and delete the last one.
|
|
firstPath := filepath.Join(storeDir, keys[0])
|
|
blob, err := afero.ReadFile(fs, firstPath)
|
|
require.NoError(t, err)
|
|
|
|
blob = append(blob, []byte("extra")...)
|
|
require.NoError(t, afero.WriteFile(fs, firstPath, blob, 0o600))
|
|
require.NoError(t, storer.Delete(ctx, keys[len(keys)-1]))
|
|
|
|
got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Equal(t, 1, got.Mismatched)
|
|
require.Equal(t, 1, got.Missing)
|
|
require.Zero(t, got.Verified)
|
|
}
|
|
|
|
// TestDeepVerifyStopsOnBlobThatCannotBeChecked checks that a blob that is
|
|
// present in storage but cannot be checked is not counted as missing:
|
|
// deep verify stops and fails with the storage error.
|
|
func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
storeDir := filepath.Join(t.TempDir(), "remote")
|
|
|
|
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
|
failing := statFailingStorer{Storer: storer, failKeys: []string{keys[0]}}
|
|
|
|
got, err := deepVerifyJSON(ctx, t, cfg, failing, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Zero(t, got.Missing)
|
|
require.Contains(t, got.ErrorMessage, errStatDenied.Error())
|
|
}
|
|
|
|
// backUpSeveralBlobs backs up random data, which does not compress or
|
|
// deduplicate, so it fills at least three blobs in a store on disk at
|
|
// storeDir. It returns the blobs' storage keys sorted, which is hash
|
|
// order, the order deep verify reads them in.
|
|
func backUpSeveralBlobs(
|
|
ctx context.Context, t *testing.T, storeDir string,
|
|
) (*config.Config, *storage.FileStorer, string, []string) {
|
|
t.Helper()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(32 * 1024)
|
|
maxBlobSize := int64(128 * 1024)
|
|
|
|
data := make([]byte, maxBlobSize*4)
|
|
_, err := rand.Read(data)
|
|
require.NoError(t, err)
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"), data, 0o644))
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
keys := storedBlobKeys(t, storeDir)
|
|
require.GreaterOrEqual(t, len(keys), 3)
|
|
|
|
return cfg, storer, snapshotID, keys
|
|
}
|
|
|
|
// deepVerifyJSON runs deep verify with --json on snapshotID and returns
|
|
// the decoded result and the error RunDeepVerify returned.
|
|
func deepVerifyJSON(
|
|
ctx context.Context, t *testing.T,
|
|
cfg *config.Config, storer storage.Storer, snapshotID string,
|
|
) (vaultik.VerifyResult, error) {
|
|
t.Helper()
|
|
|
|
var out bytes.Buffer
|
|
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: afero.NewOsFs(),
|
|
Stdout: &out,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
err := v.RunDeepVerify(snapshotID,
|
|
&vaultik.VerifyOptions{Deep: true, JSON: true})
|
|
|
|
return decodeVerifyResult(t, out.Bytes()), err
|
|
}
|
|
|
|
// corruptOneBlob flips a middle byte of the first blob file found under
|
|
// blobsDir, leaving the file length unchanged.
|
|
func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
|
|
t.Helper()
|
|
|
|
var blobPath string
|
|
|
|
err := afero.Walk(fs, blobsDir,
|
|
func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if blobPath == "" && !info.IsDir() {
|
|
blobPath = path
|
|
}
|
|
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, blobPath, "expected at least one blob on disk")
|
|
|
|
data, err := afero.ReadFile(fs, blobPath)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, data)
|
|
|
|
data[len(data)/2] ^= 0xff
|
|
require.NoError(t, afero.WriteFile(fs, blobPath, data, 0o644))
|
|
}
|