sneak
52f44af4e2
check / check (push) Waiting to run
Every backup writes vaultik-destination.txt at the destination root. While the local index holds snapshots, a destination without it is not present: snapshot create refuses before writing, prune and snapshot purge fail, and snapshot list and snapshot remove act as for a destination that cannot be listed. snapshot create checks before it drops incomplete snapshots, snapshot remove before it looks on the destination. remote nuke deletes the marker last. The file:// backend never creates its destination directory; that protects a directory on a removable volume, not a mount point. Tests back up into directories that exist; fixtures for a written destination carry the marker. Judgement call: the marker is rewritten on every backup. Deviation: for the complexity limit, two blocks moved, unchanged, into snapshotNamesToCreate and confirmRemoveSnapshot. Model: opus-5-5
288 lines
8.9 KiB
Go
288 lines
8.9 KiB
Go
package vaultik_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/ui"
|
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
)
|
|
|
|
// TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob backs up a real
|
|
// snapshot with the on-disk storage backend, runs deep verification on
|
|
// it, then flips a byte inside one stored blob and runs deep
|
|
// verification again. A healthy snapshot must pass; a corrupted blob
|
|
// must fail. The healthy case is the regression guard: deep
|
|
// verification used to hash the encrypted blob bytes and compare them
|
|
// to the blob's ID (the double SHA256 of the plaintext), so it reported
|
|
// every healthy blob as corrupt.
|
|
func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
storeDir := t.TempDir()
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(64 * 1024)
|
|
maxBlobSize := int64(512 * 1024)
|
|
|
|
// One file large enough to span several chunks within a single blob.
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"),
|
|
bytesPattern("deep-", int(chunkSize*3)), 0o644))
|
|
|
|
ctx := context.Background()
|
|
|
|
// runFileStorageBackup writes a real snapshot to storeDir and closes
|
|
// the source index, so verification runs from remote bytes only.
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
newVerifier := func() *vaultik.Vaultik {
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: fs,
|
|
Stdout: io.Discard,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
return v
|
|
}
|
|
|
|
require.NoError(t,
|
|
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
|
"deep verify should pass on a healthy snapshot")
|
|
|
|
// Flip a byte inside one blob without changing its length, so the
|
|
// blob-existence and size checks still pass and verification reaches
|
|
// the blob-content stage.
|
|
corruptOneBlob(t, fs, filepath.Join(storeDir, "blobs"))
|
|
|
|
require.Error(t,
|
|
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
|
"deep verify should fail on a corrupted blob")
|
|
}
|
|
|
|
// TestDeepVerifyJSONCountsBlobs checks the verified and missing counts
|
|
// that deep verify reports under --json: on a healthy snapshot, with a
|
|
// damaged blob, and with missing blobs. Deep verify stops at the first
|
|
// damaged blob, so the blobs read before it are the ones verified; when
|
|
// blobs are missing it reads no blob contents, but counts every missing
|
|
// blob.
|
|
func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
ctx := context.Background()
|
|
storeDir := t.TempDir()
|
|
|
|
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
|
|
|
healthy, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "ok", healthy.Status)
|
|
require.Len(t, keys, healthy.BlobCount)
|
|
require.Equal(t, healthy.BlobCount, healthy.Verified)
|
|
require.Zero(t, healthy.Missing)
|
|
|
|
// Damage the last blob read, keeping its length, so every blob before
|
|
// it is read and passes first.
|
|
lastPath := filepath.Join(storeDir, keys[len(keys)-1])
|
|
blob, err := afero.ReadFile(fs, lastPath)
|
|
require.NoError(t, err)
|
|
|
|
blob[len(blob)/2] ^= 0xff
|
|
require.NoError(t, afero.WriteFile(fs, lastPath, blob, 0o600))
|
|
|
|
got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Equal(t, healthy.BlobCount-1, got.Verified)
|
|
require.Zero(t, got.Missing)
|
|
|
|
// Delete the first blob and the damaged last one: both must be
|
|
// counted as missing, not only the first one found.
|
|
var deletedSize int64
|
|
|
|
for _, key := range []string{keys[0], keys[len(keys)-1]} {
|
|
info, statErr := storer.Stat(ctx, key)
|
|
require.NoError(t, statErr)
|
|
|
|
deletedSize += info.Size
|
|
|
|
require.NoError(t, storer.Delete(ctx, key))
|
|
}
|
|
|
|
got, err = deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Equal(t, 2, got.Missing)
|
|
require.Equal(t, deletedSize, got.MissingSize)
|
|
require.Zero(t, got.Verified)
|
|
}
|
|
|
|
// TestDeepVerifyJSONCountsWrongSizeBlob checks that under --deep --json a
|
|
// blob of the wrong size is counted in mismatched and fails the run, and
|
|
// that the check goes on past it to count a missing blob after it. The
|
|
// store is on disk, so the message must not call it S3.
|
|
func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
ctx := context.Background()
|
|
storeDir := t.TempDir()
|
|
|
|
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
|
|
|
// Make the first blob checked longer, and delete the last one.
|
|
firstPath := filepath.Join(storeDir, keys[0])
|
|
blob, err := afero.ReadFile(fs, firstPath)
|
|
require.NoError(t, err)
|
|
|
|
blob = append(blob, []byte("extra")...)
|
|
require.NoError(t, afero.WriteFile(fs, firstPath, blob, 0o600))
|
|
require.NoError(t, storer.Delete(ctx, keys[len(keys)-1]))
|
|
|
|
got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Equal(t, 1, got.Mismatched)
|
|
require.Equal(t, 1, got.Missing)
|
|
require.Zero(t, got.Verified)
|
|
require.Contains(t, got.ErrorMessage, "with the wrong size")
|
|
require.NotContains(t, got.ErrorMessage, "S3")
|
|
}
|
|
|
|
// TestDeepVerifyStopsOnBlobThatCannotBeChecked checks that a blob that is
|
|
// present in storage but cannot be checked is not counted as missing:
|
|
// deep verify stops and fails, saying it could not check the blob, with
|
|
// the storage error.
|
|
func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
ctx := context.Background()
|
|
storeDir := t.TempDir()
|
|
|
|
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
|
failing := statFailingStorer{Storer: storer, failKeys: []string{keys[0]}}
|
|
|
|
got, err := deepVerifyJSON(ctx, t, cfg, failing, snapshotID)
|
|
require.Error(t, err)
|
|
require.Equal(t, "failed", got.Status)
|
|
require.Zero(t, got.Missing)
|
|
require.Contains(t, got.ErrorMessage,
|
|
"could not check blob "+path.Base(keys[0]))
|
|
require.Contains(t, got.ErrorMessage, errStatDenied.Error())
|
|
}
|
|
|
|
// backUpSeveralBlobs backs up random data, which does not compress or
|
|
// deduplicate, so it fills at least three blobs in a store on disk at
|
|
// storeDir. It returns the blobs' storage keys sorted, which is hash
|
|
// order, the order deep verify reads them in.
|
|
func backUpSeveralBlobs(
|
|
ctx context.Context, t *testing.T, storeDir string,
|
|
) (*config.Config, *storage.FileStorer, string, []string) {
|
|
t.Helper()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
dataDir := filepath.Join(tempDir, "source")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
|
|
chunkSize := int64(32 * 1024)
|
|
maxBlobSize := int64(128 * 1024)
|
|
|
|
data := make([]byte, maxBlobSize*4)
|
|
_, err := rand.Read(data)
|
|
require.NoError(t, err)
|
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
require.NoError(t, afero.WriteFile(fs,
|
|
filepath.Join(dataDir, "data.bin"), data, 0o644))
|
|
|
|
cfg, storer, snapshotID := runFileStorageBackup(
|
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
|
|
keys := storedBlobKeys(t, storeDir)
|
|
require.GreaterOrEqual(t, len(keys), 3)
|
|
|
|
return cfg, storer, snapshotID, keys
|
|
}
|
|
|
|
// deepVerifyJSON runs deep verify with --json on snapshotID and returns
|
|
// the decoded result and the error RunDeepVerify returned.
|
|
func deepVerifyJSON(
|
|
ctx context.Context, t *testing.T,
|
|
cfg *config.Config, storer storage.Storer, snapshotID string,
|
|
) (vaultik.VerifyResult, error) {
|
|
t.Helper()
|
|
|
|
var out bytes.Buffer
|
|
|
|
v := &vaultik.Vaultik{
|
|
Config: cfg,
|
|
Storage: storer,
|
|
Fs: afero.NewOsFs(),
|
|
Stdout: &out,
|
|
Stderr: io.Discard,
|
|
UI: ui.NewWithColor(io.Discard, false),
|
|
}
|
|
v.SetContext(ctx)
|
|
|
|
err := v.RunDeepVerify(snapshotID,
|
|
&vaultik.VerifyOptions{Deep: true, JSON: true})
|
|
|
|
return decodeVerifyResult(t, out.Bytes()), err
|
|
}
|
|
|
|
// corruptOneBlob flips a middle byte of the first blob file found under
|
|
// blobsDir, leaving the file length unchanged.
|
|
func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
|
|
t.Helper()
|
|
|
|
var blobPath string
|
|
|
|
err := afero.Walk(fs, blobsDir,
|
|
func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if blobPath == "" && !info.IsDir() {
|
|
blobPath = path
|
|
}
|
|
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, blobPath, "expected at least one blob on disk")
|
|
|
|
data, err := afero.ReadFile(fs, blobPath)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, data)
|
|
|
|
data[len(data)/2] ^= 0xff
|
|
require.NoError(t, afero.WriteFile(fs, blobPath, data, 0o644))
|
|
}
|