Shorten internal/vaultik tests under the race detector (closes #416) #417
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
@@ -24,19 +25,28 @@ import (
|
||||
// that is not plugged in, cannot be listed. It is not an empty store, so
|
||||
// no command may conclude from it that the local snapshots are gone.
|
||||
|
||||
// newFileDestinationVaultik returns a Vaultik that backs up the
|
||||
// snapshot named "first" to a file:// destination at storeDir.
|
||||
// newFileDestinationVaultik writes the fault source tree and returns a
|
||||
// Vaultik that backs it up to a file:// destination at storeDir.
|
||||
func newFileDestinationVaultik(
|
||||
ctx context.Context, t *testing.T, storeDir string,
|
||||
) (*vaultik.Vaultik, *database.Repositories) {
|
||||
t.Helper()
|
||||
|
||||
osFs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
|
||||
|
||||
writeFaultSourceTree(t, osFs, dataDir)
|
||||
return openFileDestinationVaultik(ctx, t, storeDir, dataDir,
|
||||
filepath.Join(tempDir, "index.sqlite"))
|
||||
}
|
||||
|
||||
// openFileDestinationVaultik returns a Vaultik that backs up dataDir to a
|
||||
// file:// destination at storeDir, with its local index at dbPath. The
|
||||
// index is closed when the test ends.
|
||||
func openFileDestinationVaultik(
|
||||
ctx context.Context, t *testing.T, storeDir, dataDir, dbPath string,
|
||||
) (*vaultik.Vaultik, *database.Repositories) {
|
||||
t.Helper()
|
||||
|
||||
store, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
@@ -48,22 +58,84 @@ func newFileDestinationVaultik(
|
||||
repos := database.NewRepositories(db)
|
||||
cfg := changedFileConfig(dataDir, dbPath)
|
||||
|
||||
return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos
|
||||
return newBackupVaultik(ctx, cfg, store, repos, db, afero.NewOsFs()), repos
|
||||
}
|
||||
|
||||
// firstBackup is one backup of the snapshot named "first" to a file://
|
||||
// destination. dir holds its source tree in src, its store in store and
|
||||
// its closed local index in index/index.sqlite. The first test that
|
||||
// needs it makes it, and TestMain removes it. Every test that copies it
|
||||
// backs up the same source tree, so no test may change that tree.
|
||||
//
|
||||
//nolint:gochecknoglobals // made once and shared by the tests that copy it
|
||||
var firstBackup struct {
|
||||
once sync.Once
|
||||
dir string
|
||||
ready bool
|
||||
}
|
||||
|
||||
// TestMain removes firstBackup once every test has run.
|
||||
func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
|
||||
_ = os.RemoveAll(firstBackup.dir)
|
||||
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// makeFirstBackup writes the fault source tree and backs it up as the
|
||||
// snapshot named "first", all in a new directory that it records in
|
||||
// firstBackup.
|
||||
func makeFirstBackup(ctx context.Context, t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
dir, err := os.MkdirTemp("", "vaultik-first-backup-")
|
||||
require.NoError(t, err)
|
||||
|
||||
firstBackup.dir = dir
|
||||
storeDir := filepath.Join(dir, "store")
|
||||
dataDir := filepath.Join(dir, "src")
|
||||
indexDir := filepath.Join(dir, "index")
|
||||
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
require.NoError(t, os.Mkdir(indexDir, 0o750))
|
||||
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
|
||||
|
||||
v, _ := openFileDestinationVaultik(ctx, t, storeDir, dataDir,
|
||||
filepath.Join(indexDir, "index.sqlite"))
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
require.NoError(t, v.DB.Close())
|
||||
|
||||
firstBackup.ready = true
|
||||
}
|
||||
|
||||
// backUpToFileDestination creates the directory storeDir, as a user must
|
||||
// before the first backup, and backs up the snapshot named "first" to a
|
||||
// file:// destination there. Everything the returned Vaultik prints
|
||||
// after the backup goes to the returned buffer.
|
||||
// before the first backup, and returns a Vaultik whose file://
|
||||
// destination there holds a backup of the snapshot named "first". The
|
||||
// store and the local index are copies of firstBackup's, which is faster
|
||||
// than backing up in every test. The returned Vaultik backs up a source
|
||||
// tree shared with every other caller, so no test may change that tree.
|
||||
// Everything the returned Vaultik prints after the backup goes to the
|
||||
// returned buffer.
|
||||
func backUpToFileDestination(
|
||||
ctx context.Context, t *testing.T, storeDir string,
|
||||
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
|
||||
t.Helper()
|
||||
|
||||
require.NoError(t, os.MkdirAll(storeDir, 0o750))
|
||||
firstBackup.once.Do(func() { makeFirstBackup(ctx, t) })
|
||||
require.True(t, firstBackup.ready, "another test failed to make firstBackup")
|
||||
|
||||
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
require.NoError(t, os.MkdirAll(storeDir, 0o750))
|
||||
require.NoError(t, os.CopyFS(storeDir,
|
||||
os.DirFS(filepath.Join(firstBackup.dir, "store"))))
|
||||
|
||||
indexDir := t.TempDir()
|
||||
require.NoError(t, os.CopyFS(indexDir,
|
||||
os.DirFS(filepath.Join(firstBackup.dir, "index"))))
|
||||
|
||||
v, repos := openFileDestinationVaultik(ctx, t, storeDir,
|
||||
filepath.Join(firstBackup.dir, "src"),
|
||||
filepath.Join(indexDir, "index.sqlite"))
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
v.Stdout = out
|
||||
|
||||
@@ -46,11 +46,11 @@ import (
|
||||
// slice out a few KB — currently the dominant cost in restore.
|
||||
//
|
||||
// The test deliberately constructs an adversarial scenario: three
|
||||
// blobs A/B/C of ~6 MB each, nine files distributed across them, and
|
||||
// path-ordered names that interleave the blobs (a1, b1, c1, a2, b2,
|
||||
// c2, …) so naive path-order processing would touch every blob before
|
||||
// finishing any of them.
|
||||
// localitySource is one 1 MiB fixture file used by the locality test.
|
||||
// blobs A/B/C of five 64 KiB chunks each, nine files distributed across
|
||||
// them, and path-ordered names that interleave the blobs (a1, b1, c1,
|
||||
// a2, b2, c2, …) so naive path-order processing would touch every blob
|
||||
// before finishing any of them.
|
||||
// localitySource is one 64 KiB fixture file used by the locality test.
|
||||
type localitySource struct {
|
||||
path string
|
||||
data []byte
|
||||
@@ -65,10 +65,10 @@ type localityCopy struct {
|
||||
|
||||
// localitySourceBytes is the size of each source file of the locality
|
||||
// fixture.
|
||||
const localitySourceBytes = 1024 * 1024
|
||||
const localitySourceBytes = 64 * 1024
|
||||
|
||||
// buildLocalityFixture writes the adversarial source layout described
|
||||
// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the
|
||||
// in TestRestoreLocalityAndReadAt: 15 sources of 64 KiB each (which the
|
||||
// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
|
||||
// (3 per blob group).
|
||||
func buildLocalityFixture(
|
||||
|
||||
@@ -27,10 +27,10 @@ import (
|
||||
// TestRestoreSweeperEvictsBlobs exercises the reference-counted blob
|
||||
// disk cache eviction during restore.
|
||||
//
|
||||
// The scenario: 30 unique 1 MB random files plus 10 duplicates of those
|
||||
// (40 files total, 30 MB of unique content) get backed up with a 10 MB
|
||||
// blob_size_limit. After backup the snapshot's encrypted blobs are
|
||||
// restored through Vaultik.Restore, and per-key Get counts on the
|
||||
// The scenario: 30 unique 128 KiB random files plus 10 duplicates of
|
||||
// those (40 files total, 3.75 MiB of unique content) get backed up with
|
||||
// a 1 MiB blob_size_limit. After backup the snapshot's encrypted blobs
|
||||
// are restored through Vaultik.Restore, and per-key Get counts on the
|
||||
// storage layer are recorded. Each blob in the snapshot MUST be
|
||||
// downloaded exactly once — re-downloads would mean the sweeper either
|
||||
// evicted a blob that was still needed (LRU regression) or that the
|
||||
@@ -41,7 +41,7 @@ import (
|
||||
// original AND every duplicate referencing its chunks have been
|
||||
// restored.
|
||||
// buildSweeperFixture writes the source layout for the sweeper test: 30
|
||||
// unique 1 MB random files plus 10 duplicates of a random subset. The
|
||||
// unique 128 KiB random files plus 10 duplicates of a random subset. The
|
||||
// PRNG seed is fixed so failures are reproducible; the entropy is what
|
||||
// matters here — the FastCDC chunker needs realistic-looking data to
|
||||
// pick chunk boundaries naturally. Returns the expected content by
|
||||
@@ -51,7 +51,7 @@ func buildSweeperFixture(
|
||||
) map[string][]byte {
|
||||
t.Helper()
|
||||
|
||||
const fileSize = 1 * 1024 * 1024
|
||||
const fileSize = 128 * 1024
|
||||
|
||||
rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only
|
||||
|
||||
@@ -159,7 +159,7 @@ func runSweeperBackup(
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned)
|
||||
require.Greater(t, scanResult.BlobsCreated, 1,
|
||||
"30 MB of unique data at 10 MB blob size should yield multiple blobs")
|
||||
"3.75 MiB of unique data at 1 MiB blob size should yield multiple blobs")
|
||||
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
|
||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
|
||||
@@ -199,8 +199,8 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) {
|
||||
|
||||
expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles)
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
maxBlobSize := int64(10 * 1024 * 1024)
|
||||
chunkSize := int64(16 * 1024)
|
||||
maxBlobSize := int64(1024 * 1024)
|
||||
|
||||
storer, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -187,6 +187,17 @@ func (e *listEnv) addRemoteRawTimestamp(
|
||||
t.Helper()
|
||||
|
||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
||||
e.putRemoteManifest(t, remoteKey,
|
||||
encodeRemoteManifest(t, remoteKey, timestamp))
|
||||
|
||||
return remoteKey
|
||||
}
|
||||
|
||||
// encodeRemoteManifest returns the encoded manifest of a one-blob
|
||||
// snapshot whose remote key is remoteKey.
|
||||
func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
|
||||
t.Helper()
|
||||
|
||||
manifest := &snapshot.Manifest{
|
||||
// Note: the hashed key, never the human ID. That is precisely
|
||||
// why a remote-only snapshot cannot be named.
|
||||
@@ -202,11 +213,17 @@ func (e *listEnv) addRemoteRawTimestamp(
|
||||
data, err := snapshot.EncodeManifest(manifest, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = e.store.Put(context.Background(),
|
||||
return data
|
||||
}
|
||||
|
||||
// putRemoteManifest stores an encoded manifest on the destination store
|
||||
// under remoteKey.
|
||||
func (e *listEnv) putRemoteManifest(t *testing.T, remoteKey string, data []byte) {
|
||||
t.Helper()
|
||||
|
||||
err := e.store.Put(context.Background(),
|
||||
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data))
|
||||
require.NoError(t, err)
|
||||
|
||||
return remoteKey
|
||||
}
|
||||
|
||||
// Fixtures shared across the listing tests.
|
||||
@@ -791,9 +808,17 @@ func TestListSnapshots_JSONReportsTruncation(t *testing.T) {
|
||||
|
||||
timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC)
|
||||
|
||||
// One past the cap, so exactly one snapshot is omitted.
|
||||
// One past the cap, so exactly one snapshot is omitted. The listing
|
||||
// names a snapshot by the key its manifest is stored under, never by
|
||||
// the manifest's own ID. So all of them share one encoded manifest:
|
||||
// encoding 1001 took seconds.
|
||||
data := encodeRemoteManifest(t,
|
||||
snapshot.RemoteSnapshotKey("otherhost_bulk"),
|
||||
timestamp.Format(time.RFC3339))
|
||||
|
||||
for i := range maxRemoteOnlyRowsForTest + 1 {
|
||||
env.addRemote(t, fmt.Sprintf("otherhost_bulk_%04d", i), timestamp)
|
||||
env.putRemoteManifest(t, snapshot.RemoteSnapshotKey(
|
||||
fmt.Sprintf("otherhost_bulk_%04d", i)), data)
|
||||
}
|
||||
|
||||
stderr := captureProcessStderr(t, func() {
|
||||
|
||||
Reference in new issue
Block a user