diff --git a/docs/TODO.md b/docs/TODO.md index fe2b554..eb7be7c 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -23,6 +23,12 @@ runs the release build in `.gitea/workflows/release.yml`. # Completed Steps +- 2026-11-10: Shortened the tests of `internal/vaultik` under the race detector, + which on a busy host ran past the 90-second test timeout + ([issue #416](https://git.eeqj.de/sneak/vaultik/issues/416)). Tests that need + one backup to a file:// destination now copy a shared one, and the largest + fixtures are smaller. + - 2026-11-10: `script/fmt` and `script/fmt-check` now format and check all Markdown with `prettier` ([issue #411](https://git.eeqj.de/sneak/vaultik/issues/411)). The architecture diff --git a/internal/vaultik/missing_destination_test.go b/internal/vaultik/missing_destination_test.go index 1ab5a5c..b825e52 100644 --- a/internal/vaultik/missing_destination_test.go +++ b/internal/vaultik/missing_destination_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "github.com/spf13/afero" @@ -24,19 +25,28 @@ import ( // that is not plugged in, cannot be listed. It is not an empty store, so // no command may conclude from it that the local snapshots are gone. -// newFileDestinationVaultik returns a Vaultik that backs up the -// snapshot named "first" to a file:// destination at storeDir. +// newFileDestinationVaultik writes the fault source tree and returns a +// Vaultik that backs it up to a file:// destination at storeDir. func newFileDestinationVaultik( ctx context.Context, t *testing.T, storeDir string, ) (*vaultik.Vaultik, *database.Repositories) { t.Helper() - osFs := afero.NewOsFs() tempDir := t.TempDir() dataDir := filepath.Join(tempDir, "src") - dbPath := filepath.Join(tempDir, "index.sqlite") + writeFaultSourceTree(t, afero.NewOsFs(), dataDir) - writeFaultSourceTree(t, osFs, dataDir) + return openFileDestinationVaultik(ctx, t, storeDir, dataDir, + filepath.Join(tempDir, "index.sqlite")) +} + +// openFileDestinationVaultik returns a Vaultik that backs up dataDir to a +// file:// destination at storeDir, with its local index at dbPath. The +// index is closed when the test ends. +func openFileDestinationVaultik( + ctx context.Context, t *testing.T, storeDir, dataDir, dbPath string, +) (*vaultik.Vaultik, *database.Repositories) { + t.Helper() store, err := storage.NewFileStorer(storeDir) require.NoError(t, err) @@ -48,22 +58,82 @@ func newFileDestinationVaultik( repos := database.NewRepositories(db) cfg := changedFileConfig(dataDir, dbPath) - return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos + return newBackupVaultik(ctx, cfg, store, repos, db, afero.NewOsFs()), repos +} + +// firstBackup is one backup of the snapshot named "first" to a file:// +// destination. dir holds its source tree in src, its store in store and +// its closed local index in index/index.sqlite. The first test that +// needs it makes it, and TestMain removes it. Every test that copies it +// backs up the same source tree, so no test may change that tree. +// +//nolint:gochecknoglobals // made once and shared by the tests that copy it +var firstBackup struct { + once sync.Once + dir string + ready bool +} + +// TestMain removes firstBackup once every test has run. +func TestMain(m *testing.M) { + code := m.Run() + + _ = os.RemoveAll(firstBackup.dir) + + os.Exit(code) +} + +// makeFirstBackup writes the fault source tree and backs it up as the +// snapshot named "first", all in a new directory that it records in +// firstBackup. +func makeFirstBackup(ctx context.Context, t *testing.T) { + t.Helper() + + dir, err := os.MkdirTemp("", "vaultik-first-backup-") + require.NoError(t, err) + + firstBackup.dir = dir + storeDir := filepath.Join(dir, "store") + dataDir := filepath.Join(dir, "src") + indexDir := filepath.Join(dir, "index") + + require.NoError(t, os.Mkdir(storeDir, 0o750)) + require.NoError(t, os.Mkdir(indexDir, 0o750)) + writeFaultSourceTree(t, afero.NewOsFs(), dataDir) + + v, _ := openFileDestinationVaultik(ctx, t, storeDir, dataDir, + filepath.Join(indexDir, "index.sqlite")) + require.NoError(t, backUp(v, "first")) + require.NoError(t, v.DB.Close()) + + firstBackup.ready = true } // backUpToFileDestination creates the directory storeDir, as a user must -// before the first backup, and backs up the snapshot named "first" to a -// file:// destination there. Everything the returned Vaultik prints +// before the first backup, and returns a Vaultik whose file:// +// destination there holds a backup of the snapshot named "first". The +// store and the local index are copies of firstBackup's, which is faster +// than backing up in every test. Everything the returned Vaultik prints // after the backup goes to the returned buffer. func backUpToFileDestination( ctx context.Context, t *testing.T, storeDir string, ) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) { t.Helper() - require.NoError(t, os.MkdirAll(storeDir, 0o750)) + firstBackup.once.Do(func() { makeFirstBackup(ctx, t) }) + require.True(t, firstBackup.ready, "another test failed to make firstBackup") - v, repos := newFileDestinationVaultik(ctx, t, storeDir) - require.NoError(t, backUp(v, "first")) + require.NoError(t, os.MkdirAll(storeDir, 0o750)) + require.NoError(t, os.CopyFS(storeDir, + os.DirFS(filepath.Join(firstBackup.dir, "store")))) + + indexDir := t.TempDir() + require.NoError(t, os.CopyFS(indexDir, + os.DirFS(filepath.Join(firstBackup.dir, "index")))) + + v, repos := openFileDestinationVaultik(ctx, t, storeDir, + filepath.Join(firstBackup.dir, "src"), + filepath.Join(indexDir, "index.sqlite")) out := &bytes.Buffer{} v.Stdout = out diff --git a/internal/vaultik/restore_locality_test.go b/internal/vaultik/restore_locality_test.go index 19e03e6..e57778e 100644 --- a/internal/vaultik/restore_locality_test.go +++ b/internal/vaultik/restore_locality_test.go @@ -46,11 +46,11 @@ import ( // slice out a few KB — currently the dominant cost in restore. // // The test deliberately constructs an adversarial scenario: three -// blobs A/B/C of ~6 MB each, nine files distributed across them, and -// path-ordered names that interleave the blobs (a1, b1, c1, a2, b2, -// c2, …) so naive path-order processing would touch every blob before -// finishing any of them. -// localitySource is one 1 MiB fixture file used by the locality test. +// blobs A/B/C of five 64 KiB chunks each, nine files distributed across +// them, and path-ordered names that interleave the blobs (a1, b1, c1, +// a2, b2, c2, …) so naive path-order processing would touch every blob +// before finishing any of them. +// localitySource is one 64 KiB fixture file used by the locality test. type localitySource struct { path string data []byte @@ -65,10 +65,10 @@ type localityCopy struct { // localitySourceBytes is the size of each source file of the locality // fixture. -const localitySourceBytes = 1024 * 1024 +const localitySourceBytes = 64 * 1024 // buildLocalityFixture writes the adversarial source layout described -// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the +// in TestRestoreLocalityAndReadAt: 15 sources of 64 KiB each (which the // backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies // (3 per blob group). func buildLocalityFixture( diff --git a/internal/vaultik/restore_sweeper_integration_test.go b/internal/vaultik/restore_sweeper_integration_test.go index 57b8563..d723bef 100644 --- a/internal/vaultik/restore_sweeper_integration_test.go +++ b/internal/vaultik/restore_sweeper_integration_test.go @@ -27,10 +27,10 @@ import ( // TestRestoreSweeperEvictsBlobs exercises the reference-counted blob // disk cache eviction during restore. // -// The scenario: 30 unique 1 MB random files plus 10 duplicates of those -// (40 files total, 30 MB of unique content) get backed up with a 10 MB -// blob_size_limit. After backup the snapshot's encrypted blobs are -// restored through Vaultik.Restore, and per-key Get counts on the +// The scenario: 30 unique 128 KiB random files plus 10 duplicates of +// those (40 files total, 3.75 MiB of unique content) get backed up with +// a 1 MiB blob_size_limit. After backup the snapshot's encrypted blobs +// are restored through Vaultik.Restore, and per-key Get counts on the // storage layer are recorded. Each blob in the snapshot MUST be // downloaded exactly once — re-downloads would mean the sweeper either // evicted a blob that was still needed (LRU regression) or that the @@ -41,7 +41,7 @@ import ( // original AND every duplicate referencing its chunks have been // restored. // buildSweeperFixture writes the source layout for the sweeper test: 30 -// unique 1 MB random files plus 10 duplicates of a random subset. The +// unique 128 KiB random files plus 10 duplicates of a random subset. The // PRNG seed is fixed so failures are reproducible; the entropy is what // matters here — the FastCDC chunker needs realistic-looking data to // pick chunk boundaries naturally. Returns the expected content by @@ -51,7 +51,7 @@ func buildSweeperFixture( ) map[string][]byte { t.Helper() - const fileSize = 1 * 1024 * 1024 + const fileSize = 128 * 1024 rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only @@ -159,7 +159,7 @@ func runSweeperBackup( require.NoError(t, err) require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned) require.Greater(t, scanResult.BlobsCreated, 1, - "30 MB of unique data at 10 MB blob size should yield multiple blobs") + "3.75 MiB of unique data at 1 MiB blob size should yield multiple blobs") require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) @@ -199,8 +199,8 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles) - chunkSize := int64(64 * 1024) - maxBlobSize := int64(10 * 1024 * 1024) + chunkSize := int64(16 * 1024) + maxBlobSize := int64(1024 * 1024) storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) diff --git a/internal/vaultik/snapshot_list_test.go b/internal/vaultik/snapshot_list_test.go index 811d545..fa3083e 100644 --- a/internal/vaultik/snapshot_list_test.go +++ b/internal/vaultik/snapshot_list_test.go @@ -187,6 +187,17 @@ func (e *listEnv) addRemoteRawTimestamp( t.Helper() remoteKey := snapshot.RemoteSnapshotKey(snapshotID) + e.putRemoteManifest(t, remoteKey, + encodeRemoteManifest(t, remoteKey, timestamp)) + + return remoteKey +} + +// encodeRemoteManifest returns the encoded manifest of a one-blob +// snapshot whose remote key is remoteKey. +func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte { + t.Helper() + manifest := &snapshot.Manifest{ // Note: the hashed key, never the human ID. That is precisely // why a remote-only snapshot cannot be named. @@ -202,11 +213,17 @@ func (e *listEnv) addRemoteRawTimestamp( data, err := snapshot.EncodeManifest(manifest, 3) require.NoError(t, err) - err = e.store.Put(context.Background(), + return data +} + +// putRemoteManifest stores an encoded manifest on the destination store +// under remoteKey. +func (e *listEnv) putRemoteManifest(t *testing.T, remoteKey string, data []byte) { + t.Helper() + + err := e.store.Put(context.Background(), "metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data)) require.NoError(t, err) - - return remoteKey } // Fixtures shared across the listing tests. @@ -791,9 +808,17 @@ func TestListSnapshots_JSONReportsTruncation(t *testing.T) { timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC) - // One past the cap, so exactly one snapshot is omitted. + // One past the cap, so exactly one snapshot is omitted. The listing + // names a snapshot by the key its manifest is stored under, never by + // the manifest's own ID. So all of them share one encoded manifest: + // encoding 1001 took seconds. + data := encodeRemoteManifest(t, + snapshot.RemoteSnapshotKey("otherhost_bulk"), + timestamp.Format(time.RFC3339)) + for i := range maxRemoteOnlyRowsForTest + 1 { - env.addRemote(t, fmt.Sprintf("otherhost_bulk_%04d", i), timestamp) + env.putRemoteManifest(t, snapshot.RemoteSnapshotKey( + fmt.Sprintf("otherhost_bulk_%04d", i)), data) } stderr := captureProcessStderr(t, func() {