Compare commits

..
1 Commits
Author SHA1 Message Date
sneak c2ce69d258 Correct doc and help sentences that are false about the code (closes #233)
check / check (push) Waiting to run
A blob is written whole to a temporary file and uploaded once finished,
not streamed to storage. The README, ARCHITECTURE.md and
config.example.yml now say a backup needs free temporary space for each
blob (twice that for an rclone destination that cannot stream uploads)
and for the metadata export's copies of the local index, in $TMPDIR, or
partly in /var/tmp when TMPDIR is unset. Also corrected: the snapshot ID
format, what restore reads and how incomplete snapshots are removed in
docs/DATAMODEL.md, what source_path holds, the index_path default, the
config search order in the snapshot create help, what snapshot remove
cleans up in the prune help, how the release installs Go, and the
script/release and script/fmt-check comments.

Model: opus-5-5
2026-10-07 15:07:02 +00:00
12 changed files with 89 additions and 90 deletions
-10
View File
@@ -38,16 +38,6 @@ the tag exists and is exercised; what is left is merging `next` to
what `snapshot remove` cleans up, how the release gets its Go
toolchain, and the `script/release` and `script/fmt-check` comments.
- 2026-10-07: Cut the time the `internal/vaultik` and `internal/database`
tests take ([issue #235](https://git.eeqj.de/sneak/vaultik/issues/235)).
Most of the `internal/vaultik` time went to 24 tests that ran one at a
time only because they call `log.Initialize`; they now call it before
`t.Parallel()`, as the package's other tests do. `TestLargeDatasets`
committed each of its 1,500 inserts on its own and now makes them in
one transaction. `TestDedupOnlySnapshotRestores` gives its second
backup its own snapshot name instead of sleeping past the one-second
timestamp in the snapshot ID.
- 2026-10-07: Made two messages say only what is true
([issue #240](https://git.eeqj.de/sneak/vaultik/issues/240)). A config
file that others can read was warned about as containing S3
@@ -3,7 +3,6 @@ package database
import (
"context"
"database/sql"
"fmt"
"strings"
"testing"
@@ -368,7 +367,7 @@ func verifyBlobNullUploadTS(
}
// createLargeDatasetFiles creates fileCount files and adds every other
// one to the snapshot, in one transaction as a backup writes them.
// one to the snapshot.
func createLargeDatasetFiles(
t *testing.T,
repos *Repositories,
@@ -377,10 +376,9 @@ func createLargeDatasetFiles(
) {
t.Helper()
ctx := context.Background()
start := time.Now()
err := repos.WithTx(context.Background(),
func(ctx context.Context, tx *sql.Tx) error {
for i := range fileCount {
file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
@@ -391,25 +389,19 @@ func createLargeDatasetFiles(
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, tx, file)
err := repos.Files.Create(ctx, nil, file)
if err != nil {
return fmt.Errorf("creating file %d: %w", i, err)
t.Fatalf("failed to create file %d: %v", i, err)
}
// Add half to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, tx, snapshotID, file.ID)
if err != nil {
return err
}
}
}
return nil
})
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
if err != nil {
t.Fatal(err)
}
}
}
t.Logf("Created %d files in %v", fileCount, time.Since(start))
}
@@ -130,9 +130,10 @@ func assertThirdSnapshotRestores(
// up, and that snapshot is removed. The first snapshot keeps the file row,
// which now lists the appended content's chunks, while removal drops the
// blob that held them.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -177,9 +178,10 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
// The next run's prune drops that incomplete snapshot and its blob, while
// the first snapshot keeps the file row, which now lists the appended
// content's chunks.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
+22 -15
View File
@@ -38,9 +38,11 @@ import (
// (https://git.eeqj.de/sneak/vaultik/issues/130) and is not re-tested
// here; these tests target the layers above the backend.
//
// log.Initialize replaces the package-global logger that a running
// backup or restore reads, so each test calls it before t.Parallel,
// while no parallel test is running yet.
// The tests run serially, not with t.Parallel: each calls
// log.Initialize, which replaces the package-global logger, and a
// backup or restore running concurrently reads that same logger. Under
// -race the two collide. Running one at a time is the same choice
// prune_count_test.go already makes for the same reason.
const (
faultChunkSize = int64(64 * 1024)
@@ -163,19 +165,17 @@ func newReaderVaultik(
// Scenario 3: a stored blob's bytes are flipped before restore reads
// them. Restore must fail loudly, and no file must be left on the
// restore target holding corrupt content.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreRejectsCorruptBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
assertRestoreRejectsDamagedBlob(t, faultstore.GetCorrupt, "corrupt")
}
// Scenario 4: a stored blob is truncated before restore reads it. Same
// contract as the corrupt case.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreRejectsTruncatedBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
assertRestoreRejectsDamagedBlob(t, faultstore.GetTruncate, "truncated")
}
@@ -188,6 +188,7 @@ func assertRestoreRejectsDamagedBlob(
t *testing.T, fault faultstore.GetFault, name string,
) {
t.Helper()
log.Initialize(log.Config{})
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -231,9 +232,10 @@ func assertRestoreRejectsDamagedBlob(
// Scenario 6: the backend accepts blob uploads and reports success but
// stores nothing. verify --deep must catch it.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -283,9 +285,10 @@ func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
// Scenario 1a: a blob upload fails partway through. The interrupted run
// must not record the blob as uploaded, must not reference it from the
// snapshot, and must leave no blob object at the destination.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -358,9 +361,10 @@ func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
// chunks in a blob that was actually uploaded, so the retry re-chunks and
// re-uploads the affected data instead of silently referencing data that
// never reached storage.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -422,9 +426,10 @@ func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
// covered by TestBackupCompletesOnlyAfterMetadataExport
// (https://git.eeqj.de/sneak/vaultik/issues/177); this test exercises the
// lower-level export path in isolation.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -504,9 +509,10 @@ func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
// destination. Rerunning the backup must then prune the incomplete
// snapshot, produce a snapshot whose destination metadata and local index
// agree, and restore. See https://git.eeqj.de/sneak/vaultik/issues/177.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupCompletesOnlyAfterMetadataExport(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
@@ -679,9 +685,10 @@ func faultScannerFactory(
// Scenario 5: the restore target runs out of space mid-file. Restore
// must fail with an out-of-space error, and must not leave a truncated
// file at the target path presenting as a complete restore.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreReportsDiskFull(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
osFS := afero.NewOsFs()
tempDir := t.TempDir()
+11 -10
View File
@@ -928,17 +928,17 @@ func setupDedupBackupEnv(
}
}
// runDedupSnapshot creates a snapshot with the given name, scans dataDir
// into it, completes it, and exports its metadata, returning the snapshot
// ID and scan result.
// runDedupSnapshot creates a "dedup" snapshot, scans dataDir into it,
// completes it, and exports its metadata, returning the snapshot ID and
// scan result.
func runDedupSnapshot(
ctx context.Context, t *testing.T,
sm *snapshot.SnapshotManager, scanner *snapshot.Scanner,
hostname, name, dataDir, dbPath string,
hostname, dataDir, dbPath string,
) (string, *snapshot.ScanResult) {
t.Helper()
id, err := sm.CreateSnapshotWithName(ctx, hostname, name, "v", "g")
id, err := sm.CreateSnapshotWithName(ctx, hostname, "dedup", "v", "g")
require.NoError(t, err)
result, err := scanner.Scan(ctx, dataDir, id)
@@ -980,15 +980,16 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
// First snapshot — uploads all blobs.
_, r1 := runDedupSnapshot(ctx, t, sm, makeScanner(),
cfg.Hostname, "first", dataDir, dbPath)
cfg.Hostname, dataDir, dbPath)
require.Positive(t, r1.BlobsCreated,
"first snapshot should upload at least one blob")
// Second snapshot — same data, every chunk dedups. Its own name gives
// it a different snapshot ID without waiting for the one-second
// timestamp in the ID to tick over.
// Second snapshot — same data, every chunk dedups. Sleep past the
// second-precision timestamp so the snapshot IDs differ.
time.Sleep(1100 * time.Millisecond)
id2, r2 := runDedupSnapshot(ctx, t, sm, makeScanner(),
cfg.Hostname, "second", dataDir, dbPath)
cfg.Hostname, dataDir, dbPath)
require.Equal(t, 0, r2.BlobsCreated,
"second snapshot should upload zero new blobs (fully dedup'd)")
+10 -5
View File
@@ -78,9 +78,10 @@ func backUpThenUnplug(
// TestFirstBackupCreatesDestinationDirectory checks that a first backup
// to a destination directory that does not exist yet creates it, and
// that the destination can be listed afterwards.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestFirstBackupCreatesDestinationDirectory(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
storeDir := filepath.Join(t.TempDir(), "volume", "backup")
@@ -95,9 +96,10 @@ func TestFirstBackupCreatesDestinationDirectory(t *testing.T) {
// TestListSnapshotsWarnsWhenDestinationMissing checks that snapshot list
// warns and shows the local index alone, without reporting the local
// snapshot as missing from the destination.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestListSnapshotsWarnsWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, repos, out := backUpThenUnplug(ctx, t)
@@ -114,9 +116,10 @@ func TestListSnapshotsWarnsWhenDestinationMissing(t *testing.T) {
// TestRemoveSnapshotWarnsWhenDestinationMissing checks that snapshot
// remove warns that the metadata could not be removed from the
// destination, instead of reporting that it was.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRemoveSnapshotWarnsWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, repos, out := backUpThenUnplug(ctx, t)
@@ -135,9 +138,10 @@ func TestRemoveSnapshotWarnsWhenDestinationMissing(t *testing.T) {
// TestPruneKeepsLocalRecordsWhenDestinationMissing checks that prune
// fails on a destination it cannot list and deletes no local snapshot
// record.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestPruneKeepsLocalRecordsWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, repos, _ := backUpThenUnplug(ctx, t)
@@ -154,9 +158,10 @@ func TestPruneKeepsLocalRecordsWhenDestinationMissing(t *testing.T) {
// TestPurgeSaysListingFailedOnceWhenDestinationMissing checks that
// snapshot purge fails on a destination it cannot list, with an error
// that says "listing remote snapshots" once.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestPurgeSaysListingFailedOnceWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, _, _ := backUpThenUnplug(ctx, t)
+2 -1
View File
@@ -14,9 +14,10 @@ import (
// the discarded-error bug: getTableCount for a table its query cannot
// resolve must not silently become 0. A count that could not be read is
// reported as unknown, which a reader can tell apart from an empty table.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestTableCountForReportSurfacesReadFailure(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
+2 -1
View File
@@ -164,9 +164,10 @@ func scratchEntries(t *testing.T, dir string) []string {
// restore while a blob download is in progress. The download fails only
// because of the cancel, so Restore must return context.Canceled without
// reporting the file that needs the blob as failed.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
+5 -6
View File
@@ -45,10 +45,9 @@ type missingBlobBackup struct {
// after one blob of a two-blob snapshot was deleted. Every file stored in
// that blob must be reported as failed and left absent, every other file
// must be restored intact, and Restore must still return an error.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
backup := backupThenDeleteOneBlob(ctx, t)
@@ -86,10 +85,9 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
// TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
// still ends the restore with an error when SkipErrors is not set.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
backup := backupThenDeleteOneBlob(ctx, t)
@@ -109,6 +107,7 @@ func backupThenDeleteOneBlob(
ctx context.Context, t *testing.T,
) *missingBlobBackup {
t.Helper()
log.Initialize(log.Config{})
fs := afero.NewOsFs()
tempDir := t.TempDir()
+2 -1
View File
@@ -18,9 +18,10 @@ import (
// A file rewritten with its size unchanged and a new mtime in the same
// second as the mtime the index holds must still be backed up. See
// https://git.eeqj.de/sneak/vaultik/issues/226.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
+8 -9
View File
@@ -54,6 +54,8 @@ type summaryEnv struct {
func newSummaryEnv(t *testing.T) *summaryEnv {
t.Helper()
log.Initialize(log.Config{})
fs := afero.NewOsFs()
tempDir := t.TempDir()
srcDir := filepath.Join(tempDir, "src")
@@ -191,10 +193,9 @@ func (e *summaryEnv) dataLine(total, backedUp int64) string {
// A first backup stores copy.bin's chunks while backing up a.bin, so
// copy.bin's chunks are deduplicated within the run. Each file and byte
// is still counted once.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestSnapshotSummaryFirstRun(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newSummaryEnv(t)
summary := env.backUp(t, "first", false)
@@ -218,10 +219,9 @@ func TestSnapshotSummaryFirstRun(t *testing.T) {
// An incremental backup where a.bin's mtime changed but its content did
// not: a.bin is backed up again and every one of its chunks is already
// stored.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestSnapshotSummaryIncrementalRunWithDeduplicatedChunks(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newSummaryEnv(t)
env.backUp(t, "first", false)
@@ -251,10 +251,9 @@ func TestSnapshotSummaryIncrementalRunWithDeduplicatedChunks(t *testing.T) {
// Under --cron the progress reporter is off; the upload figures must
// still reach the summary and the snapshots row. The snapshot has two
// paths, each backed up by its own scan.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestSnapshotSummaryCronRunRecordsUploads(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newSummaryEnv(t)
summary := env.backUp(t, "split", true)
+2 -1
View File
@@ -18,9 +18,10 @@ import (
// A backup without --cron runs the progress reporter while one scanner
// scans each path of the snapshot in turn. See
// https://git.eeqj.de/sneak/vaultik/issues/253.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupWithoutCronOfTwoPathSnapshotRestoresBothPaths(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const snapshotName = "data"