3 Commits
Author SHA1 Message Date
sneak 80289ea14d Report an unreadable blob once, without its bytes (closes #424)
check / check (push) Successful in 9m10s
blobgen.NewReader returns ErrNotAgeStream when age cannot open the
data, in place of age's error, which quotes the data's first bytes. A
failed read of the header is still returned as the read error, and a
wrong key as age.NoIdentityMatchError.

Under --skip-errors, restore reports a blob that could not be
downloaded once, with the number of files that need it. Those files
are counted in the final warning, not listed; each is logged at debug
level.

Model: opus-5-5
2026-11-11 03:09:49 +00:00
clawbot 0d0c76fa24 Exit 1 when a deletion prompt is not confirmed (closes #423)
check / check (push) Failing after 21m27s
prune, snapshot purge, snapshot remove and database delete exited 0 when their confirmation prompt was declined or left unanswered, so a script could not tell that nothing was deleted. They now end with "deletion not confirmed" and exit 1. The [y/N] prompts accept y or Y; database delete still asks for yes.

Model: opus-5-5
2026-11-11 04:09:45 +01:00
clawbot 88a2aa7953 Shorten internal/vaultik tests under the race detector (closes #416)
check / check (push) Successful in 9m11s
The internal/vaultik tests took 36 to 56 seconds under go test -race and ran past the 90-second timeout when the host was busy. Tests that built the same backup now copy one shared backup, and the interruption tests use smaller sources. No assertion is weakened and no test is removed. Measured back to back on next under the same load, the package takes 55% of its former time.

Model: opus-5-5
2026-11-11 03:25:41 +01:00
15 changed files with 237 additions and 62 deletions

No files matched your search

+4 -3
View File
@@ -122,14 +122,15 @@ func TestNewReaderTrailingAndGarbage(t *testing.T) {
})
}
// TestNewReaderGarbageErrorQuotesNoBytes checks that random bytes in
// TestNewReaderGarbageErrorQuotesNoBytes checks that lines of text in
// place of a blob fail with ErrNotAgeStream, and that the error carries
// none of those bytes.
// none of their bytes. The lines end in a newline because age's own error
// quotes the data's first line only when it reads a whole line.
func TestNewReaderGarbageErrorQuotesNoBytes(t *testing.T) {
t.Parallel()
id, _ := makeIdentity(t)
garbage := bytes.Repeat([]byte("garbage-bytes-"), 100)
garbage := bytes.Repeat([]byte("garbage-bytes-\n"), 100)
_, err := blobgen.NewReader(bytes.NewReader(garbage), id)
+4 -3
View File
@@ -7,6 +7,7 @@ import (
"github.com/spf13/cobra"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// NewDatabaseCommand creates the database command group
@@ -90,12 +91,12 @@ Use --force to skip the confirmation prompt.`,
var confirm string
_, err = fmt.Scanln(&confirm)
// A failure to read the answer means no confirmation.
_, err = fmt.Fscanln(cmd.InOrStdin(), &confirm)
if err != nil || confirm != "yes" {
_, _ = fmt.Fprintln(w, "Aborted.")
//nolint:nilerr // a failed/aborted confirmation is a clean abort
return nil
return vaultik.ErrNotConfirmed
}
}
+48
View File
@@ -0,0 +1,48 @@
package cli //nolint:testpackage // sets the unexported rootFlags directly
import (
"bytes"
"errors"
"io"
"os"
"strings"
"testing"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// TestDatabaseDeleteDeclinedIsError checks that answering anything but
// "yes" keeps the database and returns ErrNotConfirmed, so the command
// exits non-zero.
//
//nolint:paralleltest // mutates the process-global rootFlags
func TestDatabaseDeleteDeclinedIsError(t *testing.T) {
dir := t.TempDir()
dbPath, cfgPath := seedDatabaseDeleteConfig(t, dir)
setRootFlags(t, RootFlags{ConfigPath: cfgPath})
var out bytes.Buffer
cmd := newDatabaseDeleteCommand()
cmd.SetOut(&out)
cmd.SetErr(io.Discard)
cmd.SetIn(strings.NewReader("no\n"))
cmd.SetArgs(nil)
err := cmd.Execute()
if !errors.Is(err, vaultik.ErrNotConfirmed) {
t.Errorf("declined database delete returned %v, want %v",
err, vaultik.ErrNotConfirmed)
}
_, err = os.Stat(dbPath)
if err != nil {
t.Errorf("declined database delete removed the file: %v", err)
}
if !strings.Contains(out.String(), "Aborted.") {
t.Errorf("declined database delete printed %q, want Aborted.",
out.String())
}
}
+84 -12
View File
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/spf13/afero"
@@ -24,19 +25,28 @@ import (
// that is not plugged in, cannot be listed. It is not an empty store, so
// no command may conclude from it that the local snapshots are gone.
// newFileDestinationVaultik returns a Vaultik that backs up the
// snapshot named "first" to a file:// destination at storeDir.
// newFileDestinationVaultik writes the fault source tree and returns a
// Vaultik that backs it up to a file:// destination at storeDir.
func newFileDestinationVaultik(
ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories) {
t.Helper()
osFs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
dbPath := filepath.Join(tempDir, "index.sqlite")
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
writeFaultSourceTree(t, osFs, dataDir)
return openFileDestinationVaultik(ctx, t, storeDir, dataDir,
filepath.Join(tempDir, "index.sqlite"))
}
// openFileDestinationVaultik returns a Vaultik that backs up dataDir to a
// file:// destination at storeDir, with its local index at dbPath. The
// index is closed when the test ends.
func openFileDestinationVaultik(
ctx context.Context, t *testing.T, storeDir, dataDir, dbPath string,
) (*vaultik.Vaultik, *database.Repositories) {
t.Helper()
store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err)
@@ -48,22 +58,84 @@ func newFileDestinationVaultik(
repos := database.NewRepositories(db)
cfg := changedFileConfig(dataDir, dbPath)
return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos
return newBackupVaultik(ctx, cfg, store, repos, db, afero.NewOsFs()), repos
}
// firstBackup is one backup of the snapshot named "first" to a file://
// destination. dir holds its source tree in src, its store in store and
// its closed local index in index/index.sqlite. The first test that
// needs it makes it, and TestMain removes it. Every test that copies it
// backs up the same source tree, so no test may change that tree.
//
//nolint:gochecknoglobals // made once and shared by the tests that copy it
var firstBackup struct {
once sync.Once
dir string
ready bool
}
// TestMain removes firstBackup once every test has run.
func TestMain(m *testing.M) {
code := m.Run()
_ = os.RemoveAll(firstBackup.dir)
os.Exit(code)
}
// makeFirstBackup writes the fault source tree and backs it up as the
// snapshot named "first", all in a new directory that it records in
// firstBackup.
func makeFirstBackup(ctx context.Context, t *testing.T) {
t.Helper()
dir, err := os.MkdirTemp("", "vaultik-first-backup-")
require.NoError(t, err)
firstBackup.dir = dir
storeDir := filepath.Join(dir, "store")
dataDir := filepath.Join(dir, "src")
indexDir := filepath.Join(dir, "index")
require.NoError(t, os.Mkdir(storeDir, 0o750))
require.NoError(t, os.Mkdir(indexDir, 0o750))
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
v, _ := openFileDestinationVaultik(ctx, t, storeDir, dataDir,
filepath.Join(indexDir, "index.sqlite"))
require.NoError(t, backUp(v, "first"))
require.NoError(t, v.DB.Close())
firstBackup.ready = true
}
// backUpToFileDestination creates the directory storeDir, as a user must
// before the first backup, and backs up the snapshot named "first" to a
// file:// destination there. Everything the returned Vaultik prints
// after the backup goes to the returned buffer.
// before the first backup, and returns a Vaultik whose file://
// destination there holds a backup of the snapshot named "first". The
// store and the local index are copies of firstBackup's, which is faster
// than backing up in every test. The returned Vaultik backs up a source
// tree shared with every other caller, so no test may change that tree.
// Everything the returned Vaultik prints after the backup goes to the
// returned buffer.
func backUpToFileDestination(
ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
t.Helper()
require.NoError(t, os.MkdirAll(storeDir, 0o750))
firstBackup.once.Do(func() { makeFirstBackup(ctx, t) })
require.True(t, firstBackup.ready, "another test failed to make firstBackup")
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
require.NoError(t, backUp(v, "first"))
require.NoError(t, os.MkdirAll(storeDir, 0o750))
require.NoError(t, os.CopyFS(storeDir,
os.DirFS(filepath.Join(firstBackup.dir, "store"))))
indexDir := t.TempDir()
require.NoError(t, os.CopyFS(indexDir,
os.DirFS(filepath.Join(firstBackup.dir, "index"))))
v, repos := openFileDestinationVaultik(ctx, t, storeDir,
filepath.Join(firstBackup.dir, "src"),
filepath.Join(indexDir, "index.sqlite"))
out := &bytes.Buffer{}
v.Stdout = out
+1 -1
View File
@@ -267,7 +267,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize,
dirsWithoutManifest) {
return nil
return ErrNotConfirmed
}
v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
+2 -1
View File
@@ -176,7 +176,8 @@ func TestPruneDeclinedKeepsMetadataDirWithoutManifest(t *testing.T) {
env.v.Stdin = strings.NewReader("n\n")
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
require.ErrorIs(t, env.v.PruneBlobs(&vaultik.PruneOptions{}),
vaultik.ErrNotConfirmed)
assert.True(t, env.store.hasKey(leftDB),
"a declined prune must keep the directory")
+5 -3
View File
@@ -54,9 +54,10 @@ func TestPurgeSnapshots_QuietPromptListsSnapshots(t *testing.T) {
v.UI.SetQuiet(true)
// Stdin is empty, so the prompt reads no answer and cancels.
require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true,
}))
})
require.ErrorIs(t, err, vaultik.ErrNotConfirmed)
assert.Contains(t, out.String(), snapHomeT0)
assert.Contains(t, out.String(), "Delete 1 snapshot(s)? [y/N]")
@@ -77,7 +78,8 @@ func TestPruneBlobs_QuietPromptShowsBlobTotal(t *testing.T) {
addBlob(t, env.store.testStorer, testBlobHashA)
// Stdin is empty, so the prompt reads no answer and cancels.
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
require.ErrorIs(t, env.v.PruneBlobs(&vaultik.PruneOptions{}),
vaultik.ErrNotConfirmed)
assert.Contains(t, env.stdout.String(),
"Found 1 unreferenced blob(s) totaling")
+21
View File
@@ -337,6 +337,27 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) {
assert.Contains(t, out, "vaultik prune")
}
// TestRemoveSnapshot_DeclinedIsError checks that answering no to the
// prompt keeps the snapshot and returns ErrNotConfirmed, so the command
// exits non-zero.
func TestRemoveSnapshot_DeclinedIsError(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t, "snapshot-001")
tv.Stdin.WriteString("n\n")
_, err := tv.RemoveSnapshot("snapshot-001", &vaultik.RemoveOptions{})
require.ErrorIs(t, err, vaultik.ErrNotConfirmed)
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
assert.Contains(t, tv.Stdout.String(), "Cancelled")
}
func TestRemoveSnapshot_DryRun(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
+1 -1
View File
@@ -232,7 +232,7 @@ func (v *Vaultik) finishRestore(
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
if result.BlobsFailed > 0 {
v.UI.Detailf("%d of them need one of the %d blob(s) that could not be downloaded.",
v.UI.Detailf("%d of them need a blob that could not be read (%d blob(s) in all).",
len(result.FilesNeedingFailedBlob), result.BlobsFailed)
}
+7 -7
View File
@@ -46,11 +46,11 @@ import (
// slice out a few KB — currently the dominant cost in restore.
//
// The test deliberately constructs an adversarial scenario: three
// blobs A/B/C of ~6 MB each, nine files distributed across them, and
// path-ordered names that interleave the blobs (a1, b1, c1, a2, b2,
// c2, …) so naive path-order processing would touch every blob before
// finishing any of them.
// localitySource is one 1 MiB fixture file used by the locality test.
// blobs A/B/C of five 64 KiB chunks each, nine files distributed across
// them, and path-ordered names that interleave the blobs (a1, b1, c1,
// a2, b2, c2, …) so naive path-order processing would touch every blob
// before finishing any of them.
// localitySource is one 64 KiB fixture file used by the locality test.
type localitySource struct {
path string
data []byte
@@ -65,10 +65,10 @@ type localityCopy struct {
// localitySourceBytes is the size of each source file of the locality
// fixture.
const localitySourceBytes = 1024 * 1024
const localitySourceBytes = 64 * 1024
// buildLocalityFixture writes the adversarial source layout described
// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the
// in TestRestoreLocalityAndReadAt: 15 sources of 64 KiB each (which the
// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
// (3 per blob group).
func buildLocalityFixture(
+7 -6
View File
@@ -101,7 +101,7 @@ func assertLostFilesCountedNotListed(
t.Helper()
assert.Contains(t, out, fmt.Sprintf(
"%d of them need one of the 1 blob(s) that could not be downloaded",
"%d of them need a blob that could not be read (1 blob(s) in all)",
len(backup.lost)))
for path := range backup.lost {
@@ -337,13 +337,14 @@ func backupThenDeleteOneBlob(
return backup
}
// overwriteBlobPattern repeats through the bytes that
// backupThenOverwriteOneBlob writes over a blob.
// overwriteBlobPattern repeats, each time followed by a newline, through
// the bytes that backupThenOverwriteOneBlob writes over a blob. age's own
// error quotes the data's first line only when it reads a whole line.
const overwriteBlobPattern = "overwritten-blob-bytes-"
// backupThenOverwriteOneBlob backs up missingBlobFileCount single-chunk
// files, then overwrites the blob holding the first of them with as many
// bytes of overwriteBlobPattern. It also returns that blob's hash.
// bytes of overwriteBlobPattern lines. It also returns that blob's hash.
func backupThenOverwriteOneBlob(
ctx context.Context, t *testing.T,
) (*missingBlobBackup, string) {
@@ -357,8 +358,8 @@ func backupThenOverwriteOneBlob(
info, err := backup.storer.Stat(ctx, key)
require.NoError(t, err)
pattern := bytes.Repeat([]byte(overwriteBlobPattern),
int(info.Size)/len(overwriteBlobPattern)+1)
line := overwriteBlobPattern + "\n"
pattern := bytes.Repeat([]byte(line), int(info.Size)/len(line)+1)
require.NoError(t, backup.storer.Put(
ctx, key, bytes.NewReader(pattern[:info.Size])))
@@ -27,10 +27,10 @@ import (
// TestRestoreSweeperEvictsBlobs exercises the reference-counted blob
// disk cache eviction during restore.
//
// The scenario: 30 unique 1 MB random files plus 10 duplicates of those
// (40 files total, 30 MB of unique content) get backed up with a 10 MB
// blob_size_limit. After backup the snapshot's encrypted blobs are
// restored through Vaultik.Restore, and per-key Get counts on the
// The scenario: 30 unique 128 KiB random files plus 10 duplicates of
// those (40 files total, 3.75 MiB of unique content) get backed up with
// a 1 MiB blob_size_limit. After backup the snapshot's encrypted blobs
// are restored through Vaultik.Restore, and per-key Get counts on the
// storage layer are recorded. Each blob in the snapshot MUST be
// downloaded exactly once — re-downloads would mean the sweeper either
// evicted a blob that was still needed (LRU regression) or that the
@@ -41,7 +41,7 @@ import (
// original AND every duplicate referencing its chunks have been
// restored.
// buildSweeperFixture writes the source layout for the sweeper test: 30
// unique 1 MB random files plus 10 duplicates of a random subset. The
// unique 128 KiB random files plus 10 duplicates of a random subset. The
// PRNG seed is fixed so failures are reproducible; the entropy is what
// matters here — the FastCDC chunker needs realistic-looking data to
// pick chunk boundaries naturally. Returns the expected content by
@@ -51,7 +51,7 @@ func buildSweeperFixture(
) map[string][]byte {
t.Helper()
const fileSize = 1 * 1024 * 1024
const fileSize = 128 * 1024
rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only
@@ -159,7 +159,7 @@ func runSweeperBackup(
require.NoError(t, err)
require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned)
require.Greater(t, scanResult.BlobsCreated, 1,
"30 MB of unique data at 10 MB blob size should yield multiple blobs")
"3.75 MiB of unique data at 1 MiB blob size should yield multiple blobs")
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
@@ -199,8 +199,8 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) {
expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles)
chunkSize := int64(64 * 1024)
maxBlobSize := int64(10 * 1024 * 1024)
chunkSize := int64(16 * 1024)
maxBlobSize := int64(1024 * 1024)
storer, err := storage.NewFileStorer(storeDir)
require.NoError(t, err)
+4 -9
View File
@@ -881,17 +881,12 @@ func (v *Vaultik) confirmAndExecutePurge(
var confirm string
// A failure to read the answer means no confirmation.
_, err := v.scanStdin(&confirm)
if err != nil {
if err != nil || strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled")
return nil //nolint:nilerr // treat EOF or read error as "no"
}
if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled")
return nil
return ErrNotConfirmed
}
} else if !quiet {
v.UI.Infof("The following snapshots will be deleted:")
@@ -1490,7 +1485,7 @@ func (v *Vaultik) RemoveSnapshot(
}
if !v.confirmRemoveSnapshot(snapshotID, opts) {
return result, nil
return result, ErrNotConfirmed
}
// Checked before the local record is deleted: it may be the last
+33 -7
View File
@@ -187,6 +187,17 @@ func (e *listEnv) addRemoteRawTimestamp(
t.Helper()
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
e.putRemoteManifest(t, remoteKey,
encodeRemoteManifest(t, remoteKey, timestamp))
return remoteKey
}
// encodeRemoteManifest returns the encoded manifest of a one-blob
// snapshot whose remote key is remoteKey.
func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
t.Helper()
manifest := &snapshot.Manifest{
// Note: the hashed key, never the human ID. That is precisely
// why a remote-only snapshot cannot be named.
@@ -202,11 +213,17 @@ func (e *listEnv) addRemoteRawTimestamp(
data, err := snapshot.EncodeManifest(manifest, 3)
require.NoError(t, err)
err = e.store.Put(context.Background(),
return data
}
// putRemoteManifest stores an encoded manifest on the destination store
// under remoteKey.
func (e *listEnv) putRemoteManifest(t *testing.T, remoteKey string, data []byte) {
t.Helper()
err := e.store.Put(context.Background(),
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data))
require.NoError(t, err)
return remoteKey
}
// Fixtures shared across the listing tests.
@@ -345,8 +362,9 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
// Without --force and with nothing on stdin the purge lists what it
// would delete, reads no confirmation, and deletes nothing.
env.stdout.Reset()
require.NoError(t, env.v.PurgeSnapshotsWithOptions(
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
require.ErrorIs(t, env.v.PurgeSnapshotsWithOptions(
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}),
vaultik.ErrNotConfirmed)
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
wantSize, "snapshot purge")
}
@@ -791,9 +809,17 @@ func TestListSnapshots_JSONReportsTruncation(t *testing.T) {
timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC)
// One past the cap, so exactly one snapshot is omitted.
// One past the cap, so exactly one snapshot is omitted. The listing
// names a snapshot by the key its manifest is stored under, never by
// the manifest's own ID. So all of them share one encoded manifest:
// encoding 1001 took seconds.
data := encodeRemoteManifest(t,
snapshot.RemoteSnapshotKey("otherhost_bulk"),
timestamp.Format(time.RFC3339))
for i := range maxRemoteOnlyRowsForTest + 1 {
env.addRemote(t, fmt.Sprintf("otherhost_bulk_%04d", i), timestamp)
env.putRemoteManifest(t, snapshot.RemoteSnapshotKey(
fmt.Sprintf("otherhost_bulk_%04d", i)), data)
}
stderr := captureProcessStderr(t, func() {
+7
View File
@@ -5,6 +5,7 @@ package vaultik
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
@@ -184,6 +185,12 @@ func (v *Vaultik) printlnStdout(args ...any) {
_, _ = fmt.Fprintln(v.Stdout, args...)
}
// ErrNotConfirmed is returned when a confirmation prompt gets any answer
// but the one it asks for (`y` or `Y`, or `yes` for `database delete`), or
// stdin ends first. Nothing the prompt asked about is deleted, and the
// command exits non-zero.
var ErrNotConfirmed = errors.New("deletion not confirmed")
// scanStdin reads a line of input from stdin.
func (v *Vaultik) scanStdin(a ...any) (int, error) {
return fmt.Fscanln(v.Stdin, a...)