Author SHA1 Message Date
clawbot 30bcb8f4db restore and verify report an unknown snapshot ID as not found (closes #426)
check / check (push) Successful in 12m59s
A snapshot ID that names no snapshot in the local index or on the
destination store made restore and verify fail on a missing object at
a hashed path, and verify printed a header for it first. Resolving a
snapshot ID now uses the look-up `snapshot remove` already does, so
all three commands give the same error, and verify prints its header
only after the snapshot is found.

Model: opus-5-5
2026-11-11 02:32:17 +00:00
clawbot 88a2aa7953 Shorten internal/vaultik tests under the race detector (closes #416)
check / check (push) Successful in 9m11s
The internal/vaultik tests took 36 to 56 seconds under go test -race and ran past the 90-second timeout when the host was busy. Tests that built the same backup now copy one shared backup, and the interruption tests use smaller sources. No assertion is weakened and no test is removed. Measured back to back on next under the same load, the package takes 55% of its former time.

Model: opus-5-5
2026-11-11 03:25:41 +01:00
clawbot 098ed80990 Keep only open work in docs/TODO.md (closes #440)
check / check (push) Failing after 22m25s
sneak ruled that TODO is not a history document. docs/TODO.md drops the Completed Steps section and every other record of past work, and keeps the open steps. The Workflow step that moved a finished Next Step into Completed Steps now deletes it.

Model: opus-5-5
2026-11-11 02:41:30 +01:00
13 changed files with 258 additions and 1363 deletions

No files matched your search

+1 -2
View File
@@ -831,8 +831,7 @@ Work planned after 1.0. Loosely ordered by priority.
Every command's user-facing output is governed by `internal/ui`, in one of two Every command's user-facing output is governed by `internal/ui`, in one of two
ways. Color is enabled when the stream written to is a TTY and the `NO_COLOR` ways. Color is enabled when the stream written to is a TTY and the `NO_COLOR`
environment variable is unset (https://no-color.org/). The same rule governs the environment variable is unset (https://no-color.org/).
colorized log lines `internal/log` writes to a terminal.
- **Status, progress, warnings, and errors** go through the `internal/ui` - **Status, progress, warnings, and errors** go through the `internal/ui`
message methods below: marker-prefixed, colored on a TTY, and — except message methods below: marker-prefixed, colored on a TTY, and — except
+3 -1205
View File
File diff suppressed because it is too large. Load diff
+11 -32
View File
@@ -5,7 +5,6 @@ import (
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"os"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -61,12 +60,6 @@ type TTYHandler struct {
mu *sync.Mutex mu *sync.Mutex
out io.Writer out io.Writer
// color reports whether records are written with ANSI color. It is
// false when NO_COLOR is set, matching internal/ui's shouldColor
// test; the caller already chose this handler for a terminal, so
// TTY-ness is not re-checked here.
color bool
// attrs are the attributes accumulated through WithAttrs, emitted // attrs are the attributes accumulated through WithAttrs, emitted
// ahead of each record's own attributes. Their keys already carry // ahead of each record's own attributes. Their keys already carry
// the group path that was open when they were added, so no // the group path that was open when they were added, so no
@@ -79,19 +72,16 @@ type TTYHandler struct {
groups []string groups []string
} }
// NewTTYHandler creates a new TTY handler. Color is enabled unless the // NewTTYHandler creates a new TTY handler with colored output.
// NO_COLOR environment variable is set to a non-empty value, the same
// test internal/ui applies to status lines.
func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler { func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
if opts == nil { if opts == nil {
opts = &slog.HandlerOptions{} opts = &slog.HandlerOptions{}
} }
return &TTYHandler{ return &TTYHandler{
out: out, out: out,
opts: *opts, opts: *opts,
mu: &sync.Mutex{}, mu: &sync.Mutex{},
color: os.Getenv("NO_COLOR") == "",
} }
} }
@@ -134,10 +124,10 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
// are written around it: it can carry text from an untrusted source // are written around it: it can carry text from an untrusted source
// (a storage error, for one), and a raw control character would // (a storage error, for one), and a raw control character would
// otherwise reach the terminal. // otherwise reach the terminal.
_, _ = fmt.Fprintf(h.out, "%s %s %s", _, _ = fmt.Fprintf(h.out, "%s%s%s %s%s%s %s%s%s",
h.paint(colorGray, timestamp), colorGray, timestamp, colorReset,
h.paint(levelColor, level), levelColor, level, colorReset,
h.paint(colorBold, sanitize(r.Message))) colorBold, sanitize(r.Message), colorReset)
// Attributes carried by the handler come first, then the record's // Attributes carried by the handler come first, then the record's
// own. Handler attributes were qualified when they were added; the // own. Handler attributes were qualified when they were added; the
@@ -246,7 +236,6 @@ func (h *TTYHandler) clone() *TTYHandler {
opts: h.opts, opts: h.opts,
mu: h.mu, mu: h.mu,
out: h.out, out: h.out,
color: h.color,
attrs: make([]slog.Attr, len(h.attrs), len(h.attrs)+1), attrs: make([]slog.Attr, len(h.attrs), len(h.attrs)+1),
groups: make([]string, len(h.groups), len(h.groups)+1), groups: make([]string, len(h.groups), len(h.groups)+1),
} }
@@ -284,19 +273,9 @@ func (h *TTYHandler) writeAttr(a slog.Attr) {
// the snapshot database — so a control character in one of them must // the snapshot database — so a control character in one of them must
// be rendered as an escape sequence rather than reaching the terminal, // be rendered as an escape sequence rather than reaching the terminal,
// where it could move the cursor or inject its own colours. // where it could move the cursor or inject its own colours.
_, _ = fmt.Fprintf(h.out, " %s=%s", _, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
h.paint(colorCyan, sanitize(a.Key)), colorCyan, sanitize(a.Key), colorReset,
h.paint(colorBlue, sanitize(value))) colorBlue, sanitize(value), colorReset)
}
// paint wraps s in the given ANSI color when color is enabled, and
// returns s unchanged otherwise.
func (h *TTYHandler) paint(color, s string) string {
if !h.color {
return s
}
return color + s + colorReset
} }
// sanitize returns s unchanged when every rune in it is printable, and a // sanitize returns s unchanged when every rune in it is printable, and a
-41
View File
@@ -421,44 +421,3 @@ func TestTTYHandlerEmptyGroupAndAttrsAreNoOps(t *testing.T) {
assert.Contains(t, plain, "kept=yes") assert.Contains(t, plain, "kept=yes")
assert.NotContains(t, plain, " =") assert.NotContains(t, plain, " =")
} }
// TestTTYHandlerRespectsNoColor checks that only a non-empty NO_COLOR
// turns color off, matching internal/ui's shouldColor test. The handler
// is only ever chosen for a terminal, so this is the one remaining
// condition that can still leave color in log output.
func TestTTYHandlerRespectsNoColor(t *testing.T) {
cases := []struct {
name string
noColor string
colored bool
}{
{
name: "NO_COLOR turns color off",
noColor: "1",
},
{
name: "empty NO_COLOR keeps color on",
noColor: "",
colored: true,
},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
t.Setenv("NO_COLOR", testCase.noColor)
var buf bytes.Buffer
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
logger.Info("hello", "key", "value")
if testCase.colored {
assert.Contains(t, buf.String(), "\x1b[",
"an empty NO_COLOR must leave ANSI color in log output")
} else {
assert.NotContains(t, buf.String(), "\x1b[",
"NO_COLOR must suppress every ANSI escape in log output")
}
})
}
}
+84 -12
View File
@@ -7,6 +7,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"testing" "testing"
"github.com/spf13/afero" "github.com/spf13/afero"
@@ -24,19 +25,28 @@ import (
// that is not plugged in, cannot be listed. It is not an empty store, so // that is not plugged in, cannot be listed. It is not an empty store, so
// no command may conclude from it that the local snapshots are gone. // no command may conclude from it that the local snapshots are gone.
// newFileDestinationVaultik returns a Vaultik that backs up the // newFileDestinationVaultik writes the fault source tree and returns a
// snapshot named "first" to a file:// destination at storeDir. // Vaultik that backs it up to a file:// destination at storeDir.
func newFileDestinationVaultik( func newFileDestinationVaultik(
ctx context.Context, t *testing.T, storeDir string, ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories) { ) (*vaultik.Vaultik, *database.Repositories) {
t.Helper() t.Helper()
osFs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src") dataDir := filepath.Join(tempDir, "src")
dbPath := filepath.Join(tempDir, "index.sqlite") writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
writeFaultSourceTree(t, osFs, dataDir) return openFileDestinationVaultik(ctx, t, storeDir, dataDir,
filepath.Join(tempDir, "index.sqlite"))
}
// openFileDestinationVaultik returns a Vaultik that backs up dataDir to a
// file:// destination at storeDir, with its local index at dbPath. The
// index is closed when the test ends.
func openFileDestinationVaultik(
ctx context.Context, t *testing.T, storeDir, dataDir, dbPath string,
) (*vaultik.Vaultik, *database.Repositories) {
t.Helper()
store, err := storage.NewFileStorer(storeDir) store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err) require.NoError(t, err)
@@ -48,22 +58,84 @@ func newFileDestinationVaultik(
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
cfg := changedFileConfig(dataDir, dbPath) cfg := changedFileConfig(dataDir, dbPath)
return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos return newBackupVaultik(ctx, cfg, store, repos, db, afero.NewOsFs()), repos
}
// firstBackup is one backup of the snapshot named "first" to a file://
// destination. dir holds its source tree in src, its store in store and
// its closed local index in index/index.sqlite. The first test that
// needs it makes it, and TestMain removes it. Every test that copies it
// backs up the same source tree, so no test may change that tree.
//
//nolint:gochecknoglobals // made once and shared by the tests that copy it
var firstBackup struct {
once sync.Once
dir string
ready bool
}
// TestMain removes firstBackup once every test has run.
func TestMain(m *testing.M) {
code := m.Run()
_ = os.RemoveAll(firstBackup.dir)
os.Exit(code)
}
// makeFirstBackup writes the fault source tree and backs it up as the
// snapshot named "first", all in a new directory that it records in
// firstBackup.
func makeFirstBackup(ctx context.Context, t *testing.T) {
t.Helper()
dir, err := os.MkdirTemp("", "vaultik-first-backup-")
require.NoError(t, err)
firstBackup.dir = dir
storeDir := filepath.Join(dir, "store")
dataDir := filepath.Join(dir, "src")
indexDir := filepath.Join(dir, "index")
require.NoError(t, os.Mkdir(storeDir, 0o750))
require.NoError(t, os.Mkdir(indexDir, 0o750))
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
v, _ := openFileDestinationVaultik(ctx, t, storeDir, dataDir,
filepath.Join(indexDir, "index.sqlite"))
require.NoError(t, backUp(v, "first"))
require.NoError(t, v.DB.Close())
firstBackup.ready = true
} }
// backUpToFileDestination creates the directory storeDir, as a user must // backUpToFileDestination creates the directory storeDir, as a user must
// before the first backup, and backs up the snapshot named "first" to a // before the first backup, and returns a Vaultik whose file://
// file:// destination there. Everything the returned Vaultik prints // destination there holds a backup of the snapshot named "first". The
// after the backup goes to the returned buffer. // store and the local index are copies of firstBackup's, which is faster
// than backing up in every test. The returned Vaultik backs up a source
// tree shared with every other caller, so no test may change that tree.
// Everything the returned Vaultik prints after the backup goes to the
// returned buffer.
func backUpToFileDestination( func backUpToFileDestination(
ctx context.Context, t *testing.T, storeDir string, ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) { ) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
t.Helper() t.Helper()
require.NoError(t, os.MkdirAll(storeDir, 0o750)) firstBackup.once.Do(func() { makeFirstBackup(ctx, t) })
require.True(t, firstBackup.ready, "another test failed to make firstBackup")
v, repos := newFileDestinationVaultik(ctx, t, storeDir) require.NoError(t, os.MkdirAll(storeDir, 0o750))
require.NoError(t, backUp(v, "first")) require.NoError(t, os.CopyFS(storeDir,
os.DirFS(filepath.Join(firstBackup.dir, "store"))))
indexDir := t.TempDir()
require.NoError(t, os.CopyFS(indexDir,
os.DirFS(filepath.Join(firstBackup.dir, "index"))))
v, repos := openFileDestinationVaultik(ctx, t, storeDir,
filepath.Join(firstBackup.dir, "src"),
filepath.Join(indexDir, "index.sqlite"))
out := &bytes.Buffer{} out := &bytes.Buffer{}
v.Stdout = out v.Stdout = out
+10 -12
View File
@@ -128,10 +128,15 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
"paths", opts.Paths, "paths", opts.Paths,
) )
remoteKey, err := v.resolveSnapshotRemoteKey(opts.SnapshotID)
if err != nil {
return err
}
// Step 1: Download and decrypt the snapshot metadata database // Step 1: Download and decrypt the snapshot metadata database
log.Info("Downloading snapshot metadata...") log.Info("Downloading snapshot metadata...")
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identities) tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, remoteKey, identities)
if err != nil { if err != nil {
return fmt.Errorf("downloading snapshot database: %w", err) return fmt.Errorf("downloading snapshot database: %w", err)
} }
@@ -754,19 +759,12 @@ func (v *Vaultik) handleRestoreVerification(
return nil return nil
} }
// downloadSnapshotDB downloads and decrypts the snapshot metadata // downloadSnapshotDB downloads and decrypts the metadata database of the
// database. The identifier is resolved to the snapshot's remote key: a // snapshot at remoteKey, which resolveSnapshotRemoteKey returned for
// human ID is hashed, and a remote key (or its abbreviation, as printed // snapshotID.
// for a remote-only snapshot) is used as-is, so a host with no local
// index can restore the snapshots it can only see on the store.
func (v *Vaultik) downloadSnapshotDB( func (v *Vaultik) downloadSnapshotDB(
snapshotID string, identities []age.Identity, snapshotID, remoteKey string, identities []age.Identity,
) (*database.DB, string, error) { ) (*database.DB, string, error) {
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
if err != nil {
return nil, "", err
}
// Download encrypted database from storage // Download encrypted database from storage
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
+7 -7
View File
@@ -46,11 +46,11 @@ import (
// slice out a few KB — currently the dominant cost in restore. // slice out a few KB — currently the dominant cost in restore.
// //
// The test deliberately constructs an adversarial scenario: three // The test deliberately constructs an adversarial scenario: three
// blobs A/B/C of ~6 MB each, nine files distributed across them, and // blobs A/B/C of five 64 KiB chunks each, nine files distributed across
// path-ordered names that interleave the blobs (a1, b1, c1, a2, b2, // them, and path-ordered names that interleave the blobs (a1, b1, c1,
// c2, …) so naive path-order processing would touch every blob before // a2, b2, c2, …) so naive path-order processing would touch every blob
// finishing any of them. // before finishing any of them.
// localitySource is one 1 MiB fixture file used by the locality test. // localitySource is one 64 KiB fixture file used by the locality test.
type localitySource struct { type localitySource struct {
path string path string
data []byte data []byte
@@ -65,10 +65,10 @@ type localityCopy struct {
// localitySourceBytes is the size of each source file of the locality // localitySourceBytes is the size of each source file of the locality
// fixture. // fixture.
const localitySourceBytes = 1024 * 1024 const localitySourceBytes = 64 * 1024
// buildLocalityFixture writes the adversarial source layout described // buildLocalityFixture writes the adversarial source layout described
// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the // in TestRestoreLocalityAndReadAt: 15 sources of 64 KiB each (which the
// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies // backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
// (3 per blob group). // (3 per blob group).
func buildLocalityFixture( func buildLocalityFixture(
@@ -27,10 +27,10 @@ import (
// TestRestoreSweeperEvictsBlobs exercises the reference-counted blob // TestRestoreSweeperEvictsBlobs exercises the reference-counted blob
// disk cache eviction during restore. // disk cache eviction during restore.
// //
// The scenario: 30 unique 1 MB random files plus 10 duplicates of those // The scenario: 30 unique 128 KiB random files plus 10 duplicates of
// (40 files total, 30 MB of unique content) get backed up with a 10 MB // those (40 files total, 3.75 MiB of unique content) get backed up with
// blob_size_limit. After backup the snapshot's encrypted blobs are // a 1 MiB blob_size_limit. After backup the snapshot's encrypted blobs
// restored through Vaultik.Restore, and per-key Get counts on the // are restored through Vaultik.Restore, and per-key Get counts on the
// storage layer are recorded. Each blob in the snapshot MUST be // storage layer are recorded. Each blob in the snapshot MUST be
// downloaded exactly once — re-downloads would mean the sweeper either // downloaded exactly once — re-downloads would mean the sweeper either
// evicted a blob that was still needed (LRU regression) or that the // evicted a blob that was still needed (LRU regression) or that the
@@ -41,7 +41,7 @@ import (
// original AND every duplicate referencing its chunks have been // original AND every duplicate referencing its chunks have been
// restored. // restored.
// buildSweeperFixture writes the source layout for the sweeper test: 30 // buildSweeperFixture writes the source layout for the sweeper test: 30
// unique 1 MB random files plus 10 duplicates of a random subset. The // unique 128 KiB random files plus 10 duplicates of a random subset. The
// PRNG seed is fixed so failures are reproducible; the entropy is what // PRNG seed is fixed so failures are reproducible; the entropy is what
// matters here — the FastCDC chunker needs realistic-looking data to // matters here — the FastCDC chunker needs realistic-looking data to
// pick chunk boundaries naturally. Returns the expected content by // pick chunk boundaries naturally. Returns the expected content by
@@ -51,7 +51,7 @@ func buildSweeperFixture(
) map[string][]byte { ) map[string][]byte {
t.Helper() t.Helper()
const fileSize = 1 * 1024 * 1024 const fileSize = 128 * 1024
rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only
@@ -159,7 +159,7 @@ func runSweeperBackup(
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned) require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned)
require.Greater(t, scanResult.BlobsCreated, 1, require.Greater(t, scanResult.BlobsCreated, 1,
"30 MB of unique data at 10 MB blob size should yield multiple blobs") "3.75 MiB of unique data at 1 MiB blob size should yield multiple blobs")
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
@@ -199,8 +199,8 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) {
expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles) expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles)
chunkSize := int64(64 * 1024) chunkSize := int64(16 * 1024)
maxBlobSize := int64(10 * 1024 * 1024) maxBlobSize := int64(1024 * 1024)
storer, err := storage.NewFileStorer(storeDir) storer, err := storage.NewFileStorer(storeDir)
require.NoError(t, err) require.NoError(t, err)
+23 -23
View File
@@ -969,25 +969,22 @@ func (v *Vaultik) VerifySnapshotWithOptions(
Mode: "shallow", Mode: "shallow",
} }
v.printVerifyHeader(snapshotID, opts) // The key is kept so we can also check for the snapshot's encrypted
// database below.
// Resolve the identifier to the snapshot's remote key. A human ID is
// hashed; a remote key (or its abbreviation, as printed for a
// remote-only snapshot) is used as-is, so a host with no local index
// can verify a snapshot it can only see on the store. The key is kept
// so we can also check for the snapshot's encrypted database below.
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID) remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
if err != nil { if err != nil {
if opts.JSON { if opts.JSON {
result.Status = verifyStatusFailed result.Status = verifyStatusFailed
result.ErrorMessage = fmt.Sprintf("resolving snapshot identifier: %v", err) result.ErrorMessage = err.Error()
return v.outputVerifyJSON(result) return v.outputVerifyJSON(result)
} }
return fmt.Errorf("resolving snapshot identifier: %w", err) return err
} }
v.printVerifyHeader(snapshotID, opts)
manifest, err := v.downloadManifestByKey(remoteKey) manifest, err := v.downloadManifestByKey(remoteKey)
if err != nil { if err != nil {
if opts.JSON { if opts.JSON {
@@ -1480,7 +1477,7 @@ func (v *Vaultik) RemoveSnapshot(
return result, err return result, err
} }
inLocalIndex, err := v.findSnapshotToRemove(snapshotID, opts) inLocalIndex, err := v.checkSnapshotExists(snapshotID, opts.LocalOnly)
if err != nil { if err != nil {
return result, err return result, err
} }
@@ -1543,17 +1540,20 @@ func (v *Vaultik) RemoveSnapshot(
return result, nil return result, nil
} }
// findSnapshotToRemove reports whether the local index has a row for // checkSnapshotExists returns errSnapshotNotFound when the local index
// snapshotID, and returns errSnapshotNotFound when there is nothing to // has no row for snapshotID and, unless localOnly is set, the
// remove: no row and, unless LocalOnly is set, no metadata on the // destination store has no metadata for it either. inLocalIndex reports
// destination store either. The destination store is consulted only // whether the local index has that row. The destination store is
// when the local index has no row, so a snapshot in the local index can // consulted only when the local index has no row, so a snapshot in the
// still be removed from it while the destination store is unreachable, // local index can still be removed from it while the destination store
// and only after checkDestinationMarker passes, so an unplugged volume's // is unreachable, and only after checkDestinationMarker passes, so an
// empty mount point is not taken for a store without the snapshot. // unplugged volume's empty mount point is not taken for a store without
func (v *Vaultik) findSnapshotToRemove( // the snapshot.
snapshotID string, opts *RemoveOptions, //
) (bool, error) { //nolint:nonamedreturns // the result name says what the bool means
func (v *Vaultik) checkSnapshotExists(
snapshotID string, localOnly bool,
) (inLocalIndex bool, err error) {
if v.Repositories != nil { if v.Repositories != nil {
snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID) snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID)
if err != nil { if err != nil {
@@ -1565,12 +1565,12 @@ func (v *Vaultik) findSnapshotToRemove(
} }
} }
if opts.LocalOnly { if localOnly {
return false, fmt.Errorf("%w in the local index: %s", return false, fmt.Errorf("%w in the local index: %s",
errSnapshotNotFound, snapshotID) errSnapshotNotFound, snapshotID)
} }
err := v.checkDestinationMarker() err = v.checkDestinationMarker()
if err != nil { if err != nil {
return false, err return false, err
} }
+8 -2
View File
@@ -28,8 +28,9 @@ var (
// Two forms are accepted, matching the two things a host can know: // Two forms are accepted, matching the two things a host can know:
// //
// - A human snapshot ID (hostname_name_timestamp), which a host holding // - A human snapshot ID (hostname_name_timestamp), which a host holding
// the local index has. It is hashed to its remote key; the store is // the local index has. It is hashed to its remote key. An ID that
// not consulted. // names no snapshot in the local index or on the destination store
// is an error (see checkSnapshotExists).
// - A remote key, or the leading part of one, which is all a host with // - A remote key, or the leading part of one, which is all a host with
// no local index can know — it is exactly what `snapshot list` prints // no local index can know — it is exactly what `snapshot list` prints
// for a remote-only snapshot (see formatRemoteOnlyID). It is resolved // for a remote-only snapshot (see formatRemoteOnlyID). It is resolved
@@ -41,6 +42,11 @@ var (
// RFC3339 timestamp). // RFC3339 timestamp).
func (v *Vaultik) resolveSnapshotRemoteKey(identifier string) (string, error) { func (v *Vaultik) resolveSnapshotRemoteKey(identifier string) (string, error) {
if !isRemoteKeyOrPrefix(identifier) { if !isRemoteKeyOrPrefix(identifier) {
_, err := v.checkSnapshotExists(identifier, false)
if err != nil {
return "", err
}
return snapshot.RemoteSnapshotKey(identifier), nil return snapshot.RemoteSnapshotKey(identifier), nil
} }
+30 -5
View File
@@ -187,6 +187,17 @@ func (e *listEnv) addRemoteRawTimestamp(
t.Helper() t.Helper()
remoteKey := snapshot.RemoteSnapshotKey(snapshotID) remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
e.putRemoteManifest(t, remoteKey,
encodeRemoteManifest(t, remoteKey, timestamp))
return remoteKey
}
// encodeRemoteManifest returns the encoded manifest of a one-blob
// snapshot whose remote key is remoteKey.
func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
t.Helper()
manifest := &snapshot.Manifest{ manifest := &snapshot.Manifest{
// Note: the hashed key, never the human ID. That is precisely // Note: the hashed key, never the human ID. That is precisely
// why a remote-only snapshot cannot be named. // why a remote-only snapshot cannot be named.
@@ -202,11 +213,17 @@ func (e *listEnv) addRemoteRawTimestamp(
data, err := snapshot.EncodeManifest(manifest, 3) data, err := snapshot.EncodeManifest(manifest, 3)
require.NoError(t, err) require.NoError(t, err)
err = e.store.Put(context.Background(), return data
}
// putRemoteManifest stores an encoded manifest on the destination store
// under remoteKey.
func (e *listEnv) putRemoteManifest(t *testing.T, remoteKey string, data []byte) {
t.Helper()
err := e.store.Put(context.Background(),
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data)) "metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data))
require.NoError(t, err) require.NoError(t, err)
return remoteKey
} }
// Fixtures shared across the listing tests. // Fixtures shared across the listing tests.
@@ -791,9 +808,17 @@ func TestListSnapshots_JSONReportsTruncation(t *testing.T) {
timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC) timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC)
// One past the cap, so exactly one snapshot is omitted. // One past the cap, so exactly one snapshot is omitted. The listing
// names a snapshot by the key its manifest is stored under, never by
// the manifest's own ID. So all of them share one encoded manifest:
// encoding 1001 took seconds.
data := encodeRemoteManifest(t,
snapshot.RemoteSnapshotKey("otherhost_bulk"),
timestamp.Format(time.RFC3339))
for i := range maxRemoteOnlyRowsForTest + 1 { for i := range maxRemoteOnlyRowsForTest + 1 {
env.addRemote(t, fmt.Sprintf("otherhost_bulk_%04d", i), timestamp) env.putRemoteManifest(t, snapshot.RemoteSnapshotKey(
fmt.Sprintf("otherhost_bulk_%04d", i)), data)
} }
stderr := captureProcessStderr(t, func() { stderr := captureProcessStderr(t, func() {
+62
View File
@@ -0,0 +1,62 @@
package vaultik_test
import (
"bytes"
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// TestUnknownSnapshotID checks that restore and verify, given an ID that
// names no snapshot in the local index or on the destination store, say
// so in the words `snapshot remove` uses and print nothing else.
func TestUnknownSnapshotID(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const unknownID = "host1_home_2026-01-01T00:00:00Z"
commands := map[string]func(v *vaultik.Vaultik) error{
"restore": func(v *vaultik.Vaultik) error {
return v.Restore(&vaultik.RestoreOptions{
SnapshotID: unknownID,
TargetDir: t.TempDir(),
})
},
"verify": func(v *vaultik.Vaultik) error {
return v.VerifySnapshotWithOptions(unknownID, &vaultik.VerifyOptions{})
},
"verify --deep": func(v *vaultik.Vaultik) error {
return v.VerifySnapshotWithOptions(unknownID,
&vaultik.VerifyOptions{Deep: true})
},
}
for name, run := range commands {
t.Run(name, func(t *testing.T) {
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the marker.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
tv := vaultik.NewForTesting(store)
tv.Config = &config.Config{AgeSecretKey: testAgeSecretKey}
tv.Repositories = newLocalIndex(t, "snapshot-001")
err := run(tv.Vaultik)
require.EqualError(t, err, "snapshot not found in the local "+
"index or on the destination store: "+unknownID)
assert.Empty(t, tv.Stdout.String())
})
}
}
+10 -13
View File
@@ -120,12 +120,17 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
if err != nil {
return v.deepVerifyFailure(result, opts, err.Error(), err)
}
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON && !v.UI.Quiet() {
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
} }
manifest, tempDB, dbBlobs, err := v.loadVerificationData( manifest, tempDB, dbBlobs, err := v.loadVerificationData(
snapshotID, opts, result, identities) snapshotID, remoteKey, opts, result, identities)
if err != nil { if err != nil {
return err return err
} }
@@ -170,21 +175,13 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
return nil return nil
} }
// loadVerificationData downloads manifest, database, and blob list for verification // loadVerificationData downloads manifest, database, and blob list for
// verification of the snapshot at remoteKey, which
// resolveSnapshotRemoteKey returned for snapshotID.
func (v *Vaultik) loadVerificationData( func (v *Vaultik) loadVerificationData(
snapshotID string, opts *VerifyOptions, result *VerifyResult, snapshotID, remoteKey string, opts *VerifyOptions, result *VerifyResult,
identities []age.Identity, identities []age.Identity,
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { ) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
// Resolve the identifier to the snapshot's remote key. A human ID is
// hashed; a remote key (or its abbreviation, as printed for a
// remote-only snapshot) is used as-is, so a host with no local index
// can verify a snapshot it can only see on the store.
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
if err != nil {
return nil, nil, nil, v.deepVerifyFailure(result, opts,
fmt.Sprintf("resolving snapshot identifier: %v", err), err)
}
// Download manifest. downloadManifestByKey is the single reader for // Download manifest. downloadManifestByKey is the single reader for
// remote manifests; see its doc comment. // remote manifests; see its doc comment.
log.Info("Downloading manifest", "remote_key", remoteKey) log.Info("Downloading manifest", "remote_key", remoteKey)