diff --git a/README.md b/README.md index b411fd4..6e4b177 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,57 @@ VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot restore ` rather than by +a `hostname_name_timestamp` name, because the name lives only in the +local index and the encrypted database and cannot be recovered from the +store. Its timestamp and compressed size are real. (See the `snapshot +list` description under [command details](#command-details) for the full +explanation.) + +Use that remote key — the hex printed inside ``, or the +full `remote_key` from `snapshot list --json` — to restore and verify: + +```sh +# restore everything to /tmp/restored, then check every restored file's +# chunk hashes +VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' \ + vaultik snapshot restore --verify /tmp/restored + +# optionally, deep-verify the snapshot against the store (downloads and +# cryptographically checks every blob) +VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' \ + vaultik snapshot verify --deep +``` + +`age_recipients` (the public key) is not needed to restore — only the +private key in `VAULTIK_AGE_SECRET_KEY`. Both the abbreviated key printed +in the table and the full 64-character key from `--json` are accepted; a +leading part of the key is enough as long as it is unambiguous. + --- ## cli @@ -245,6 +296,8 @@ local index alone, and still exits zero. * Default (shallow): checks that all blobs referenced in the manifest exist in storage * `--deep`: Downloads and decrypts each blob, verifies chunk hashes against the encrypted metadata database +* Accepts the same identifiers as `snapshot restore`: a snapshot ID, or a + remote-only snapshot's remote key (or an unambiguous leading part of it) * `--json`: Output results as JSON **`snapshot purge`**: Remove old snapshots based on criteria. Retention is @@ -275,6 +328,10 @@ on the destination in one go, use `vaultik remote nuke --force`. **`snapshot restore`**: Restore files from a backup snapshot. * Requires `VAULTIK_AGE_SECRET_KEY` environment variable +* Accepts a snapshot ID, or — for a snapshot only on the destination + store — its remote key (or an unambiguous leading part of it) as shown + by `snapshot list`. See + [restoring on another machine](#restoring-on-another-machine). * Optional path arguments to restore specific files/directories (default: all) * Preserves file permissions, timestamps, ownership (ownership requires root), symlinks, and empty directories @@ -525,10 +582,6 @@ priority. ### infrastructure -* **Cross-machine restore documentation.** The "restore from - another host" workflow works but isn't documented as a - first-class operation in this README. Worth a dedicated section - once it's settled. * **Schema migrations.** Currently nonexistent — pre-1.0 schema changes are handled by `vaultik database delete` plus a full re-scan. Post-1.0 we'll need a migration story to keep existing diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index a702386..0bad591 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -221,8 +221,11 @@ func newSnapshotVerifyCommand() *cobra.Command { cmd := &cobra.Command{ Use: "verify ", Short: "Verify snapshot integrity", - Long: "Verifies that all blobs referenced in a snapshot exist", - Args: requireSnapshotIDArg, + Long: "Verifies that all blobs referenced in a snapshot exist.\n\n" + + "The snapshot may be named by its ID or, on a host with no local\n" + + "index, by the remote key that 'snapshot list' prints for a\n" + + "remote-only snapshot (an unambiguous leading part is enough).", + Args: requireSnapshotIDArg, RunE: func(cmd *cobra.Command, args []string) error { snapshotID := args[0] diff --git a/internal/cli/snapshot_restore.go b/internal/cli/snapshot_restore.go index cf28e8e..188ee90 100644 --- a/internal/cli/snapshot_restore.go +++ b/internal/cli/snapshot_restore.go @@ -48,6 +48,10 @@ target directory. If no paths are specified, all files are restored. If paths are specified, only matching files/directories are restored. +The snapshot may be named by its ID or, when restoring on a host with no +local index, by the remote key that 'snapshot list' prints for a +remote-only snapshot (an unambiguous leading part is enough). + Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age private key. diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index c26c19e..c46ff5f 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -18,7 +18,6 @@ import ( "sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" - "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/types" ) @@ -577,14 +576,20 @@ func (v *Vaultik) handleRestoreVerification( } // downloadSnapshotDB downloads and decrypts the snapshot metadata -// database. The snapshotID is the human ID; we hash it to the remote -// key for the storage path. +// database. The identifier is resolved to the snapshot's remote key: a +// human ID is hashed, and a remote key (or its abbreviation, as printed +// for a remote-only snapshot) is used as-is, so a host with no local +// index can restore the snapshots it can only see on the store. func (v *Vaultik) downloadSnapshotDB( snapshotID string, identity age.Identity, ) (*database.DB, error) { + remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID) + if err != nil { + return nil, err + } + // Download encrypted database from storage - dbKey := fmt.Sprintf("metadata/%s/db.zst.age", - snapshot.RemoteSnapshotKey(snapshotID)) + dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) reader, err := v.Storage.Get(v.ctx, dbKey) if err != nil { diff --git a/internal/vaultik/restore_another_machine_test.go b/internal/vaultik/restore_another_machine_test.go new file mode 100644 index 0000000..59268a4 --- /dev/null +++ b/internal/vaultik/restore_another_machine_test.go @@ -0,0 +1,167 @@ +package vaultik_test + +import ( + "bytes" + "context" + "io" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/config" + "sneak.berlin/go/vaultik/internal/database" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/snapshot" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/ui" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// TestRestoreOnAnotherMachine proves the disaster-recovery path: a host +// that has only the vaultik binary, the age secret key, and the storage +// credentials — no local index, a different hostname, and no +// age_recipients configured — can list, restore, and verify a snapshot +// straight from the destination store. +// +// The backup half writes a snapshot with one index and hostname. The +// restore half throws that index away entirely: a fresh, empty index and +// a config that shares nothing with the original but the storage location +// and the secret key. If restore or verify needed the original local +// index — or the human snapshot ID that only that index holds — this test +// could not run, because the recovery host can know neither. +func TestRestoreOnAnotherMachine(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + chunkSize := int64(64 * 1024) + maxBlobSize := int64(512 * 1024) + + sourceFiles := writeRecoverySourceTree(t, fs, dataDir, chunkSize) + + ctx := context.Background() + + // Backup host: one index, hostname test-host, age_recipients set. + // runFileStorageBackup closes the index before returning, so nothing + // below can lean on it. + _, storer, originalID := runFileStorageBackup( + ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) + + // Recovery host: a fresh empty index, a different hostname, and no + // age_recipients — only the secret key and the same storage location. + recovery, stdout := newRecoveryHost(ctx, t, fs, storer) + + // The recovery index really is empty. This is the assertion that makes + // the test a guard against restore quietly depending on the original + // index: if it did, an empty index would make restore fail. + localSnaps, err := recovery.Repositories.Snapshots.ListRecent(ctx, 100) + require.NoError(t, err) + require.Empty(t, localSnaps, "recovery host must start with no local index") + + // List: the snapshot shows up as remote-only, identified by its remote + // key, with no recoverable human ID. + require.NoError(t, recovery.ListSnapshots(true)) + + rows := decodeListJSON(t, stdout.String()) + require.Len(t, rows, 1) + + remote := rows[0] + assert.False(t, remote.LocallyTracked, "snapshot must be remote-only here") + assert.Empty(t, remote.ID, "the human ID is unknown to the recovery host") + require.Len(t, remote.RemoteKey, 64) + assert.Equal(t, snapshot.RemoteSnapshotKey(originalID), remote.RemoteKey, + "the listed key is the hashed snapshot ID") + + // Restore driven by the abbreviated identifier the table prints (the + // first 12 hex of the remote key), then deep-verify from the store + // keyed by the full remote key. Both are what a recovery host can know. + require.NoError(t, recovery.Restore(&vaultik.RestoreOptions{ + SnapshotID: remote.RemoteKey[:12], + TargetDir: restoreDir, + Verify: true, + })) + require.NoError(t, recovery.RunDeepVerify( + remote.RemoteKey, &vaultik.VerifyOptions{Deep: true})) + + assertRestoredTreeMatches(t, fs, restoreDir, sourceFiles) +} + +// writeRecoverySourceTree writes a small source tree spanning several +// chunks (so restore reassembles real multi-chunk files) and returns the +// content keyed by absolute path. +func writeRecoverySourceTree( + t *testing.T, fs afero.Fs, dataDir string, chunkSize int64, +) map[string][]byte { + t.Helper() + + sourceFiles := map[string][]byte{ + filepath.Join(dataDir, "notes.txt"): []byte("recover me"), + filepath.Join(dataDir, "sub", "big.bin"): bytesPattern("big-", int(chunkSize*3)), + filepath.Join(dataDir, "sub", "small.bin"): bytesPattern("small-", 128), + } + + for path, content := range sourceFiles { + require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, afero.WriteFile(fs, path, content, 0o644)) + } + + return sourceFiles +} + +// newRecoveryHost builds the Vaultik a replacement machine would run: an +// empty in-memory index, a hostname different from the backup host, no +// age_recipients, and only the secret key plus the shared storer. It +// returns the instance and the buffer its stdout is wired to. +func newRecoveryHost( + ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer, +) (*vaultik.Vaultik, *bytes.Buffer) { + t.Helper() + + recoveryDB, err := database.New(ctx, ":memory:") + require.NoError(t, err) + t.Cleanup(func() { _ = recoveryDB.Close() }) + + stdout := &bytes.Buffer{} + + recovery := &vaultik.Vaultik{ + Config: &config.Config{ + AgeSecretKey: testAgeSecretKey, + Hostname: "recovery-host", + }, + Storage: storer, + Fs: fs, + Repositories: database.NewRepositories(recoveryDB), + DB: recoveryDB, + Stdout: stdout, + Stderr: io.Discard, + UI: ui.NewWithColor(io.Discard, false), + } + recovery.SetContext(ctx) + + return recovery, stdout +} + +// assertRestoredTreeMatches byte-compares every restored file against its +// source content. +func assertRestoredTreeMatches( + t *testing.T, fs afero.Fs, restoreDir string, sourceFiles map[string][]byte, +) { + t.Helper() + + for origPath, expected := range sourceFiles { + restored := filepath.Join(restoreDir, origPath) + got, err := afero.ReadFile(fs, restored) + require.NoErrorf(t, err, "restored file missing: %s", restored) + require.Truef(t, bytes.Equal(got, expected), + "byte mismatch for %s", origPath) + } +} diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 8c12ec7..406d9ad 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -670,9 +670,11 @@ func (v *Vaultik) VerifySnapshotWithOptions( v.printVerifyHeader(snapshotID, opts) - // Download and parse manifest. The caller supplies a human - // snapshot ID; we hash it to address remote storage. - manifest, err := v.downloadManifestByKey(snapshot.RemoteSnapshotKey(snapshotID)) + // Resolve the identifier to the snapshot's remote key and download the + // manifest. A human ID is hashed; a remote key (or its abbreviation, + // as printed for a remote-only snapshot) is used as-is, so a host with + // no local index can verify a snapshot it can only see on the store. + manifest, err := v.resolveAndDownloadManifest(snapshotID) if err != nil { if opts.JSON { result.Status = verifyStatusFailed diff --git a/internal/vaultik/snapshot_identifier.go b/internal/vaultik/snapshot_identifier.go new file mode 100644 index 0000000..5f3903d --- /dev/null +++ b/internal/vaultik/snapshot_identifier.go @@ -0,0 +1,101 @@ +package vaultik + +import ( + "errors" + "fmt" + "strings" + + "sneak.berlin/go/vaultik/internal/snapshot" +) + +// remoteKeyHexLen is the length of a full remote snapshot key: a SHA256 +// digest rendered as lowercase hex. +const remoteKeyHexLen = 64 + +// Sentinel errors for resolving a snapshot identifier against the store. +var ( + errSnapshotKeyNotFound = errors.New( + "no snapshot on the destination store matches this identifier") + errSnapshotKeyAmbiguous = errors.New( + "identifier matches more than one snapshot on the destination store") +) + +// resolveSnapshotRemoteKey turns a snapshot identifier supplied on the +// command line into the remote key that names the snapshot's metadata +// directory on the destination store. Every remote path a restore or +// verify reads is built from that key. +// +// Two forms are accepted, matching the two things a host can know: +// +// - A human snapshot ID (hostname_name_timestamp), which a host holding +// the local index has. It is hashed to its remote key; the store is +// not consulted. +// - A remote key, or the leading part of one, which is all a host with +// no local index can know — it is exactly what `snapshot list` prints +// for a remote-only snapshot (see formatRemoteOnlyID). It is resolved +// against the destination store's metadata listing; an identifier that +// matches no snapshot, or more than one, is an error. +// +// The two are told apart by shape: a remote key is lowercase hex, and a +// human snapshot ID never is (it carries a hostname, underscores, and an +// RFC3339 timestamp). +func (v *Vaultik) resolveSnapshotRemoteKey(identifier string) (string, error) { + if !isRemoteKeyOrPrefix(identifier) { + return snapshot.RemoteSnapshotKey(identifier), nil + } + + keys, err := v.listAllRemoteSnapshotKeys() + if err != nil { + return "", fmt.Errorf( + "listing destination store to resolve %q: %w", identifier, err) + } + + var matches []string + + for _, key := range keys { + if strings.HasPrefix(key, identifier) { + matches = append(matches, key) + } + } + + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return "", fmt.Errorf("%w: %s", errSnapshotKeyNotFound, identifier) + default: + return "", fmt.Errorf("%w: %s (%d matches)", + errSnapshotKeyAmbiguous, identifier, len(matches)) + } +} + +// resolveAndDownloadManifest resolves a snapshot identifier to its remote +// key (see resolveSnapshotRemoteKey) and downloads that snapshot's +// manifest. +func (v *Vaultik) resolveAndDownloadManifest( + identifier string, +) (*snapshot.Manifest, error) { + remoteKey, err := v.resolveSnapshotRemoteKey(identifier) + if err != nil { + return nil, err + } + + return v.downloadManifestByKey(remoteKey) +} + +// isRemoteKeyOrPrefix reports whether s is a full remote key or the +// leading part of one: 1 to 64 lowercase hex characters. A human snapshot +// ID is never all hex, so this shape test is enough to tell the two apart. +func isRemoteKeyOrPrefix(s string) bool { + if s == "" || len(s) > remoteKeyHexLen { + return false + } + + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + + return true +} diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 46f4be8..9a0d5bf 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -138,8 +138,15 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { func (v *Vaultik) loadVerificationData( snapshotID string, opts *VerifyOptions, result *VerifyResult, ) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { - // All remote paths use the hashed key derived from the human ID. - remoteKey := snapshot.RemoteSnapshotKey(snapshotID) + // Resolve the identifier to the snapshot's remote key. A human ID is + // hashed; a remote key (or its abbreviation, as printed for a + // remote-only snapshot) is used as-is, so a host with no local index + // can verify a snapshot it can only see on the store. + remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID) + if err != nil { + return nil, nil, nil, v.deepVerifyFailure(result, opts, + fmt.Sprintf("resolving snapshot identifier: %v", err), err) + } // Download manifest. downloadManifestByKey is the single reader for // remote manifests; see its doc comment. @@ -186,7 +193,7 @@ func (v *Vaultik) loadVerificationData( fmt.Errorf("failed to decrypt database: %w", err)) } - dbBlobs, err := v.getBlobsFromDatabase(snapshotID, tdb.DB) + dbBlobs, err := v.getBlobsFromDatabase(tdb.DB) if err != nil { _ = tdb.Close() @@ -501,19 +508,21 @@ func (v *Vaultik) verifyBlobFinalIntegrity( return nil } -// getBlobsFromDatabase gets all blobs for the snapshot from the database -func (v *Vaultik) getBlobsFromDatabase( - snapshotID string, db *sql.DB, -) ([]snapshot.BlobInfo, error) { +// getBlobsFromDatabase gets all blobs for the snapshot from the database. +// +// The exported per-snapshot database holds exactly one snapshot's data +// (see cleanSnapshotDB), so every row in snapshot_blobs belongs to it. +// We select them directly rather than filtering by the human snapshot ID, +// which a host restoring from the store alone does not have. +func (v *Vaultik) getBlobsFromDatabase(db *sql.DB) ([]snapshot.BlobInfo, error) { query := ` SELECT b.blob_hash, b.compressed_size FROM snapshot_blobs sb JOIN blobs b ON sb.blob_hash = b.blob_hash - WHERE sb.snapshot_id = ? ORDER BY b.blob_hash ` - rows, err := db.QueryContext(v.ctx, query, snapshotID) + rows, err := db.QueryContext(v.ctx, query) if err != nil { return nil, fmt.Errorf("failed to query snapshot blobs: %w", err) }