Compare commits
2
Commits
01d3ffc0a8
...
522d802ba1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
522d802ba1 | ||
|
|
66c80a70e3 |
@@ -178,7 +178,7 @@ vaultik version
|
|||||||
* `--verbose`, `-v`: Enable verbose output (on stderr — see below)
|
* `--verbose`, `-v`: Enable verbose output (on stderr — see below)
|
||||||
* `--debug`: Enable debug output (on stderr — see below)
|
* `--debug`: Enable debug output (on stderr — see below)
|
||||||
* `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner)
|
* `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner)
|
||||||
* `--skip-errors`: Skip files that cannot be read when creating a snapshot, or that cannot be restored when restoring, instead of aborting. Packing and storage errors (which would leave a chunk recorded but not stored) still abort the run.
|
* `--skip-errors`: Skip files that cannot be read when creating a snapshot, or that cannot be restored when restoring, instead of aborting. Packing and storage errors while creating a snapshot (which would leave a chunk recorded but not stored) still abort the run.
|
||||||
|
|
||||||
### locking
|
### locking
|
||||||
|
|
||||||
@@ -530,7 +530,7 @@ complete annotated example also lives in
|
|||||||
|
|
||||||
| Field | Default | Description |
|
| Field | Default | Description |
|
||||||
|-------|---------|-------------|
|
|-------|---------|-------------|
|
||||||
| `age_recipients` | (required) | Age public keys for encryption |
|
| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty |
|
||||||
| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. |
|
| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. |
|
||||||
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
||||||
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
||||||
|
|||||||
@@ -22,6 +22,22 @@ the tag exists and is exercised; what is left is merging `next` to
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-06: Made the README's steps for restoring on another machine
|
||||||
|
work ([issue #221](https://git.eeqj.de/sneak/vaultik/issues/221)).
|
||||||
|
`config init` wrote a placeholder recipient that `config.Load`
|
||||||
|
rejects, so every command on the new machine failed before reaching
|
||||||
|
the store. The file now has an empty `age_recipients` list,
|
||||||
|
`config.Load` accepts an empty list, and `snapshot create` refuses to
|
||||||
|
run without a recipient.
|
||||||
|
|
||||||
|
- 2026-10-06: Made `snapshot restore --skip-errors` skip the files that
|
||||||
|
need a blob it cannot download
|
||||||
|
([issue #218](https://git.eeqj.de/sneak/vaultik/issues/218)). A missing
|
||||||
|
or damaged blob ended the restore even with the flag, after restoring
|
||||||
|
whichever files happened to come first. Every file that needs such a
|
||||||
|
blob is now reported as failed, the rest are restored, and the command
|
||||||
|
still exits non-zero. Without the flag the blob error still aborts.
|
||||||
|
|
||||||
- 2026-10-06: Re-vendored the canonical files from `sneak/prompts` at
|
- 2026-10-06: Re-vendored the canonical files from `sneak/prompts` at
|
||||||
`dd4027b` ([issue #213](https://git.eeqj.de/sneak/vaultik/issues/213)).
|
`dd4027b` ([issue #213](https://git.eeqj.de/sneak/vaultik/issues/213)).
|
||||||
Linting and testing are now the `lint` and `test` phases of the
|
Linting and testing are now the `lint` and `test` phases of the
|
||||||
|
|||||||
+2
-1
@@ -3,7 +3,8 @@
|
|||||||
# Copy this file and uncomment/modify the values you need
|
# Copy this file and uncomment/modify the values you need
|
||||||
|
|
||||||
# Age recipient public keys for encryption
|
# Age recipient public keys for encryption
|
||||||
# This is REQUIRED - backups are encrypted to these public keys
|
# Backups are encrypted to these public keys. snapshot create needs at least
|
||||||
|
# one; listing, verifying and restoring do not
|
||||||
# Generate with: age-keygen | grep "public key"
|
# Generate with: age-keygen | grep "public key"
|
||||||
age_recipients:
|
age_recipients:
|
||||||
- age1cj2k2addawy294f6k2gr2mf9gps9r3syplryxca3nvxj3daqm96qfp84tz
|
- age1cj2k2addawy294f6k2gr2mf9gps9r3syplryxca3nvxj3daqm96qfp84tz
|
||||||
|
|||||||
@@ -45,16 +45,19 @@ const defaultConfigTemplate = `# vaultik configuration
|
|||||||
|
|
||||||
# ─── REQUIRED ────────────────────────────────────────────────────────────────
|
# ─── REQUIRED ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
# Age recipient public keys for encryption.
|
# Age recipient public keys for encryption. snapshot create needs at least
|
||||||
|
# one; listing, verifying and restoring do not, so a machine that only
|
||||||
|
# restores can leave this empty.
|
||||||
# Backups are encrypted to ALL listed recipients; any one of the corresponding
|
# Backups are encrypted to ALL listed recipients; any one of the corresponding
|
||||||
# private keys can decrypt. Adding a recipient later does not re-encrypt data
|
# private keys can decrypt. Adding a recipient later does not re-encrypt data
|
||||||
# already stored: deduplicated chunks and existing blobs stay encrypted to the
|
# already stored: deduplicated chunks and existing blobs stay encrypted to the
|
||||||
# earlier recipients, so a newly added key cannot restore them on its own (see
|
# earlier recipients, so a newly added key cannot restore them on its own (see
|
||||||
# docs/REPOSTRUCTURE.md, Accepted Risks). Generate a keypair with:
|
# docs/REPOSTRUCTURE.md, Accepted Risks). Generate a keypair and add its
|
||||||
|
# public key with:
|
||||||
# age-keygen -o vaultik_backup_private_key.txt
|
# age-keygen -o vaultik_backup_private_key.txt
|
||||||
# grep 'public key' vaultik_backup_private_key.txt
|
# grep 'public key' vaultik_backup_private_key.txt
|
||||||
age_recipients:
|
# vaultik config set age_recipients.0 age1...
|
||||||
- age1REPLACE_WITH_YOUR_PUBLIC_KEY
|
age_recipients: []
|
||||||
|
|
||||||
# Named snapshots. Each snapshot backs up one or more paths and can have its
|
# Named snapshots. Each snapshot backs up one or more paths and can have its
|
||||||
# own exclude patterns in addition to the global excludes below.
|
# own exclude patterns in addition to the global excludes below.
|
||||||
|
|||||||
@@ -24,8 +24,10 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
|
|||||||
t.Fatalf("default config template is not valid YAML: %v", err)
|
t.Fatalf("default config template is not valid YAML: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(cfg.AgeRecipients) != 1 {
|
// A placeholder recipient would fail config.Load, so the template
|
||||||
t.Errorf("expected 1 placeholder age recipient, got %d", len(cfg.AgeRecipients))
|
// leaves the list empty.
|
||||||
|
if len(cfg.AgeRecipients) != 0 {
|
||||||
|
t.Errorf("expected no age recipients, got %d", len(cfg.AgeRecipients))
|
||||||
}
|
}
|
||||||
|
|
||||||
home, ok := cfg.Snapshots["home"]
|
home, ok := cfg.Snapshots["home"]
|
||||||
@@ -55,6 +57,43 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
|
||||||
|
// file `config init` writes, `config set age_recipients.0` and
|
||||||
|
// `config set storage_url` give a config that loads with that recipient.
|
||||||
|
func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
|
||||||
|
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out := ui.NewWithColor(&bytes.Buffer{}, false)
|
||||||
|
|
||||||
|
err = writeConfigSet(out, path, "age_recipients.0", recipient)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config set age_recipients.0: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = writeConfigSet(out, path, "storage_url", "file:///mnt/backups")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config set storage_url: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.Load(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
|
||||||
|
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const testYAML = `# top comment
|
const testYAML = `# top comment
|
||||||
compression_level: 3
|
compression_level: 3
|
||||||
age_recipients:
|
age_recipients:
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ on the source system.`,
|
|||||||
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
|
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
|
||||||
"Skip files that cannot be read when creating a snapshot, or "+
|
"Skip files that cannot be read when creating a snapshot, or "+
|
||||||
"that cannot be restored when restoring, instead of aborting "+
|
"that cannot be restored when restoring, instead of aborting "+
|
||||||
"(packing and storage errors still abort)")
|
"(packing and storage errors while creating a snapshot still "+
|
||||||
|
"abort)")
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
|
|||||||
@@ -43,8 +43,6 @@ const (
|
|||||||
// Sentinel validation errors.
|
// Sentinel validation errors.
|
||||||
var (
|
var (
|
||||||
errNoConfigPath = errors.New("config path not provided")
|
errNoConfigPath = errors.New("config path not provided")
|
||||||
errNoAgeRecipients = errors.New(
|
|
||||||
"at least one age_recipient is required (generate with: age-keygen)")
|
|
||||||
errRecipientIsSecretKey = errors.New(
|
errRecipientIsSecretKey = errors.New(
|
||||||
"an age secret key was given where a public key (age1...) belongs")
|
"an age secret key was given where a public key (age1...) belongs")
|
||||||
errRecipientNotX25519 = errors.New(
|
errRecipientNotX25519 = errors.New(
|
||||||
@@ -323,9 +321,10 @@ func Load(path string) (*Config, error) {
|
|||||||
|
|
||||||
// Validate checks if the configuration is valid and complete.
|
// Validate checks if the configuration is valid and complete.
|
||||||
// It ensures all required fields are present and have valid values:
|
// It ensures all required fields are present and have valid values:
|
||||||
// - At least one age recipient must be specified, and every recipient must
|
// - Every age recipient must parse as an X25519 age1... public key (so a
|
||||||
// parse as an X25519 age1... public key (so a bad entry fails at load, not
|
// bad entry fails at load, not mid-backup); errors name the position,
|
||||||
// mid-backup); errors name the position, never the value
|
// never the value. An empty list is accepted, because only snapshot
|
||||||
|
// create needs a recipient and it checks for one itself
|
||||||
// - At least one snapshot must be configured with at least one path
|
// - At least one snapshot must be configured with at least one path
|
||||||
// - Storage must be configured (either storage_url or s3.* fields)
|
// - Storage must be configured (either storage_url or s3.* fields)
|
||||||
// - Chunk size must be at least 1MB
|
// - Chunk size must be at least 1MB
|
||||||
@@ -336,10 +335,6 @@ func Load(path string) (*Config, error) {
|
|||||||
//
|
//
|
||||||
// Returns an error describing the first validation failure encountered.
|
// Returns an error describing the first validation failure encountered.
|
||||||
func (c *Config) Validate() error {
|
func (c *Config) Validate() error {
|
||||||
if len(c.AgeRecipients) == 0 {
|
|
||||||
return errNoAgeRecipients
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, recipient := range c.AgeRecipients {
|
for i, recipient := range c.AgeRecipients {
|
||||||
err := validateAgeRecipient(recipient)
|
err := validateAgeRecipient(recipient)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -223,7 +223,8 @@ func TestValidateBlobSizeLimit(t *testing.T) {
|
|||||||
|
|
||||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||||
// least of all a pasted secret key — is echoed in the error.
|
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||||
|
// loads, because only snapshot create needs a recipient.
|
||||||
func TestValidateAgeRecipients(t *testing.T) {
|
func TestValidateAgeRecipients(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -244,7 +245,12 @@ func TestValidateAgeRecipients(t *testing.T) {
|
|||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "config init placeholder is rejected",
|
name: "no recipients is accepted",
|
||||||
|
recipients: nil,
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "placeholder recipient is rejected",
|
||||||
recipients: []string{"age1REPLACE_WITH_YOUR_PUBLIC_KEY"},
|
recipients: []string{"age1REPLACE_WITH_YOUR_PUBLIC_KEY"},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -355,7 +355,7 @@ func (v *Vaultik) runRestoreLoop(
|
|||||||
|
|
||||||
fileID, ready := plan.popReady()
|
fileID, ready := plan.popReady()
|
||||||
if !ready {
|
if !ready {
|
||||||
downloaded, err := session.downloadNextBlobSet(plan)
|
downloaded, err := session.downloadNextBlobSet(plan, filesByID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -411,7 +411,13 @@ func (v *Vaultik) runRestoreLoop(
|
|||||||
// blob set and downloads its blobs; after each blob lands, the plan
|
// blob set and downloads its blobs; after each blob lands, the plan
|
||||||
// moves any pending file whose set just emptied onto the ready queue.
|
// moves any pending file whose set just emptied onto the ready queue.
|
||||||
// Returns false when nothing is pending download (the caller stops).
|
// Returns false when nothing is pending download (the caller stops).
|
||||||
func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
|
//
|
||||||
|
// A blob that cannot be downloaded is reported through
|
||||||
|
// handleRestoreFileError for every pending file that references it, so
|
||||||
|
// it aborts the restore unless --skip-errors is set.
|
||||||
|
func (s *restoreSession) downloadNextBlobSet(
|
||||||
|
plan *restorePlan, filesByID map[types.FileID]*database.File,
|
||||||
|
) (bool, error) {
|
||||||
s.sweeper.sweep()
|
s.sweeper.sweep()
|
||||||
|
|
||||||
next, ok := plan.pickNextDownload()
|
next, ok := plan.pickNextDownload()
|
||||||
@@ -433,7 +439,25 @@ func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
|
|||||||
|
|
||||||
err := s.downloadBlobToCache(hash, blob.CompressedSize, blob.UncompressedSize)
|
err := s.downloadBlobToCache(hash, blob.CompressedSize, blob.UncompressedSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("downloading blob %s: %w", shortHash(hash), err)
|
err = fmt.Errorf("downloading blob %s: %w", shortHash(hash), err)
|
||||||
|
|
||||||
|
// On cancel the error says nothing about the blob, so it ends
|
||||||
|
// the restore instead of failing the files that need it.
|
||||||
|
if s.ctx.Err() != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, fileID := range plan.filesReferencingBlob(hash) {
|
||||||
|
fileErr := s.v.handleRestoreFileError(
|
||||||
|
plan, s.opts, s.result, filesByID[fileID], fileID, err)
|
||||||
|
if fileErr != nil {
|
||||||
|
return false, fileErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// next is among the failed files, so the rest of its blob set
|
||||||
|
// is left for any other file that still needs it.
|
||||||
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
s.result.BlobsDownloaded++
|
s.result.BlobsDownloaded++
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/cli"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
@@ -27,10 +28,12 @@ import (
|
|||||||
//
|
//
|
||||||
// The backup half writes a snapshot with one index and hostname. The
|
// The backup half writes a snapshot with one index and hostname. The
|
||||||
// restore half throws that index away entirely: a fresh, empty index and
|
// restore half throws that index away entirely: a fresh, empty index and
|
||||||
// a config that shares nothing with the original but the storage location
|
// a config written by `config init` and `config set storage_url`, as in
|
||||||
// and the secret key. If restore or verify needed the original local
|
// the README's steps for restoring on another machine, which shares
|
||||||
// index — or the human snapshot ID that only that index holds — this test
|
// nothing with the original but the storage location and the secret key.
|
||||||
// could not run, because the recovery host can know neither.
|
// If restore or verify needed the original local index — or
|
||||||
|
// the human snapshot ID that only that index holds — this test could not
|
||||||
|
// run, because the recovery host can know neither.
|
||||||
func TestRestoreOnAnotherMachine(t *testing.T) {
|
func TestRestoreOnAnotherMachine(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -58,7 +61,12 @@ func TestRestoreOnAnotherMachine(t *testing.T) {
|
|||||||
|
|
||||||
// Recovery host: a fresh empty index, a different hostname, and no
|
// Recovery host: a fresh empty index, a different hostname, and no
|
||||||
// age_recipients — only the secret key and the same storage location.
|
// age_recipients — only the secret key and the same storage location.
|
||||||
recovery, stdout := newRecoveryHost(ctx, t, fs, storer)
|
configPath := filepath.Join(tempDir, "config.yml")
|
||||||
|
runVaultikCommand(t, "--config", configPath, "config", "init")
|
||||||
|
runVaultikCommand(t, "--config", configPath,
|
||||||
|
"config", "set", "storage_url", "file://"+storeDir)
|
||||||
|
|
||||||
|
recovery, stdout := newRecoveryHost(ctx, t, fs, storer, configPath)
|
||||||
|
|
||||||
// The recovery index really is empty. This is the assertion that makes
|
// The recovery index really is empty. This is the assertion that makes
|
||||||
// the test a guard against restore quietly depending on the original
|
// the test a guard against restore quietly depending on the original
|
||||||
@@ -93,6 +101,10 @@ func TestRestoreOnAnotherMachine(t *testing.T) {
|
|||||||
remote.RemoteKey, &vaultik.VerifyOptions{Deep: true}))
|
remote.RemoteKey, &vaultik.VerifyOptions{Deep: true}))
|
||||||
|
|
||||||
assertRestoredTreeMatches(t, fs, restoreDir, sourceFiles)
|
assertRestoredTreeMatches(t, fs, restoreDir, sourceFiles)
|
||||||
|
|
||||||
|
// With no public key configured, a backup must refuse to start.
|
||||||
|
err = recovery.CreateSnapshot(&vaultik.SnapshotCreateOptions{Cron: true})
|
||||||
|
require.ErrorContains(t, err, "age_recipients")
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeRecoverySourceTree writes a small source tree spanning several
|
// writeRecoverySourceTree writes a small source tree spanning several
|
||||||
@@ -118,14 +130,24 @@ func writeRecoverySourceTree(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newRecoveryHost builds the Vaultik a replacement machine would run: an
|
// newRecoveryHost builds the Vaultik a replacement machine would run: an
|
||||||
// empty in-memory index, a hostname different from the backup host, no
|
// empty in-memory index, a hostname different from the backup host, and
|
||||||
// age_recipients, and only the secret key plus the shared storer. It
|
// only the secret key plus the shared storer. Its config is read from
|
||||||
// returns the instance and the buffer its stdout is wired to.
|
// configPath by config.Load, as every command reads it. It returns the
|
||||||
|
// instance and the buffer its stdout is wired to.
|
||||||
func newRecoveryHost(
|
func newRecoveryHost(
|
||||||
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
||||||
|
configPath string,
|
||||||
) (*vaultik.Vaultik, *bytes.Buffer) {
|
) (*vaultik.Vaultik, *bytes.Buffer) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
cfg, err := config.Load(configPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Set directly rather than through VAULTIK_AGE_SECRET_KEY, which a
|
||||||
|
// parallel test cannot change.
|
||||||
|
cfg.AgeSecretKey = testAgeSecretKey
|
||||||
|
cfg.Hostname = "recovery-host"
|
||||||
|
|
||||||
recoveryDB, err := database.New(ctx, ":memory:")
|
recoveryDB, err := database.New(ctx, ":memory:")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
t.Cleanup(func() { _ = recoveryDB.Close() })
|
t.Cleanup(func() { _ = recoveryDB.Close() })
|
||||||
@@ -133,10 +155,7 @@ func newRecoveryHost(
|
|||||||
stdout := &bytes.Buffer{}
|
stdout := &bytes.Buffer{}
|
||||||
|
|
||||||
recovery := &vaultik.Vaultik{
|
recovery := &vaultik.Vaultik{
|
||||||
Config: &config.Config{
|
Config: cfg,
|
||||||
AgeSecretKey: testAgeSecretKey,
|
|
||||||
Hostname: "recovery-host",
|
|
||||||
},
|
|
||||||
Storage: storer,
|
Storage: storer,
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
Repositories: database.NewRepositories(recoveryDB),
|
Repositories: database.NewRepositories(recoveryDB),
|
||||||
@@ -150,6 +169,20 @@ func newRecoveryHost(
|
|||||||
return recovery, stdout
|
return recovery, stdout
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runVaultikCommand runs one vaultik command line in-process and fails the
|
||||||
|
// test if it returns an error. The command writes the cli package's global
|
||||||
|
// flag variables, so it must not be called from two tests that run at the
|
||||||
|
// same time.
|
||||||
|
func runVaultikCommand(t *testing.T, args ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cmd := cli.NewRootCommand()
|
||||||
|
cmd.SetArgs(args)
|
||||||
|
cmd.SetOut(io.Discard)
|
||||||
|
cmd.SetErr(io.Discard)
|
||||||
|
require.NoError(t, cmd.Execute())
|
||||||
|
}
|
||||||
|
|
||||||
// assertRestoredTreeMatches byte-compares every restored file against its
|
// assertRestoredTreeMatches byte-compares every restored file against its
|
||||||
// source content.
|
// source content.
|
||||||
func assertRestoredTreeMatches(
|
func assertRestoredTreeMatches(
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package vaultik //nolint:testpackage // sets ctx/cancel and inspects scratch files
|
package vaultik //nolint:testpackage // sets ctx/cancel and inspects scratch files
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
@@ -157,3 +159,51 @@ func scratchEntries(t *testing.T, dir string) []string {
|
|||||||
|
|
||||||
return matches
|
return matches
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRestoreSkipErrorsCancelDuringBlobDownload cancels a SkipErrors
|
||||||
|
// restore while a blob download is in progress. The download fails only
|
||||||
|
// because of the cancel, so Restore must return context.Canceled without
|
||||||
|
// reporting the file that needs the blob as failed.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||||
|
func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
cfg, storer, snapshotID, srcPath := backupOneFile(context.Background(),
|
||||||
|
t, fs, tempDir, "a.txt", []byte("hello vaultik"), 0o644)
|
||||||
|
|
||||||
|
gate := newBlockingBlobStorer(storer)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
v := newRestoreVaultik(ctx, cfg, gate, fs)
|
||||||
|
v.UI = ui.NewWithColor(&out, false)
|
||||||
|
|
||||||
|
restoreErr := make(chan error, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
restoreErr <- v.Restore(&RestoreOptions{
|
||||||
|
SnapshotID: snapshotID,
|
||||||
|
TargetDir: filepath.Join(tempDir, "restored"),
|
||||||
|
SkipErrors: true,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-gate.entered:
|
||||||
|
case <-time.After(30 * time.Second):
|
||||||
|
t.Fatal("restore never reached the blob-download phase")
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
require.ErrorIs(t, <-restoreErr, context.Canceled)
|
||||||
|
assert.NotContains(t, out.String(), srcPath,
|
||||||
|
"the cancel was reported as a failed file")
|
||||||
|
}
|
||||||
|
|||||||
@@ -214,6 +214,20 @@ func (p *restorePlan) blobsNeeded(fileID types.FileID) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// filesReferencingBlob returns the pending files that reference the
|
||||||
|
// named blob, in any order. The result is a copy, so the caller may
|
||||||
|
// finishFile each of them while ranging over it.
|
||||||
|
func (p *restorePlan) filesReferencingBlob(blobHash string) []types.FileID {
|
||||||
|
files := p.blobFiles[blobHash]
|
||||||
|
|
||||||
|
out := make([]types.FileID, 0, len(files))
|
||||||
|
for id := range files {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// hasPending reports whether any unfinished files remain.
|
// hasPending reports whether any unfinished files remain.
|
||||||
func (p *restorePlan) hasPending() bool {
|
func (p *restorePlan) hasPending() bool {
|
||||||
return len(p.fileBlobs) > 0
|
return len(p.fileBlobs) > 0
|
||||||
|
|||||||
@@ -0,0 +1,194 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A file no larger than the chunker's minimum chunk size (a quarter of
|
||||||
|
// faultChunkSize) is stored as one chunk, so it lives in exactly one blob.
|
||||||
|
// More of them than fit in faultMaxBlobSize make the snapshot span two
|
||||||
|
// blobs.
|
||||||
|
const (
|
||||||
|
missingBlobFileBytes = int(faultChunkSize / 4)
|
||||||
|
missingBlobFileCount = 20
|
||||||
|
)
|
||||||
|
|
||||||
|
// missingBlobBackup is a snapshot of single-chunk files spread over two
|
||||||
|
// blobs, with one of those blobs deleted from the store.
|
||||||
|
type missingBlobBackup struct {
|
||||||
|
fs afero.Fs
|
||||||
|
cfg *config.Config
|
||||||
|
storer storage.Storer
|
||||||
|
snapshotID string
|
||||||
|
restoreDir string
|
||||||
|
// files holds the original content by source path.
|
||||||
|
files map[string][]byte
|
||||||
|
// lost holds the source paths whose chunk is in the deleted blob.
|
||||||
|
lost map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
|
||||||
|
// after one blob of a two-blob snapshot was deleted. Every file stored in
|
||||||
|
// that blob must be reported as failed and left absent, every other file
|
||||||
|
// must be restored intact, and Restore must still return an error.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||||
|
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
backup := backupThenDeleteOneBlob(ctx, t)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
||||||
|
v.UI = ui.NewWithColor(&out, false)
|
||||||
|
|
||||||
|
err := v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: backup.snapshotID,
|
||||||
|
TargetDir: backup.restoreDir,
|
||||||
|
SkipErrors: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
require.Error(t, err, "restore must fail when files were skipped")
|
||||||
|
assert.Contains(t, err.Error(),
|
||||||
|
fmt.Sprintf("%d file(s) failed to restore", len(backup.lost)))
|
||||||
|
|
||||||
|
for path, content := range backup.files {
|
||||||
|
restored := filepath.Join(backup.restoreDir, path)
|
||||||
|
|
||||||
|
if backup.lost[path] {
|
||||||
|
assert.Containsf(t, out.String(), path,
|
||||||
|
"%s needs the deleted blob and must be reported", path)
|
||||||
|
assert.NoFileExists(t, restored)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := afero.ReadFile(backup.fs, restored)
|
||||||
|
require.NoErrorf(t, err, "%s does not need the deleted blob", path)
|
||||||
|
assert.Equalf(t, content, got, "%s restored with wrong content", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
|
||||||
|
// still ends the restore with an error when SkipErrors is not set.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||||
|
func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
backup := backupThenDeleteOneBlob(ctx, t)
|
||||||
|
|
||||||
|
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
|
||||||
|
|
||||||
|
err := v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: backup.snapshotID,
|
||||||
|
TargetDir: backup.restoreDir,
|
||||||
|
})
|
||||||
|
|
||||||
|
require.ErrorIs(t, err, storage.ErrNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
|
||||||
|
// files, then deletes from the store the blob holding the first of them.
|
||||||
|
func backupThenDeleteOneBlob(
|
||||||
|
ctx context.Context, t *testing.T,
|
||||||
|
) *missingBlobBackup {
|
||||||
|
t.Helper()
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
cfg := faultTestConfig()
|
||||||
|
|
||||||
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||||
|
|
||||||
|
files := make(map[string][]byte, missingBlobFileCount)
|
||||||
|
|
||||||
|
for i := range missingBlobFileCount {
|
||||||
|
path := filepath.Join(dataDir, fmt.Sprintf("file-%02d.bin", i))
|
||||||
|
files[path] = bytesPattern(
|
||||||
|
fmt.Sprintf("file-%02d-", i), missingBlobFileBytes)
|
||||||
|
require.NoError(t, afero.WriteFile(fs, path, files[path], 0o644))
|
||||||
|
}
|
||||||
|
|
||||||
|
storer, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
|
id := fullFaultBackup(
|
||||||
|
ctx, t, fs, storer, cfg, repos, dataDir, dbPath, "missingblob")
|
||||||
|
|
||||||
|
blobOfFile := make(map[string]string, len(files))
|
||||||
|
for path := range files {
|
||||||
|
blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
deleted := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
|
||||||
|
require.NoError(t, storer.Delete(ctx, fmt.Sprintf(
|
||||||
|
"blobs/%s/%s/%s", deleted[:2], deleted[2:4], deleted)))
|
||||||
|
|
||||||
|
lost := make(map[string]bool)
|
||||||
|
|
||||||
|
for path, hash := range blobOfFile {
|
||||||
|
if hash == deleted {
|
||||||
|
lost[path] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Less(t, len(lost), len(files),
|
||||||
|
"the snapshot must span more than one blob")
|
||||||
|
|
||||||
|
return &missingBlobBackup{
|
||||||
|
fs: fs,
|
||||||
|
cfg: cfg,
|
||||||
|
storer: storer,
|
||||||
|
snapshotID: id,
|
||||||
|
restoreDir: filepath.Join(tempDir, "restored"),
|
||||||
|
files: files,
|
||||||
|
lost: lost,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// blobHashOfSingleChunkFile returns the hash of the blob holding the one
|
||||||
|
// chunk of the file at path, as recorded in the local index.
|
||||||
|
func blobHashOfSingleChunkFile(
|
||||||
|
ctx context.Context, t *testing.T,
|
||||||
|
repos *database.Repositories, path string,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
chunks, err := repos.FileChunks.GetByPath(ctx, path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Lenf(t, chunks, 1, "%s must be a single chunk", path)
|
||||||
|
|
||||||
|
blobChunk, err := repos.BlobChunks.GetByChunkHash(
|
||||||
|
ctx, chunks[0].ChunkHash.String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNilf(t, blobChunk, "chunk of %s is in no blob", path)
|
||||||
|
|
||||||
|
blob, err := repos.Blobs.GetByID(ctx, blobChunk.BlobID.String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return blob.Hash.String()
|
||||||
|
}
|
||||||
@@ -23,6 +23,9 @@ var (
|
|||||||
errSnapshotVerifyFailed = errors.New("verification failed")
|
errSnapshotVerifyFailed = errors.New("verification failed")
|
||||||
errRemoveAllNeedsForce = errors.New("--all requires --force")
|
errRemoveAllNeedsForce = errors.New("--all requires --force")
|
||||||
errInvalidTableName = errors.New("invalid table name")
|
errInvalidTableName = errors.New("invalid table name")
|
||||||
|
errNoAgeRecipients = errors.New(
|
||||||
|
"creating a snapshot needs at least one public key in " +
|
||||||
|
"age_recipients (generate a keypair with: age-keygen)")
|
||||||
)
|
)
|
||||||
|
|
||||||
// listRecentLimit caps how many snapshot rows are fetched from the
|
// listRecentLimit caps how many snapshot rows are fetched from the
|
||||||
@@ -42,6 +45,12 @@ type SnapshotCreateOptions struct {
|
|||||||
|
|
||||||
// CreateSnapshot executes the snapshot creation operation
|
// CreateSnapshot executes the snapshot creation operation
|
||||||
func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||||
|
// config.Load accepts an empty list, since listing, verifying and
|
||||||
|
// restoring need no public key.
|
||||||
|
if len(v.Config.AgeRecipients) == 0 {
|
||||||
|
return errNoAgeRecipients
|
||||||
|
}
|
||||||
|
|
||||||
overallStartTime := time.Now()
|
overallStartTime := time.Now()
|
||||||
|
|
||||||
log.Info("Starting snapshot creation",
|
log.Info("Starting snapshot creation",
|
||||||
|
|||||||
Reference in New Issue
Block a user