Compare commits

..
Author SHA1 Message Date
sneak 68ba6bafa8 Fix missing-key errors for restore and deep verify to name both key sources (closes #401)
check / check (push) Successful in 10m25s
The missing-key error for restore and deep verify is one value, errDecryptionKeyRequired, whose text names both places the key can be set: the VAULTIK_AGE_SECRET_KEY environment variable first, then the age_secret_key config file field.

Model: deepseek-v4-pro
2026-11-13 22:24:34 +00:00
10 changed files with 40 additions and 233 deletions

No files matched your search

+5 -11
View File
@@ -468,12 +468,10 @@ destination store. Blobs are NOT touched — deleting blobs requires reading eve
remaining remote manifest (the destination store may hold snapshots this host remaining remote manifest (the destination store may hold snapshots this host
doesn't know about), which is what `vaultik prune` does. The command prints the doesn't know about), which is what `vaultik prune` does. The command prints the
`vaultik prune` invocation to run as a follow-up only when it removed the `vaultik prune` invocation to run as a follow-up only when it removed the
snapshot's metadata from the destination store, and never under `--json`. It snapshot's metadata from the destination store, and never under `--json`. An ID
accepts the same identifiers as `snapshot restore`: a snapshot ID, or a that names no snapshot in the local index or on the destination store is an
remote-only snapshot's remote key (or an unambiguous leading part of it). An error; with `--local-only`, the snapshot must be in the local index. Local row
identifier that names no snapshot in the local index or on the destination store cleanup (files, chunks, blobs the snapshot was the last referrer for) runs
is an error; with `--local-only`, the snapshot must be in the local index. Local
row cleanup (files, chunks, blobs the snapshot was the last referrer for) runs
automatically. If removing the snapshot's metadata from the destination store automatically. If removing the snapshot's metadata from the destination store
fails (for example because the store is unreachable, or the local index holds fails (for example because the store is unreachable, or the local index holds
snapshots and the destination answers that it has no `vaultik-destination.txt`), snapshots and the destination answers that it has no `vaultik-destination.txt`),
@@ -602,11 +600,7 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots. index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). The providers. Requires rclone to be configured separately (`rclone config`).
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem shows an object only once its upload has completed, and on the local filesystem
+2 -4
View File
@@ -331,10 +331,8 @@ know about), which is what 'vaultik prune' does. The command prints the
'vaultik prune' invocation to run as a follow-up only when it removed the 'vaultik prune' invocation to run as a follow-up only when it removed the
snapshot's metadata from the destination store, and never under --json. snapshot's metadata from the destination store, and never under --json.
The snapshot may be named by its ID or by its remote key, which An ID that names no snapshot in the local index or on the destination
'snapshot list' prints for a remote-only snapshot. An unambiguous store is an error.
leading part of the key is enough. An identifier that names no snapshot
in the local index or on the destination store is an error.
Use --local-only to skip the remote half (e.g. when you want to forget a Use --local-only to skip the remote half (e.g. when you want to forget a
snapshot locally without touching the destination store); the snapshot snapshot locally without touching the destination store); the snapshot
-23
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
@@ -24,15 +23,6 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured. // ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config") var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction. // RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers. // This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct { type RcloneStorer struct {
@@ -63,19 +53,6 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path rclonePath += path
} }
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in // On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read. // each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the // rclone's use_server_modtime setting makes ModTime the time the
-73
View File
@@ -436,79 +436,6 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
} }
} }
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an // TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By // rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one // default rclone reads the time it keeps in each object's metadata, one
-62
View File
@@ -472,68 +472,6 @@ func TestRemoveSnapshot_NotInLocalIndex(t *testing.T) {
assert.Contains(t, out, "Removed snapshot metadata from remote storage") assert.Contains(t, out, "Removed snapshot metadata from remote storage")
} }
// TestRemoveSnapshot_ByRemoteKey checks that a snapshot only the
// destination store holds can be removed by the remote key that
// `snapshot list` shows for it, or by a leading part of that key.
func TestRemoveSnapshot_ByRemoteKey(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
for name, identifier := range map[string]string{
"full key": remoteKey,
"leading part": remoteKey[:8],
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t)
result, err := tv.RemoveSnapshot(identifier,
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.True(t, result.RemoteRemoved)
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
})
}
}
// TestRemoveSnapshot_ByRemoteKeyInLocalIndex checks that a snapshot the
// local index holds, named by its remote key, is removed from the local
// index as well as from the destination store.
func TestRemoveSnapshot_ByRemoteKeyInLocalIndex(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the marker.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t, "snapshot-001")
result, err := tv.RemoveSnapshot(snapshot.RemoteSnapshotKey("snapshot-001"),
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.True(t, result.RemoteRemoved)
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
snap, err := tv.Repositories.Snapshots.GetByID(context.Background(),
"snapshot-001")
require.NoError(t, err)
assert.Nil(t, snap)
}
// TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a // TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a
// snapshot only the destination store holds fails when its metadata // snapshot only the destination store holds fails when its metadata
// cannot be deleted there, under --force and under --json: nothing was // cannot be deleted there, under --force and under --json: nothing was
+3 -4
View File
@@ -31,10 +31,9 @@ var (
errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot") errPathsNotInSnapshot = errors.New("requested path(s) not in the snapshot")
errFilesFailedVerify = errors.New("files failed verification") errFilesFailedVerify = errors.New("files failed verification")
errDecryptionKeyRequired = errors.New( errDecryptionKeyRequired = errors.New(
"decryption key required\n\n" + "age secret key is missing; set it in the " +
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " + "VAULTIK_AGE_SECRET_KEY environment variable or the " +
"age private key file:\n" + "age_secret_key config file field")
" export VAULTIK_AGE_SECRET_KEY=\"$(cat vaultik_backup_private_key.txt)\"")
// errInvalidAgeSecretKey is returned when the configured key does not // errInvalidAgeSecretKey is returned when the configured key does not
// parse as any age identity. It names the source but never the value, // parse as any age identity. It names the source but never the value,
// which is secret, so the message is safe to print and log. // which is secret, so the message is safe to print and log.
+20
View File
@@ -106,3 +106,23 @@ func TestRestoreIdentitiesMissingKey(t *testing.T) {
_, err := v.restoreIdentities() _, err := v.restoreIdentities()
require.ErrorIs(t, err, errDecryptionKeyRequired) require.ErrorIs(t, err, errDecryptionKeyRequired)
} }
// TestMissingKeyErrorsNameBothSources asserts restore and deep verify
// return the same missing-key error, whose text names both places the key
// can be set: the VAULTIK_AGE_SECRET_KEY environment variable first since
// it is preferred, then the age_secret_key config file field.
func TestMissingKeyErrorsNameBothSources(t *testing.T) {
t.Parallel()
const want = "age secret key is missing; set it in " +
"the VAULTIK_AGE_SECRET_KEY environment variable or the " +
"age_secret_key config file field"
v := &Vaultik{Config: &config.Config{}}
_, err := v.restoreIdentities()
require.EqualError(t, err, want)
err = v.RunDeepVerify("any-snapshot", &VerifyOptions{})
require.EqualError(t, err, want)
}
+7 -42
View File
@@ -1467,8 +1467,7 @@ func (v *Vaultik) RemoveSnapshot(
return result, err return result, err
} }
snapshotID, remoteKey, inLocalIndex, err := v.findSnapshotToRemove( inLocalIndex, err := v.checkSnapshotExists(snapshotID, opts.LocalOnly)
snapshotID, opts.LocalOnly)
if err != nil { if err != nil {
return result, err return result, err
} }
@@ -1499,7 +1498,7 @@ func (v *Vaultik) RemoveSnapshot(
} }
result.RemoteRemoved, err = v.removeSnapshotRemote( result.RemoteRemoved, err = v.removeSnapshotRemote(
snapshotID, remoteKey, inLocalIndex, markerErr, opts) snapshotID, inLocalIndex, markerErr, opts)
if err != nil { if err != nil {
return result, err return result, err
} }
@@ -1531,41 +1530,6 @@ func (v *Vaultik) RemoveSnapshot(
return result, nil return result, nil
} }
// findSnapshotToRemove returns the ID of the snapshot that identifier
// names, its remote key, and whether the local index holds it. A remote
// key, or the leading part of one, is resolved on the destination store;
// when the local index holds the snapshot with that key, its ID is
// returned, so it is removed from both.
func (v *Vaultik) findSnapshotToRemove(
identifier string, localOnly bool,
) (string, string, bool, error) {
if !isRemoteKeyOrPrefix(identifier) || localOnly {
inLocalIndex, err := v.checkSnapshotExists(identifier, localOnly)
return identifier, snapshot.RemoteSnapshotKey(identifier),
inLocalIndex, err
}
remoteKey, err := v.resolveSnapshotRemoteKey(identifier)
if err != nil {
return "", "", false, err
}
localIDs, err := v.localSnapshotIDs()
if err != nil {
return "", "", false, fmt.Errorf(
"looking up snapshot in local index: %w", err)
}
for _, id := range localIDs {
if snapshot.RemoteSnapshotKey(id) == remoteKey {
return id, remoteKey, true, nil
}
}
return identifier, remoteKey, false, nil
}
// checkSnapshotExists returns errSnapshotNotFound when the local index // checkSnapshotExists returns errSnapshotNotFound when the local index
// has no row for snapshotID and, unless localOnly is set, the // has no row for snapshotID and, unless localOnly is set, the
// destination store has no metadata for it either. inLocalIndex reports // destination store has no metadata for it either. inLocalIndex reports
@@ -1674,8 +1638,8 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
return true return true
} }
// removeSnapshotRemote strips the snapshot's metadata, under remoteKey, // removeSnapshotRemote strips the snapshot's metadata from the
// from the destination store. It returns true only when the delete finished // destination store. It returns true only when the delete finished
// without an error and removed at least one object; under LocalOnly it // without an error and removed at least one object; under LocalOnly it
// does nothing. markerErr is the result of checkDestinationMarker; when // does nothing. markerErr is the result of checkDestinationMarker; when
// it is set, nothing is deleted and it is the failure. When the snapshot // it is set, nothing is deleted and it is the failure. When the snapshot
@@ -1687,7 +1651,7 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
// (`vaultik prune` never removes snapshot metadata that has its // (`vaultik prune` never removes snapshot metadata that has its
// manifest). // manifest).
func (v *Vaultik) removeSnapshotRemote( func (v *Vaultik) removeSnapshotRemote(
snapshotID, remoteKey string, inLocalIndex bool, markerErr error, snapshotID string, inLocalIndex bool, markerErr error,
opts *RemoveOptions, opts *RemoveOptions,
) (bool, error) { ) (bool, error) {
if opts.LocalOnly { if opts.LocalOnly {
@@ -1701,7 +1665,8 @@ func (v *Vaultik) removeSnapshotRemote(
err := markerErr err := markerErr
if err == nil { if err == nil {
deleted, err = v.deleteRemoteSnapshotByKey(remoteKey) deleted, err = v.deleteRemoteSnapshotByKey(
snapshot.RemoteSnapshotKey(snapshotID))
} }
if err != nil && !inLocalIndex { if err != nil && !inLocalIndex {
-5
View File
@@ -163,11 +163,6 @@ func (v *Vaultik) StartOperation(fn func()) func(context.Context) bool {
} }
} }
// CanDecrypt returns true if this Vaultik instance has decryption capabilities
func (v *Vaultik) CanDecrypt() bool {
return v.Config.AgeSecretKey != ""
}
// GetFilesystem returns the filesystem instance used by Vaultik // GetFilesystem returns the filesystem instance used by Vaultik
// //
//nolint:ireturn // afero.Fs is the filesystem abstraction by design //nolint:ireturn // afero.Fs is the filesystem abstraction by design
+3 -9
View File
@@ -22,9 +22,7 @@ import (
// Sentinel errors for snapshot verification failures. // Sentinel errors for snapshot verification failures.
var ( var (
errVerificationFailed = errors.New("verification failed") errVerificationFailed = errors.New("verification failed")
errSecretKeyRequired = errors.New(
"VAULTIK_AGE_SECRET_KEY not set; required for deep verification")
errChunksOutOfOrder = errors.New("chunks out of order") errChunksOutOfOrder = errors.New("chunks out of order")
errChunkHashMismatch = errors.New("chunk hash mismatch") errChunkHashMismatch = errors.New("chunk hash mismatch")
errNegativeChunkLength = errors.New("chunk length is negative") errNegativeChunkLength = errors.New("chunk length is negative")
@@ -106,13 +104,9 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
Mode: "deep", Mode: "deep",
} }
if !v.CanDecrypt() {
return v.deepVerifyFailure(result, opts,
errSecretKeyRequired.Error(), errSecretKeyRequired)
}
// Parse the age secret key once, the same way restore does, and reuse // Parse the age secret key once, the same way restore does, and reuse
// the identities for the database and every blob. // the identities for the database and every blob. A missing key fails
// here, before anything is downloaded.
identities, err := v.restoreIdentities() identities, err := v.restoreIdentities()
if err != nil { if err != nil {
return v.deepVerifyFailure(result, opts, err.Error(), err) return v.deepVerifyFailure(result, opts, err.Error(), err)