Compare commits
6
Commits
fe612c8b74
...
6120522fff
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6120522fff | ||
|
|
d8d13d1c8b | ||
|
|
028a166256 | ||
|
|
c07db21b45 | ||
|
|
1df1d23094 | ||
|
|
0d0c76fa24 |
No files matched your search
@@ -313,12 +313,10 @@ when the next path element is a number (so
|
||||
`vaultik config set age_recipients.0 age1...` works on a file without
|
||||
`age_recipients`), otherwise as a map. If the config loads, a change after which
|
||||
it would not, such as `vaultik config set compression_level 99`, is refused and
|
||||
the file is left unchanged. A config that does not load, such as the one
|
||||
`config init` writes before `storage_url` is set or one with a `${ENV:...}`
|
||||
reference to a variable not set in this shell, is checked one setting at a time:
|
||||
only a change after which the changed setting cannot be read, such as one giving
|
||||
it the wrong type (a single value where a list belongs or a word where a number
|
||||
belongs), is refused.
|
||||
the file is left unchanged. A config that does not load is checked one setting
|
||||
at a time. Examples are the file `config init` writes before `storage_url` is
|
||||
set, and one with a `${ENV:...}` reference to a variable not set in this shell.
|
||||
In such a config, an unknown key or a value of the wrong type is refused.
|
||||
|
||||
**`snapshot create`**: Perform incremental backup of configured snapshots.
|
||||
|
||||
|
||||
@@ -2,8 +2,10 @@ package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -120,6 +122,38 @@ func TestNewReaderTrailingAndGarbage(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestNewReaderGarbageErrorQuotesNoBytes checks that lines of text in
|
||||
// place of a blob fail with ErrNotAgeStream, and that the error carries
|
||||
// none of their bytes. The lines end in a newline because age's own error
|
||||
// quotes the data's first line only when it reads a whole line.
|
||||
func TestNewReaderGarbageErrorQuotesNoBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, _ := makeIdentity(t)
|
||||
garbage := bytes.Repeat([]byte("garbage-bytes-\n"), 100)
|
||||
|
||||
_, err := blobgen.NewReader(bytes.NewReader(garbage), id)
|
||||
|
||||
require.ErrorIs(t, err, blobgen.ErrNotAgeStream)
|
||||
assert.NotContains(t, err.Error(), "garbage-bytes-")
|
||||
}
|
||||
|
||||
// errConnectionReset stands for a storage read that fails.
|
||||
var errConnectionReset = errors.New("connection reset")
|
||||
|
||||
// TestNewReaderReadFailure checks that a failed read of the age header is
|
||||
// returned as that read error, not as ErrNotAgeStream.
|
||||
func TestNewReaderReadFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, _ := makeIdentity(t)
|
||||
|
||||
_, err := blobgen.NewReader(iotest.ErrReader(errConnectionReset), id)
|
||||
|
||||
require.ErrorIs(t, err, errConnectionReset)
|
||||
assert.NotErrorIs(t, err, blobgen.ErrNotAgeStream)
|
||||
}
|
||||
|
||||
// TestNewWriterInvalidLevel covers the rejected end of issue case 9: an
|
||||
// out-of-range compression level errors and writes nothing to the destination.
|
||||
func TestNewWriterInvalidLevel(t *testing.T) {
|
||||
|
||||
@@ -11,6 +11,11 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// ErrNotAgeStream is returned by NewReader when the data cannot be opened
|
||||
// as an age stream, for example because its header is malformed. It
|
||||
// replaces age's own error, which can quote the data's first bytes.
|
||||
var ErrNotAgeStream = errors.New("not a valid age stream")
|
||||
|
||||
// Reader wraps decompression and decryption with SHA256 verification
|
||||
type Reader struct {
|
||||
reader io.Reader
|
||||
@@ -24,11 +29,25 @@ type Reader struct {
|
||||
// NewReader creates a new Reader that decrypts, decompresses, and verifies
|
||||
// data. Every supplied identity is offered to age.Decrypt, so a blob
|
||||
// encrypted to any one of them can be read.
|
||||
//
|
||||
// A failure to read r is returned as that error, and a blob encrypted to
|
||||
// none of the identities as age.NoIdentityMatchError. Any other failure
|
||||
// to open the age stream is ErrNotAgeStream.
|
||||
func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
|
||||
// Create decryption reader
|
||||
decReader, err := age.Decrypt(r, identities...)
|
||||
src := &readErrorRecorder{reader: r}
|
||||
|
||||
decReader, err := age.Decrypt(src, identities...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
var noMatch *age.NoIdentityMatchError
|
||||
|
||||
switch {
|
||||
case src.err != nil:
|
||||
return nil, fmt.Errorf("reading age header: %w", src.err)
|
||||
case errors.As(err, &noMatch):
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
default:
|
||||
return nil, ErrNotAgeStream
|
||||
}
|
||||
}
|
||||
|
||||
// Create decompression reader
|
||||
@@ -52,6 +71,23 @@ func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readErrorRecorder passes reads through to reader and keeps the first
|
||||
// error other than io.EOF, so NewReader can tell a failed read from bad
|
||||
// data.
|
||||
type readErrorRecorder struct {
|
||||
reader io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *readErrorRecorder) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
if err != nil && !errors.Is(err, io.EOF) && r.err == nil {
|
||||
r.err = err
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.teeReader.Read(p)
|
||||
|
||||
+14
-38
@@ -443,11 +443,10 @@ number, otherwise as a map.
|
||||
|
||||
If the config loads, a change after which it would not, such as
|
||||
compression_level 99, is refused and the file is left unchanged. A
|
||||
config that does not load, such as the one config init writes before
|
||||
storage_url is set or one with a ${ENV:...} reference to a variable
|
||||
not set here, is checked one setting at a time: only a change after
|
||||
which the changed setting cannot be read, such as one giving it the
|
||||
wrong type, is refused.
|
||||
config that does not load is checked one setting at a time. Examples
|
||||
are the file config init writes before storage_url is set, and one
|
||||
with a ${ENV:...} reference to a variable not set in this shell. In
|
||||
such a config, an unknown key or a value of the wrong type is refused.
|
||||
|
||||
Examples:
|
||||
vaultik config set storage_url "file:///mnt/backups"
|
||||
@@ -484,6 +483,13 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
|
||||
|
||||
keys := strings.Split(key, ".")
|
||||
|
||||
if keys[0] != config.EnvSection {
|
||||
_, err = config.SettingType(keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, keys, value)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -739,41 +745,11 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
}
|
||||
|
||||
// configKeyIsString reports whether the dotted key names a string in
|
||||
// config.Config, following the fields' yaml tags, as s3.access_key_id and
|
||||
// snapshots.home.exclude.0 do.
|
||||
// config.Config, as s3.access_key_id and snapshots.home.exclude.0 do.
|
||||
func configKeyIsString(keys []string) bool {
|
||||
typ := reflect.TypeFor[config.Config]()
|
||||
typ, err := config.SettingType(keys)
|
||||
|
||||
for _, key := range keys {
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
|
||||
// The key is a snapshot name or a list index.
|
||||
typ = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
typ = field.Type
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return typ.Kind() == reflect.String
|
||||
}
|
||||
|
||||
// yamlField returns the field of struct type typ whose yaml tag names key.
|
||||
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
|
||||
for field := range typ.Fields() {
|
||||
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
|
||||
if name == key {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
|
||||
return reflect.StructField{}, false
|
||||
return err == nil && typ.Kind() == reflect.String
|
||||
}
|
||||
|
||||
// yamlSetInMapping resolves (creating if needed) the value node for the
|
||||
|
||||
@@ -225,7 +225,7 @@ func TestConfigSetIndexUnderMissingKey(t *testing.T) {
|
||||
// change after which the config would not load, with an error giving the
|
||||
// reason, and leaves the file as it was. On the file `config init` writes,
|
||||
// which does not load until storage_url is set, and on one that cannot be
|
||||
// read here, only a value of the wrong type is refused.
|
||||
// read here, only an unknown key or a value of the wrong type is refused.
|
||||
func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -237,6 +237,14 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
||||
value string
|
||||
reason string
|
||||
}{
|
||||
// Keys that no setting has, on a config that loads and on one
|
||||
// that does not.
|
||||
{configWithoutRecipients, "nosuchkey", "1",
|
||||
`unknown config key "nosuchkey"`},
|
||||
{configWithoutRecipients, "snapshots.home.exlude", "/tmp",
|
||||
`unknown config key "snapshots.home.exlude"`},
|
||||
{defaultConfigText(macOS), "compression_lvl", "9",
|
||||
`unknown config key "compression_lvl"`},
|
||||
// Index 1 of the empty list the missing key gives.
|
||||
{configWithoutRecipients, "age_recipients.1", testRecipient,
|
||||
"index out of range"},
|
||||
@@ -335,6 +343,39 @@ func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetWritesEnvSection checks that config set writes a key under
|
||||
// the top-level env section, whose keys are not settings, on a config that
|
||||
// loads.
|
||||
func TestConfigSetWritesEnvSection(t *testing.T) {
|
||||
const name = "VAULTIK_TEST_CONFIG_SET_ENV"
|
||||
|
||||
// Loading the changed config sets the variable; t.Setenv restores it
|
||||
// after the test.
|
||||
t.Setenv(name, "")
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
|
||||
path, "env."+name, "value")
|
||||
if err != nil {
|
||||
t.Fatalf("config set env.%s: %v", name, err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(data), "env:\n "+name+": value\n") {
|
||||
t.Errorf("config set env.%s wrote:\n%s", name, data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetStringLooksLikeNumber sets string settings to values that
|
||||
// YAML reads as numbers or booleans when they are unquoted, and checks that
|
||||
// config.Load returns each one unchanged.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewDatabaseCommand creates the database command group
|
||||
@@ -90,12 +91,12 @@ Use --force to skip the confirmation prompt.`,
|
||||
|
||||
var confirm string
|
||||
|
||||
_, err = fmt.Scanln(&confirm)
|
||||
// A failure to read the answer means no confirmation.
|
||||
_, err = fmt.Fscanln(cmd.InOrStdin(), &confirm)
|
||||
if err != nil || confirm != "yes" {
|
||||
_, _ = fmt.Fprintln(w, "Aborted.")
|
||||
|
||||
//nolint:nilerr // a failed/aborted confirmation is a clean abort
|
||||
return nil
|
||||
return vaultik.ErrNotConfirmed
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package cli //nolint:testpackage // sets the unexported rootFlags directly
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestDatabaseDeleteDeclinedIsError checks that answering anything but
|
||||
// "yes" keeps the database and returns ErrNotConfirmed, so the command
|
||||
// exits non-zero.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestDatabaseDeleteDeclinedIsError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath, cfgPath := seedDatabaseDeleteConfig(t, dir)
|
||||
|
||||
setRootFlags(t, RootFlags{ConfigPath: cfgPath})
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
cmd := newDatabaseDeleteCommand()
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(io.Discard)
|
||||
cmd.SetIn(strings.NewReader("no\n"))
|
||||
cmd.SetArgs(nil)
|
||||
|
||||
err := cmd.Execute()
|
||||
if !errors.Is(err, vaultik.ErrNotConfirmed) {
|
||||
t.Errorf("declined database delete returned %v, want %v",
|
||||
err, vaultik.ErrNotConfirmed)
|
||||
}
|
||||
|
||||
_, err = os.Stat(dbPath)
|
||||
if err != nil {
|
||||
t.Errorf("declined database delete removed the file: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(out.String(), "Aborted.") {
|
||||
t.Errorf("declined database delete printed %q, want Aborted.",
|
||||
out.String())
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -286,15 +287,37 @@ func Load(path string) (*Config, error) {
|
||||
}
|
||||
|
||||
// Parse reads YAML config data as Load does, without validating it. It
|
||||
// applies default values for optional fields, fills in ${...} references
|
||||
// using smartconfig, and expands ~ in paths.
|
||||
// refuses a key that no setting has, applies default values for optional
|
||||
// fields, fills in ${...} references using smartconfig, and expands ~ in
|
||||
// paths.
|
||||
func Parse(data []byte) (*Config, error) {
|
||||
// smartconfig drops line numbers, so keys are checked in the data as
|
||||
// written.
|
||||
var root yaml.Node
|
||||
|
||||
err := yaml.Unmarshal(data, &root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
err = checkKeys(&root, reflect.TypeFor[Config](), "",
|
||||
map[*yaml.Node]bool{}, map[nodeAndType]bool{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Load config using smartconfig for interpolation
|
||||
sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load config file: %w", err)
|
||||
}
|
||||
|
||||
// A ${...} reference can bring in keys that are not in the file.
|
||||
err = checkFilledKeys(sc.Data())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
// Set defaults
|
||||
BlobSizeLimit: defaultBlobSizeLimit,
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// yamlMergeTag is the tag YAML gives a merge key (<<), which adds the keys
|
||||
// of the map it names to the map it sits in.
|
||||
const yamlMergeTag = "!!merge"
|
||||
|
||||
// EnvSection is the top-level key of a map of environment variables that
|
||||
// smartconfig sets while it loads the config. It is not a vaultik setting,
|
||||
// so its keys are not checked.
|
||||
const EnvSection = "env"
|
||||
|
||||
var (
|
||||
errUnknownKey = errors.New("unknown config key")
|
||||
errAnchorContainsItself = errors.New("anchor contains itself")
|
||||
)
|
||||
|
||||
// nodeAndType is a node of a parsed config and the setting type it is
|
||||
// checked against.
|
||||
type nodeAndType struct {
|
||||
node *yaml.Node
|
||||
typ reflect.Type
|
||||
}
|
||||
|
||||
// SettingType returns the type of the setting that the dotted key names in
|
||||
// Config, following the fields' yaml tags, as for s3.bucket or
|
||||
// snapshots.home.exclude.0. A snapshot name or a list index may be any
|
||||
// word. It returns an unknown key error when no setting has that key.
|
||||
func SettingType(keys []string) (reflect.Type, error) {
|
||||
typ := reflect.TypeFor[Config]()
|
||||
|
||||
for _, key := range keys {
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
|
||||
typ = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w %q", errUnknownKey, strings.Join(keys, "."))
|
||||
}
|
||||
|
||||
typ = field.Type
|
||||
default:
|
||||
return nil, fmt.Errorf("%w %q", errUnknownKey, strings.Join(keys, "."))
|
||||
}
|
||||
}
|
||||
|
||||
return typ, nil
|
||||
}
|
||||
|
||||
// yamlField returns the field of struct type typ whose yaml tag names key.
|
||||
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
|
||||
for field := range typ.Fields() {
|
||||
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
|
||||
if name == key && name != "-" {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
|
||||
return reflect.StructField{}, false
|
||||
}
|
||||
|
||||
// checkKeys returns an error naming the first key in node, a parsed config
|
||||
// or part of one at the dotted path, that no setting in typ has. The error
|
||||
// gives the key's line in the file, when it has one. A value of the wrong
|
||||
// type is left for the YAML decoder to report. following holds the aliases
|
||||
// the walk is inside. Meeting one of them again means its anchor contains
|
||||
// itself, which is an error. checked holds each anchored node the walk has
|
||||
// checked completely, with the type it was checked against. An alias to
|
||||
// that node is not followed again for that type.
|
||||
func checkKeys(node *yaml.Node, typ reflect.Type, path string,
|
||||
following map[*yaml.Node]bool, checked map[nodeAndType]bool,
|
||||
) error {
|
||||
switch node.Kind {
|
||||
case yaml.DocumentNode:
|
||||
for _, child := range node.Content {
|
||||
err := checkKeys(child, typ, path, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.AliasNode:
|
||||
if following[node] {
|
||||
return fmt.Errorf("%w: %q on line %d",
|
||||
errAnchorContainsItself, node.Value, node.Line)
|
||||
}
|
||||
|
||||
target := nodeAndType{node.Alias, typ}
|
||||
if checked[target] {
|
||||
return nil
|
||||
}
|
||||
|
||||
following[node] = true
|
||||
defer delete(following, node)
|
||||
|
||||
err := checkKeys(node.Alias, typ, path, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
checked[target] = true
|
||||
case yaml.SequenceNode:
|
||||
if typ.Kind() != reflect.Slice {
|
||||
return nil
|
||||
}
|
||||
|
||||
for i, child := range node.Content {
|
||||
err := checkKeys(child, typ.Elem(), joinKey(path, strconv.Itoa(i)),
|
||||
following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case yaml.MappingNode:
|
||||
return checkMappingKeys(node, typ, path, following, checked)
|
||||
case yaml.ScalarNode:
|
||||
// A scalar has no keys.
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkFilledKeys is checkKeys for data, the config after smartconfig has
|
||||
// filled in its ${...} references. A key that a reference brings in is not
|
||||
// in the file, and the copy that Encode makes has no line numbers.
|
||||
func checkFilledKeys(data map[string]any) error {
|
||||
var filled yaml.Node
|
||||
|
||||
err := filled.Encode(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal config data: %w", err)
|
||||
}
|
||||
|
||||
return checkKeys(&filled, reflect.TypeFor[Config](), "",
|
||||
map[*yaml.Node]bool{}, map[nodeAndType]bool{})
|
||||
}
|
||||
|
||||
// checkMappingKeys is checkKeys for a map.
|
||||
func checkMappingKeys(node *yaml.Node, typ reflect.Type, path string,
|
||||
following map[*yaml.Node]bool, checked map[nodeAndType]bool,
|
||||
) error {
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
key, value := node.Content[i], node.Content[i+1]
|
||||
|
||||
if key.Tag == yamlMergeTag {
|
||||
// The value is one map or a list of maps.
|
||||
merged := []*yaml.Node{value}
|
||||
if value.Kind == yaml.SequenceNode {
|
||||
merged = value.Content
|
||||
}
|
||||
|
||||
for _, m := range merged {
|
||||
err := checkKeys(m, typ, path, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
keyPath := joinKey(path, key.Value)
|
||||
if keyPath == EnvSection {
|
||||
continue
|
||||
}
|
||||
|
||||
var valueType reflect.Type
|
||||
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map:
|
||||
valueType = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key.Value)
|
||||
if !ok && key.Line == 0 {
|
||||
// A key that a ${...} reference brought in has no line.
|
||||
return fmt.Errorf("%w %q", errUnknownKey, keyPath)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return fmt.Errorf("%w %q on line %d",
|
||||
errUnknownKey, keyPath, key.Line)
|
||||
}
|
||||
|
||||
valueType = field.Type
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
err := checkKeys(value, valueType, keyPath, following, checked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// joinKey appends key to the dotted path.
|
||||
func joinKey(path, key string) string {
|
||||
if path == "" {
|
||||
return key
|
||||
}
|
||||
|
||||
return path + "." + key
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package config //nolint:testpackage // checks the unexported errUnknownKey
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// keysTestConfig loads; each case below adds one line to it.
|
||||
const keysTestConfig = "storage_url: file:///tmp/vaultik-test-storage\n" +
|
||||
"s3:\n" +
|
||||
" region: us-east-1\n" +
|
||||
"snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n"
|
||||
|
||||
// TestParseRefusesUnknownKey checks that a key no setting has, at any
|
||||
// depth, fails the parse with an error naming the key and its line. A key
|
||||
// that a ${...} reference brings in is not in the file, so its error gives
|
||||
// no line.
|
||||
func TestParseRefusesUnknownKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
snapshotPath := filepath.Join(t.TempDir(), "snapshot.yml")
|
||||
|
||||
err := os.WriteFile(snapshotPath, []byte("other:\n"+
|
||||
" paths: [/tmp/vaultik-test-source]\n"+
|
||||
" exlude: [/tmp/cache]\n"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write snapshot: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{"top level", keysTestConfig + "compression_lvl: 9\n",
|
||||
`unknown config key "compression_lvl" on line 7`},
|
||||
{"in a snapshot", keysTestConfig + " exlude: [/tmp/cache]\n",
|
||||
`unknown config key "snapshots.test.exlude" on line 7`},
|
||||
{"in s3", strings.Replace(keysTestConfig, " region:",
|
||||
" bukcet: b\n region:", 1),
|
||||
`unknown config key "s3.bukcet" on line 3`},
|
||||
{"from a reference",
|
||||
keysTestConfig + " other: ${YAML:" + snapshotPath + ":other}\n",
|
||||
`unknown config key "snapshots.other.exlude"`},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := Parse([]byte(tt.yaml))
|
||||
if !errors.Is(err, errUnknownKey) {
|
||||
t.Fatalf("Parse() error = %v, want errUnknownKey", err)
|
||||
}
|
||||
|
||||
if err.Error() != tt.want {
|
||||
t.Errorf("Parse() error = %q, want %q", err.Error(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRefusesAnchorThatContainsItself checks that a YAML merge key (<<)
|
||||
// naming the anchor of the map it sits in fails the parse with an error.
|
||||
func TestParseRefusesAnchorThatContainsItself(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := strings.Replace(keysTestConfig, " test:", " test: &test", 1) +
|
||||
" <<: *test\n"
|
||||
|
||||
_, err := Parse([]byte(data))
|
||||
if !errors.Is(err, errAnchorContainsItself) {
|
||||
t.Fatalf("Parse() error = %v, want errAnchorContainsItself", err)
|
||||
}
|
||||
|
||||
want := `anchor contains itself: "test" on line 7`
|
||||
if err.Error() != want {
|
||||
t.Errorf("Parse() error = %q, want %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseRefusesExcessiveAliasing checks that Parse returns an error for
|
||||
// a file whose merge keys (<<) fan out. Each of eight snapshots merges the
|
||||
// one before it ten times. A key check that followed every alias would walk
|
||||
// the first snapshot over 10^8 times and run past the package's test
|
||||
// timeout.
|
||||
func TestParseRefusesExcessiveAliasing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const snapshots, merges = 8, 10
|
||||
|
||||
var data strings.Builder
|
||||
|
||||
data.WriteString(strings.Replace(keysTestConfig, " test:", " test: &s0", 1))
|
||||
|
||||
for i := 1; i <= snapshots; i++ {
|
||||
aliases := slices.Repeat([]string{"*s" + strconv.Itoa(i-1)}, merges)
|
||||
fmt.Fprintf(&data, " s%d: &s%d\n <<: [%s]\n",
|
||||
i, i, strings.Join(aliases, ", "))
|
||||
}
|
||||
|
||||
_, err := Parse([]byte(data.String()))
|
||||
if err == nil {
|
||||
t.Fatal("Parse() returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseAcceptsKnownKeys checks that a config with a ${...} reference
|
||||
// and a YAML merge key (<<) still parses.
|
||||
func TestParseAcceptsKnownKeys(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
levelPath := filepath.Join(t.TempDir(), "level")
|
||||
|
||||
err := os.WriteFile(levelPath, []byte("9"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write level: %v", err)
|
||||
}
|
||||
|
||||
data := strings.Replace(keysTestConfig, " test:", " test: &test", 1) +
|
||||
" other:\n" +
|
||||
" <<: *test\n" +
|
||||
" exclude: [/tmp/vaultik-test-source/cache]\n" +
|
||||
"compression_level: ${FILE:" + levelPath + "}\n"
|
||||
|
||||
cfg, err := Parse([]byte(data))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.CompressionLevel != 9 {
|
||||
t.Errorf("compression_level = %d, want 9", cfg.CompressionLevel)
|
||||
}
|
||||
|
||||
other := cfg.Snapshots["other"]
|
||||
if len(other.Paths) != 1 || len(other.Exclude) != 1 {
|
||||
t.Errorf("snapshot other = %+v, want the paths of test and one exclude",
|
||||
other)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseAcceptsEnvSection checks that a config with a top-level env
|
||||
// section still parses. smartconfig sets the environment variables it
|
||||
// lists, so its keys are not settings.
|
||||
func TestParseAcceptsEnvSection(t *testing.T) {
|
||||
const name = "VAULTIK_TEST_ENV_SECTION"
|
||||
|
||||
// Parse sets the variable; t.Setenv restores it after the test.
|
||||
t.Setenv(name, "")
|
||||
|
||||
_, err := Parse([]byte(keysTestConfig + "env:\n " + name + ": set\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,10 @@ var errBlobHashMismatch = errors.New("blob hash mismatch")
|
||||
var errBlobNotFullyRead = errors.New(
|
||||
"blob closed before fully read; hash not verified")
|
||||
|
||||
// errWrongPrivateKeyForBlob is returned when the configured key parses
|
||||
// but is not one of the keys a blob was encrypted to.
|
||||
var errWrongPrivateKeyForBlob = errors.New("cannot decrypt this blob")
|
||||
|
||||
// hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash
|
||||
// of decrypted plaintext when Close is called. It reuses the hash that
|
||||
// blobgen.Reader already computes internally via its TeeReader, avoiding
|
||||
@@ -87,10 +91,14 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader, err := blobgen.NewReader(rc, identities...)
|
||||
reader, err := v.newBlobReader(rc, identities)
|
||||
if err != nil {
|
||||
_ = rc.Close()
|
||||
|
||||
if errors.Is(err, errWrongPrivateKeyForBlob) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("creating blob reader: %w", err)
|
||||
}
|
||||
|
||||
@@ -102,6 +110,28 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newBlobReader opens the decrypting reader for a blob. When none of the
|
||||
// identities can decrypt it, the error says in plain words that the
|
||||
// configured private key is not one of the keys the blob was encrypted
|
||||
// to and names where the key was configured.
|
||||
func (v *Vaultik) newBlobReader(
|
||||
r io.Reader, identities []age.Identity,
|
||||
) (*blobgen.Reader, error) {
|
||||
blobReader, err := blobgen.NewReader(r, identities...)
|
||||
|
||||
if _, ok := errors.AsType[*age.NoIdentityMatchError](err); ok {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: the private key in %s is not one of the keys the blob was encrypted to",
|
||||
errWrongPrivateKeyForBlob, v.Config.AgeSecretKeySourceName())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blobReader, nil
|
||||
}
|
||||
|
||||
// FetchBlob downloads a blob and returns a reader for the encrypted data.
|
||||
// The reader is closed as soon as ctx is cancelled, which ends a read
|
||||
// blocked inside it. A storage error is returned as is: restore and deep
|
||||
|
||||
@@ -51,7 +51,7 @@ func (v *Vaultik) checkDestinationMarker() error {
|
||||
}
|
||||
|
||||
// Config is nil only in tests, which then record no binding (see
|
||||
// EnsureStorageBinding).
|
||||
// checkStorageBinding).
|
||||
if v.Config != nil {
|
||||
stored, configured, err := v.storageBinding()
|
||||
if err != nil {
|
||||
@@ -83,11 +83,19 @@ func (v *Vaultik) checkDestinationMarker() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// markDestination checks the backup destination store with
|
||||
// checkDestinationMarker and then writes the marker, so a first backup
|
||||
// marks its destination and a later one writes the same text again.
|
||||
// markDestination refuses a local index bound to a different
|
||||
// destination, checks the backup destination store for its marker,
|
||||
// writes the marker, and then records the binding. A first backup marks
|
||||
// its destination and a later one writes the same text again. The
|
||||
// binding is recorded last, so a first backup that cannot write to the
|
||||
// destination leaves the local index unbound.
|
||||
func (v *Vaultik) markDestination() error {
|
||||
err := v.checkDestinationMarker()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -99,5 +107,5 @@ func (v *Vaultik) markDestination() error {
|
||||
destinationMarkerKey, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return v.EnsureStorageBinding()
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func (v *Vaultik) NukeRemote(force bool) error {
|
||||
// the same way, so the advice below must not follow these errors.
|
||||
// The listing also fails on a destination that answers that it has
|
||||
// no marker while the local index holds snapshots.
|
||||
err := v.EnsureStorageBinding()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -192,7 +192,7 @@ type PruneBlobsResult struct {
|
||||
// PruneDatabase or PruneBlobs individually unless it specifically wants
|
||||
// one half.
|
||||
func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
err := v.EnsureStorageBinding()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -267,7 +267,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
|
||||
if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize,
|
||||
dirsWithoutManifest) {
|
||||
return nil
|
||||
return ErrNotConfirmed
|
||||
}
|
||||
|
||||
v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
|
||||
|
||||
@@ -176,7 +176,8 @@ func TestPruneDeclinedKeepsMetadataDirWithoutManifest(t *testing.T) {
|
||||
|
||||
env.v.Stdin = strings.NewReader("n\n")
|
||||
|
||||
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
|
||||
require.ErrorIs(t, env.v.PruneBlobs(&vaultik.PruneOptions{}),
|
||||
vaultik.ErrNotConfirmed)
|
||||
|
||||
assert.True(t, env.store.hasKey(leftDB),
|
||||
"a declined prune must keep the directory")
|
||||
|
||||
@@ -54,9 +54,10 @@ func TestPurgeSnapshots_QuietPromptListsSnapshots(t *testing.T) {
|
||||
v.UI.SetQuiet(true)
|
||||
|
||||
// Stdin is empty, so the prompt reads no answer and cancels.
|
||||
require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
}))
|
||||
})
|
||||
require.ErrorIs(t, err, vaultik.ErrNotConfirmed)
|
||||
|
||||
assert.Contains(t, out.String(), snapHomeT0)
|
||||
assert.Contains(t, out.String(), "Delete 1 snapshot(s)? [y/N]")
|
||||
@@ -77,7 +78,8 @@ func TestPruneBlobs_QuietPromptShowsBlobTotal(t *testing.T) {
|
||||
addBlob(t, env.store.testStorer, testBlobHashA)
|
||||
|
||||
// Stdin is empty, so the prompt reads no answer and cancels.
|
||||
require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
|
||||
require.ErrorIs(t, env.v.PruneBlobs(&vaultik.PruneOptions{}),
|
||||
vaultik.ErrNotConfirmed)
|
||||
|
||||
assert.Contains(t, env.stdout.String(),
|
||||
"Found 1 unreferenced blob(s) totaling")
|
||||
|
||||
@@ -337,6 +337,27 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) {
|
||||
assert.Contains(t, out, "vaultik prune")
|
||||
}
|
||||
|
||||
// TestRemoveSnapshot_DeclinedIsError checks that answering no to the
|
||||
// prompt keeps the snapshot and returns ErrNotConfirmed, so the command
|
||||
// exits non-zero.
|
||||
func TestRemoveSnapshot_DeclinedIsError(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", nil)
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||
tv.Stdin.WriteString("n\n")
|
||||
|
||||
_, err := tv.RemoveSnapshot("snapshot-001", &vaultik.RemoveOptions{})
|
||||
|
||||
require.ErrorIs(t, err, vaultik.ErrNotConfirmed)
|
||||
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
|
||||
assert.Contains(t, tv.Stdout.String(), "Cancelled")
|
||||
}
|
||||
|
||||
func TestRemoveSnapshot_DryRun(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
+96
-25
@@ -40,6 +40,9 @@ var (
|
||||
// which is secret, so the message is safe to print and log.
|
||||
errInvalidAgeSecretKey = errors.New(
|
||||
"configured age secret key holds no usable age identity")
|
||||
// errWrongPrivateKey is returned when the configured key parses but
|
||||
// is not one of the keys the snapshot was encrypted to.
|
||||
errWrongPrivateKey = errors.New("cannot decrypt this snapshot")
|
||||
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
|
||||
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
|
||||
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
|
||||
@@ -106,7 +109,12 @@ type RestoreResult struct {
|
||||
BytesDownloaded int64
|
||||
Duration time.Duration
|
||||
FilesFailed int
|
||||
FailedFiles []string // Paths of files that failed to restore
|
||||
FailedFiles []string // Paths of files that failed to restore for their own reason
|
||||
// Blobs that could not be downloaded, and the paths of the files
|
||||
// skipped because they need one of them. Those files are counted in
|
||||
// FilesFailed but not listed in the summary.
|
||||
BlobsFailed int
|
||||
FilesNeedingFailedBlob []string
|
||||
// Verification results (only populated if Verify option is set)
|
||||
FilesVerified int
|
||||
BytesVerified int64
|
||||
@@ -128,10 +136,19 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
"paths", opts.Paths,
|
||||
)
|
||||
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(opts.SnapshotID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Step 1: Download and decrypt the snapshot metadata database
|
||||
log.Info("Downloading snapshot metadata...")
|
||||
|
||||
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identities)
|
||||
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, remoteKey, identities)
|
||||
if errors.Is(err, errWrongPrivateKey) {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
||||
}
|
||||
@@ -226,6 +243,11 @@ func (v *Vaultik) finishRestore(
|
||||
if result.FilesFailed > 0 {
|
||||
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
|
||||
|
||||
if result.BlobsFailed > 0 {
|
||||
v.UI.Detailf("%d of them need a blob that could not be read (%d blob(s) in all).",
|
||||
len(result.FilesNeedingFailedBlob), result.BlobsFailed)
|
||||
}
|
||||
|
||||
for _, path := range result.FailedFiles {
|
||||
v.UI.Detailf("%s", v.UI.Path(path))
|
||||
}
|
||||
@@ -323,6 +345,28 @@ func (v *Vaultik) restoreIdentities() ([]age.Identity, error) {
|
||||
return identities, nil
|
||||
}
|
||||
|
||||
// newSnapshotDBReader opens the decrypting reader for a snapshot
|
||||
// database. When none of the identities can decrypt it, the error says
|
||||
// in plain words that the configured private key is the wrong one and
|
||||
// names where it was configured.
|
||||
func (v *Vaultik) newSnapshotDBReader(
|
||||
r io.Reader, identities []age.Identity,
|
||||
) (*blobgen.Reader, error) {
|
||||
blobReader, err := blobgen.NewReader(r, identities...)
|
||||
|
||||
if _, ok := errors.AsType[*age.NoIdentityMatchError](err); ok {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: the private key in %s is not one of the keys the backup was encrypted to",
|
||||
errWrongPrivateKey, v.Config.AgeSecretKeySourceName())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blobReader, nil
|
||||
}
|
||||
|
||||
// restoreAllFiles processes files in blob-locality order: drain every
|
||||
// file whose blob set is on disk, download the missing blobs for the
|
||||
// pending file with the smallest uncached count, repeat. This keeps
|
||||
@@ -494,8 +538,8 @@ func (v *Vaultik) runRestoreLoop(
|
||||
// Returns false when nothing is pending download (the caller stops).
|
||||
//
|
||||
// A blob that cannot be downloaded is reported through
|
||||
// handleRestoreFileError for every pending file that references it, so
|
||||
// it aborts the restore unless --skip-errors is set.
|
||||
// handleRestoreBlobError, so it aborts the restore unless --skip-errors
|
||||
// is set.
|
||||
func (s *restoreSession) downloadNextBlobSet(
|
||||
plan *restorePlan, filesByID map[types.FileID]*database.File,
|
||||
) (bool, error) {
|
||||
@@ -528,12 +572,11 @@ func (s *restoreSession) downloadNextBlobSet(
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, fileID := range plan.filesReferencingBlob(hash) {
|
||||
fileErr := s.v.handleRestoreFileError(
|
||||
plan, s.opts, s.result, filesByID[fileID], fileID, err)
|
||||
if fileErr != nil {
|
||||
return false, fileErr
|
||||
}
|
||||
blobErr := s.v.handleRestoreBlobError(
|
||||
plan, s.opts, s.result, filesByID,
|
||||
plan.filesReferencingBlob(hash), err)
|
||||
if blobErr != nil {
|
||||
return false, blobErr
|
||||
}
|
||||
|
||||
// next is among the failed files, so the rest of its blob set
|
||||
@@ -648,6 +691,41 @@ func (v *Vaultik) handleRestoreFileError(
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleRestoreBlobError records a blob that could not be downloaded.
|
||||
// fileIDs are the pending files that need it, never empty. Without
|
||||
// --skip-errors the first of them fails the restore. With it, the blob is
|
||||
// reported once, and its files are counted as failed and dropped from
|
||||
// the plan without being reported one by one.
|
||||
func (v *Vaultik) handleRestoreBlobError(
|
||||
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
|
||||
filesByID map[types.FileID]*database.File, fileIDs []types.FileID,
|
||||
err error,
|
||||
) error {
|
||||
if !opts.SkipErrors {
|
||||
return v.handleRestoreFileError(
|
||||
plan, opts, result, filesByID[fileIDs[0]], fileIDs[0], err)
|
||||
}
|
||||
|
||||
log.Error("Failed to restore files", "files", len(fileIDs), "error", err)
|
||||
v.UI.Errorf("Failed to restore %d file(s): %v. Skipping them (--skip-errors).",
|
||||
len(fileIDs), err)
|
||||
|
||||
result.BlobsFailed++
|
||||
|
||||
for _, fileID := range fileIDs {
|
||||
path := filesByID[fileID].Path.String()
|
||||
|
||||
log.Debug("Skipped file that needs a failed blob", "path", path)
|
||||
|
||||
result.FilesFailed++
|
||||
result.FilesNeedingFailedBlob = append(result.FilesNeedingFailedBlob, path)
|
||||
|
||||
plan.finishFile(fileID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// printRestoreProgress emits a periodic restore-phase status line via
|
||||
// the UI writer, mirroring scanner.printProcessingProgress so the two
|
||||
// long-running commands have the same on-screen rhythm.
|
||||
@@ -719,8 +797,8 @@ func (v *Vaultik) handleRestoreVerification(
|
||||
) error {
|
||||
// A file that failed to restore was not written and is already
|
||||
// reported as such, so it is not verified.
|
||||
failedToRestore := make(map[string]bool, len(result.FailedFiles))
|
||||
for _, path := range result.FailedFiles {
|
||||
failedToRestore := make(map[string]bool, result.FilesFailed)
|
||||
for _, path := range slices.Concat(result.FailedFiles, result.FilesNeedingFailedBlob) {
|
||||
failedToRestore[path] = true
|
||||
}
|
||||
|
||||
@@ -754,19 +832,12 @@ func (v *Vaultik) handleRestoreVerification(
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadSnapshotDB downloads and decrypts the snapshot metadata
|
||||
// database. The identifier is resolved to the snapshot's remote key: a
|
||||
// human ID is hashed, and a remote key (or its abbreviation, as printed
|
||||
// for a remote-only snapshot) is used as-is, so a host with no local
|
||||
// index can restore the snapshots it can only see on the store.
|
||||
// downloadSnapshotDB downloads and decrypts the metadata database of the
|
||||
// snapshot at remoteKey, which resolveSnapshotRemoteKey returned for
|
||||
// snapshotID.
|
||||
func (v *Vaultik) downloadSnapshotDB(
|
||||
snapshotID string, identities []age.Identity,
|
||||
snapshotID, remoteKey string, identities []age.Identity,
|
||||
) (*database.DB, string, error) {
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Download encrypted database from storage
|
||||
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||
|
||||
@@ -783,9 +854,9 @@ func (v *Vaultik) downloadSnapshotDB(
|
||||
// the plaintext to a temp file. Neither the encrypted bytes nor the
|
||||
// decrypted database is ever held whole in memory; a snapshot database
|
||||
// can be large.
|
||||
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||
blobReader, err := v.newSnapshotDBReader(reader, identities)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("creating decryption reader: %w", err)
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
|
||||
@@ -51,27 +51,72 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
|
||||
backup := backupThenDeleteOneBlob(ctx, t)
|
||||
out := assertRestoreSkipsLostFiles(ctx, t, backup)
|
||||
|
||||
assertLostFilesCountedNotListed(t, backup, out)
|
||||
}
|
||||
|
||||
// TestRestoreSkipErrorsReportsUnreadableBlobOnce restores with SkipErrors
|
||||
// after one blob of a two-blob snapshot was overwritten with other bytes
|
||||
// of the same size. The blob must be reported once, by its hash, as not a
|
||||
// valid age stream, without quoting its bytes, and its files must be
|
||||
// counted, not listed.
|
||||
func TestRestoreSkipErrorsReportsUnreadableBlobOnce(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
backup, lostBlob := backupThenOverwriteOneBlob(ctx, t)
|
||||
out := assertRestoreSkipsLostFiles(ctx, t, backup)
|
||||
|
||||
assert.Equal(t, 1, strings.Count(out, "not a valid age stream"), out)
|
||||
assert.Contains(t, out, lostBlob[:16])
|
||||
assert.NotContains(t, out, overwriteBlobPattern)
|
||||
assertLostFilesCountedNotListed(t, backup, out)
|
||||
}
|
||||
|
||||
// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
|
||||
// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
|
||||
// earlier backup defects left behind.
|
||||
// earlier backup defects left behind. Each such file fails for its own
|
||||
// reason, so each is listed.
|
||||
func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
|
||||
backup := backupThenDropOneBlobFromMetadata(ctx, t)
|
||||
out := assertRestoreSkipsLostFiles(ctx, t, backup)
|
||||
|
||||
for path := range backup.lost {
|
||||
assert.Containsf(t, out, path, "%s must be reported", path)
|
||||
}
|
||||
}
|
||||
|
||||
// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
|
||||
// stored in the lost blob must be reported as failed and left absent,
|
||||
// every other file must be restored intact, and Restore must still return
|
||||
// an error.
|
||||
// assertLostFilesCountedNotListed checks that the restore output out
|
||||
// counts the files of the lost blob and names none of them.
|
||||
func assertLostFilesCountedNotListed(
|
||||
t *testing.T, backup *missingBlobBackup, out string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
assert.Contains(t, out, fmt.Sprintf(
|
||||
"%d of them need a blob that could not be read (1 blob(s) in all)",
|
||||
len(backup.lost)))
|
||||
|
||||
for path := range backup.lost {
|
||||
assert.NotContainsf(t, out, path,
|
||||
"%s needs the lost blob and must be counted, not listed", path)
|
||||
}
|
||||
}
|
||||
|
||||
// assertRestoreSkipsLostFiles restores backup with SkipErrors and returns
|
||||
// what it printed. Every file stored in the lost blob must be left
|
||||
// absent, every other file must be restored intact, and Restore must
|
||||
// still return an error.
|
||||
func assertRestoreSkipsLostFiles(
|
||||
ctx context.Context, t *testing.T, backup *missingBlobBackup,
|
||||
) {
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
var out bytes.Buffer
|
||||
@@ -93,8 +138,6 @@ func assertRestoreSkipsLostFiles(
|
||||
restored := filepath.Join(backup.restoreDir, path)
|
||||
|
||||
if backup.lost[path] {
|
||||
assert.Containsf(t, out.String(), path,
|
||||
"%s needs the lost blob and must be reported", path)
|
||||
assert.NoFileExists(t, restored)
|
||||
|
||||
continue
|
||||
@@ -104,6 +147,8 @@ func assertRestoreSkipsLostFiles(
|
||||
require.NoErrorf(t, err, "%s does not need the lost blob", path)
|
||||
assert.Equalf(t, content, got, "%s restored with wrong content", path)
|
||||
}
|
||||
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with
|
||||
@@ -144,9 +189,9 @@ func TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles(t *testing.T) {
|
||||
// TestRestoreSkipErrorsVerifyReportsFailuresApart restores with
|
||||
// SkipErrors and Verify after one blob of a two-blob snapshot was
|
||||
// deleted, while one file that does restore is written one byte too long.
|
||||
// The restore warning must list only the files of the deleted blob, the
|
||||
// verification summary only the long file, each path once, and Restore
|
||||
// must return an error.
|
||||
// The restore warning must count the files of the deleted blob without
|
||||
// listing them, the verification summary must list only the long file,
|
||||
// once, and Restore must return an error.
|
||||
func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
@@ -186,9 +231,9 @@ func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
|
||||
"Verification failed: 1 files did not match")
|
||||
require.Truef(t, found, "no verification summary in:\n%s", out.String())
|
||||
|
||||
assertLostFilesCountedNotListed(t, backup, warning)
|
||||
|
||||
for path := range backup.lost {
|
||||
assert.Equalf(t, 1, strings.Count(warning, path),
|
||||
"%s must be listed once as failed to restore", path)
|
||||
assert.NotContainsf(t, summary, path,
|
||||
"%s was not restored and must not fail verification", path)
|
||||
}
|
||||
@@ -292,6 +337,35 @@ func backupThenDeleteOneBlob(
|
||||
return backup
|
||||
}
|
||||
|
||||
// overwriteBlobPattern repeats, each time followed by a newline, through
|
||||
// the bytes that backupThenOverwriteOneBlob writes over a blob. age's own
|
||||
// error quotes the data's first line only when it reads a whole line.
|
||||
const overwriteBlobPattern = "overwritten-blob-bytes-"
|
||||
|
||||
// backupThenOverwriteOneBlob backs up missingBlobFileCount single-chunk
|
||||
// files, then overwrites the blob holding the first of them with as many
|
||||
// bytes of overwriteBlobPattern lines. It also returns that blob's hash.
|
||||
func backupThenOverwriteOneBlob(
|
||||
ctx context.Context, t *testing.T,
|
||||
) (*missingBlobBackup, string) {
|
||||
t.Helper()
|
||||
|
||||
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
key := fmt.Sprintf("blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)
|
||||
|
||||
info, err := backup.storer.Stat(ctx, key)
|
||||
require.NoError(t, err)
|
||||
|
||||
line := overwriteBlobPattern + "\n"
|
||||
pattern := bytes.Repeat([]byte(line), int(info.Size)/len(line)+1)
|
||||
require.NoError(t, backup.storer.Put(
|
||||
ctx, key, bytes.NewReader(pattern[:info.Size])))
|
||||
|
||||
return backup, lostBlob
|
||||
}
|
||||
|
||||
// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
|
||||
// single-chunk files, then removes the blob holding the first of them from
|
||||
// the snapshot and exports the snapshot metadata again. The exported
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestWrongPrivateKeyError checks that restore and deep verify, given a
|
||||
// valid private key the backup was not encrypted to, fail with one plain
|
||||
// sentence that names where the key was configured.
|
||||
func TestWrongPrivateKeyError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
store := NewMockStorer()
|
||||
|
||||
const snapshotID = "test-host_2026-01-01T00:00:00Z"
|
||||
|
||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
||||
|
||||
manifest, err := snapshot.EncodeManifest(
|
||||
&snapshot.Manifest{SnapshotID: remoteKey}, 3)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, store.Put(ctx,
|
||||
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(manifest)))
|
||||
|
||||
// The database is encrypted to the test recipient only.
|
||||
var db bytes.Buffer
|
||||
|
||||
writer, err := blobgen.NewWriter(&db, 1, []string{testAgePublicKey})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write([]byte("snapshot database"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
require.NoError(t, store.Put(ctx,
|
||||
"metadata/"+remoteKey+"/db.zst.age", &db))
|
||||
|
||||
otherKey, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
v := &vaultik.Vaultik{
|
||||
Config: &config.Config{AgeSecretKey: otherKey.String()},
|
||||
Storage: store,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
// A Config built directly has no recorded source, so the error names
|
||||
// the config-file field.
|
||||
const want = "cannot decrypt this snapshot: the private key in " +
|
||||
"age_secret_key is not one of the keys the backup was encrypted to"
|
||||
|
||||
err = v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: t.TempDir(),
|
||||
})
|
||||
require.EqualError(t, err, want)
|
||||
|
||||
err = v.RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true})
|
||||
require.EqualError(t, err, want)
|
||||
}
|
||||
|
||||
// TestWrongPrivateKeyForBlobError backs up a file, then backs it up again
|
||||
// from an empty local index with a second recipient added. The second
|
||||
// snapshot reuses the stored blob, which stays encrypted to the first
|
||||
// recipient only. With only the added key configured, restore and deep
|
||||
// verify read the database but fail on the blob with one plain sentence
|
||||
// that names where the key was configured.
|
||||
func TestWrongPrivateKeyForBlobError(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
dataPath := filepath.Join(dataDir, "data.bin")
|
||||
storeDir := t.TempDir()
|
||||
secondDBPath := filepath.Join(tempDir, "second.sqlite")
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
maxBlobSize := int64(512 * 1024)
|
||||
|
||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, dataPath,
|
||||
bytesPattern("key-", int(chunkSize)), 0o644))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
|
||||
filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
|
||||
|
||||
env := setupDedupBackupEnv(
|
||||
ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
|
||||
|
||||
defer func() { _ = env.db.Close() }()
|
||||
|
||||
addedKey, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
|
||||
addedKey.Recipient().String())
|
||||
|
||||
snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
|
||||
env.cfg.Hostname, "second", dataDir, secondDBPath)
|
||||
require.Zero(t, result.BlobsUploaded,
|
||||
"the second backup should reuse the stored blob")
|
||||
|
||||
require.NoError(t, env.db.Close())
|
||||
|
||||
blobKeys, err := env.storer.List(ctx, "blobs/")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, blobKeys, 1)
|
||||
|
||||
blobHash := path.Base(blobKeys[0])
|
||||
|
||||
env.cfg.AgeSecretKey = addedKey.String()
|
||||
|
||||
v := &vaultik.Vaultik{
|
||||
Config: env.cfg,
|
||||
Storage: env.storer,
|
||||
Fs: fs,
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
const sentence = "cannot decrypt this blob: the private key in " +
|
||||
"age_secret_key is not one of the keys the blob was encrypted to"
|
||||
|
||||
// Restore names the blob by the first 16 characters of its hash.
|
||||
err = v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: t.TempDir(),
|
||||
})
|
||||
require.EqualError(t, err, "restoring "+dataPath+": downloading blob "+
|
||||
blobHash[:16]+": "+sentence+
|
||||
" (pass --skip-errors to continue past restore failures)")
|
||||
|
||||
err = v.RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true})
|
||||
require.EqualError(t, err,
|
||||
"blob "+blobHash+" verification failed: "+sentence)
|
||||
}
|
||||
@@ -115,14 +115,9 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
"index_path", v.Config.IndexPath,
|
||||
)
|
||||
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Before PruneDatabase: it drops incomplete snapshots, which could
|
||||
// leave the local index without one and so skip the marker check.
|
||||
err = v.markDestination()
|
||||
err := v.markDestination()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -749,7 +744,7 @@ type SnapshotPurgeOptions struct {
|
||||
// snapshot name, not the latest globally. This prevents `home` and `system`
|
||||
// snapshots from cannibalizing each other.
|
||||
func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
|
||||
err := v.EnsureStorageBinding()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -881,17 +876,12 @@ func (v *Vaultik) confirmAndExecutePurge(
|
||||
|
||||
var confirm string
|
||||
|
||||
// A failure to read the answer means no confirmation.
|
||||
_, err := v.scanStdin(&confirm)
|
||||
if err != nil {
|
||||
if err != nil || strings.ToLower(confirm) != "y" {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return nil //nolint:nilerr // treat EOF or read error as "no"
|
||||
}
|
||||
|
||||
if strings.ToLower(confirm) != "y" {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return nil
|
||||
return ErrNotConfirmed
|
||||
}
|
||||
} else if !quiet {
|
||||
v.UI.Infof("The following snapshots will be deleted:")
|
||||
@@ -969,25 +959,22 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
||||
Mode: "shallow",
|
||||
}
|
||||
|
||||
v.printVerifyHeader(snapshotID, opts)
|
||||
|
||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||
// hashed; a remote key (or its abbreviation, as printed for a
|
||||
// remote-only snapshot) is used as-is, so a host with no local index
|
||||
// can verify a snapshot it can only see on the store. The key is kept
|
||||
// so we can also check for the snapshot's encrypted database below.
|
||||
// The key is kept so we can also check for the snapshot's encrypted
|
||||
// database below.
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
if opts.JSON {
|
||||
result.Status = verifyStatusFailed
|
||||
result.ErrorMessage = fmt.Sprintf("resolving snapshot identifier: %v", err)
|
||||
result.ErrorMessage = err.Error()
|
||||
|
||||
return v.outputVerifyJSON(result)
|
||||
}
|
||||
|
||||
return fmt.Errorf("resolving snapshot identifier: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
v.printVerifyHeader(snapshotID, opts)
|
||||
|
||||
manifest, err := v.downloadManifestByKey(remoteKey)
|
||||
if err != nil {
|
||||
if opts.JSON {
|
||||
@@ -1285,7 +1272,7 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
|
||||
// jq` fail (issue #108). The narration is duplicated as log records,
|
||||
// which go to stderr and so cannot corrupt the document.
|
||||
func (v *Vaultik) CleanupLocalSnapshots(opts *PruneOptions) error {
|
||||
err := v.EnsureStorageBinding()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1475,12 +1462,12 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
SnapshotID: snapshotID,
|
||||
}
|
||||
|
||||
err := v.EnsureStorageBinding()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
inLocalIndex, err := v.findSnapshotToRemove(snapshotID, opts)
|
||||
inLocalIndex, err := v.checkSnapshotExists(snapshotID, opts.LocalOnly)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -1490,7 +1477,7 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
}
|
||||
|
||||
if !v.confirmRemoveSnapshot(snapshotID, opts) {
|
||||
return result, nil
|
||||
return result, ErrNotConfirmed
|
||||
}
|
||||
|
||||
// Checked before the local record is deleted: it may be the last
|
||||
@@ -1543,17 +1530,20 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// findSnapshotToRemove reports whether the local index has a row for
|
||||
// snapshotID, and returns errSnapshotNotFound when there is nothing to
|
||||
// remove: no row and, unless LocalOnly is set, no metadata on the
|
||||
// destination store either. The destination store is consulted only
|
||||
// when the local index has no row, so a snapshot in the local index can
|
||||
// still be removed from it while the destination store is unreachable,
|
||||
// and only after checkDestinationMarker passes, so an unplugged volume's
|
||||
// empty mount point is not taken for a store without the snapshot.
|
||||
func (v *Vaultik) findSnapshotToRemove(
|
||||
snapshotID string, opts *RemoveOptions,
|
||||
) (bool, error) {
|
||||
// checkSnapshotExists returns errSnapshotNotFound when the local index
|
||||
// has no row for snapshotID and, unless localOnly is set, the
|
||||
// destination store has no metadata for it either. inLocalIndex reports
|
||||
// whether the local index has that row. The destination store is
|
||||
// consulted only when the local index has no row, so a snapshot in the
|
||||
// local index can still be removed from it while the destination store
|
||||
// is unreachable, and only after checkDestinationMarker passes, so an
|
||||
// unplugged volume's empty mount point is not taken for a store without
|
||||
// the snapshot.
|
||||
//
|
||||
//nolint:nonamedreturns // the result name says what the bool means
|
||||
func (v *Vaultik) checkSnapshotExists(
|
||||
snapshotID string, localOnly bool,
|
||||
) (inLocalIndex bool, err error) {
|
||||
if v.Repositories != nil {
|
||||
snap, err := v.Repositories.Snapshots.GetByID(v.ctx, snapshotID)
|
||||
if err != nil {
|
||||
@@ -1565,12 +1555,12 @@ func (v *Vaultik) findSnapshotToRemove(
|
||||
}
|
||||
}
|
||||
|
||||
if opts.LocalOnly {
|
||||
if localOnly {
|
||||
return false, fmt.Errorf("%w in the local index: %s",
|
||||
errSnapshotNotFound, snapshotID)
|
||||
}
|
||||
|
||||
err := v.checkDestinationMarker()
|
||||
err = v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -1712,7 +1702,7 @@ func (v *Vaultik) removeSnapshotRemote(
|
||||
// before it removes anything when the destination store cannot be
|
||||
// listed or is not present by checkDestinationMarker.
|
||||
func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) {
|
||||
err := v.EnsureStorageBinding()
|
||||
err := v.checkStorageBinding()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -28,8 +28,9 @@ var (
|
||||
// Two forms are accepted, matching the two things a host can know:
|
||||
//
|
||||
// - A human snapshot ID (hostname_name_timestamp), which a host holding
|
||||
// the local index has. It is hashed to its remote key; the store is
|
||||
// not consulted.
|
||||
// the local index has. It is hashed to its remote key. An ID that
|
||||
// names no snapshot in the local index or on the destination store
|
||||
// is an error (see checkSnapshotExists).
|
||||
// - A remote key, or the leading part of one, which is all a host with
|
||||
// no local index can know — it is exactly what `snapshot list` prints
|
||||
// for a remote-only snapshot (see formatRemoteOnlyID). It is resolved
|
||||
@@ -41,6 +42,11 @@ var (
|
||||
// RFC3339 timestamp).
|
||||
func (v *Vaultik) resolveSnapshotRemoteKey(identifier string) (string, error) {
|
||||
if !isRemoteKeyOrPrefix(identifier) {
|
||||
_, err := v.checkSnapshotExists(identifier, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return snapshot.RemoteSnapshotKey(identifier), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -362,8 +362,9 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
|
||||
// Without --force and with nothing on stdin the purge lists what it
|
||||
// would delete, reads no confirmation, and deletes nothing.
|
||||
env.stdout.Reset()
|
||||
require.NoError(t, env.v.PurgeSnapshotsWithOptions(
|
||||
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
|
||||
require.ErrorIs(t, env.v.PurgeSnapshotsWithOptions(
|
||||
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}),
|
||||
vaultik.ErrNotConfirmed)
|
||||
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
|
||||
wantSize, "snapshot purge")
|
||||
}
|
||||
|
||||
@@ -13,35 +13,30 @@ import (
|
||||
var errStorageBindingMismatch = errors.New(
|
||||
"local index is bound to a different backup destination")
|
||||
|
||||
// EnsureStorageBinding guarantees that the local index database is
|
||||
// bound to the currently-configured storage destination. Every mutating
|
||||
// command must call this before touching either the local index or the
|
||||
// destination store, because the two live in lockstep: the local index
|
||||
// records which chunks/blobs already exist at the destination, and
|
||||
// mismatched destination + local index produces silent corruption (the
|
||||
// scanner sees "known" chunks and skips uploads, then writes snapshots
|
||||
// whose manifests reference blobs that aren't on the new destination).
|
||||
// checkStorageBinding refuses when the local index database is bound to
|
||||
// a different destination than the configured one, with an error that
|
||||
// tells the user how to recover (revert the config, or run `vaultik
|
||||
// database delete`). Every mutating command calls it before touching
|
||||
// either the local index or the destination store, because the two live
|
||||
// in lockstep: the local index records which chunks/blobs already exist
|
||||
// at the destination, and a mismatched pair produces silent corruption
|
||||
// (the scanner sees "known" chunks and skips uploads, then writes
|
||||
// snapshots whose manifests reference blobs that aren't on the new
|
||||
// destination).
|
||||
//
|
||||
// Behaviour:
|
||||
// - On first use (empty stored value), record the configured
|
||||
// destination (storage_url, or the s3.* endpoint, bucket and
|
||||
// prefix) and log the binding.
|
||||
// - When the stored value matches the configured destination, do
|
||||
// nothing and return nil.
|
||||
// - When the two differ, refuse with an error that tells the user
|
||||
// how to recover (revert the config, or run `vaultik database
|
||||
// delete` to discard the local index and rebuild against the new
|
||||
// destination on the next backup).
|
||||
// It records nothing. Only a backup binds the local index, through
|
||||
// EnsureStorageBinding, and it does so before adding anything to it. An
|
||||
// unbound local index therefore holds nothing from any destination.
|
||||
//
|
||||
// Read-only inspection commands (remote info, snapshot list, etc.)
|
||||
// deliberately don't call this: they can be run against a bare
|
||||
// destination store without any binding state.
|
||||
func (v *Vaultik) EnsureStorageBinding() error {
|
||||
func (v *Vaultik) checkStorageBinding() error {
|
||||
if v.Repositories == nil || v.Config == nil {
|
||||
// NewForTesting builds a Vaultik with no DB or config;
|
||||
// there's nothing to bind and no binding to check. Callers
|
||||
// exercising the bind path use a real DB and populate Config
|
||||
// explicitly (see storage_bind_test.go).
|
||||
// there's no binding to check. Callers exercising the bind
|
||||
// path use a real DB and populate Config explicitly (see
|
||||
// storage_bind_test.go).
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -50,6 +45,24 @@ func (v *Vaultik) EnsureStorageBinding() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if stored == "" || stored == configured {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w\n%s",
|
||||
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
|
||||
}
|
||||
|
||||
// EnsureStorageBinding records the configured destination (storage_url,
|
||||
// or the s3.* endpoint, bucket and prefix) as the binding of an unbound
|
||||
// local index. It refuses as checkStorageBinding does when the local
|
||||
// index is bound to a different destination.
|
||||
func (v *Vaultik) EnsureStorageBinding() error {
|
||||
stored, configured, err := v.storageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if stored == "" {
|
||||
err = v.Repositories.LocalMeta.Set(
|
||||
v.ctx, database.LocalMetaKeyStorageURL, configured)
|
||||
@@ -62,12 +75,12 @@ func (v *Vaultik) EnsureStorageBinding() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if stored == configured {
|
||||
return nil
|
||||
if stored != configured {
|
||||
return fmt.Errorf("%w\n%s",
|
||||
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w\n%s",
|
||||
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
|
||||
return nil
|
||||
}
|
||||
|
||||
// storageBinding returns the destination the local index is bound to
|
||||
|
||||
@@ -2,6 +2,7 @@ package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -9,9 +10,126 @@ import (
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/storage/faultstore"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// storageBindingOf returns the destination the local index is bound to,
|
||||
// or "" when it is not bound.
|
||||
func storageBindingOf(
|
||||
ctx context.Context, t *testing.T, repos *database.Repositories,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
// TestFailedFirstBackupLeavesIndexUnbound checks that a first backup
|
||||
// the destination refuses, as a mistyped bucket would, leaves the local
|
||||
// index unbound, so a backup with the corrected destination succeeds.
|
||||
func TestFailedFirstBackupLeavesIndexUnbound(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := t.TempDir()
|
||||
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
|
||||
working := v.Storage
|
||||
|
||||
refusing := faultstore.New(working)
|
||||
refusing.OnPut = func(string) faultstore.PutAction { return faultstore.PutFail }
|
||||
refusing.OnStat = func(string) bool { return true }
|
||||
v.Storage = refusing
|
||||
v.Config.StorageURL = "file://" + filepath.Join(t.TempDir(), "typo")
|
||||
|
||||
require.ErrorIs(t, backUp(v, "first"), faultstore.ErrInjectedUpload)
|
||||
assert.Empty(t, storageBindingOf(ctx, t, repos))
|
||||
|
||||
v.Storage = working
|
||||
v.Config.StorageURL = "file://" + storeDir
|
||||
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
assert.Equal(t, "file://"+storeDir, storageBindingOf(ctx, t, repos))
|
||||
}
|
||||
|
||||
// TestBackupToOtherDestinationWritesNoMarker checks that a backup
|
||||
// refuses a local index bound to a different destination before it
|
||||
// writes the marker to the configured one.
|
||||
func TestBackupToOtherDestinationWritesNoMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
firstDir := t.TempDir()
|
||||
v, _ := newFileDestinationVaultik(ctx, t, firstDir)
|
||||
v.Config.StorageURL = "file://" + firstDir
|
||||
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
|
||||
secondDir := t.TempDir()
|
||||
secondStore, err := storage.NewFileStorer(secondDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
v.Storage = secondStore
|
||||
v.Config.StorageURL = "file://" + secondDir
|
||||
|
||||
require.ErrorContains(t, backUp(v, "first"),
|
||||
"local index is bound to a different backup destination")
|
||||
assert.NoFileExists(t, filepath.Join(secondDir, markerFile))
|
||||
}
|
||||
|
||||
// TestCommandsOtherThanBackupLeaveIndexUnbound checks that commands
|
||||
// other than a backup do not bind an unbound local index, whether or not
|
||||
// they fail. Each runs on a fresh local index.
|
||||
func TestCommandsOtherThanBackupLeaveIndexUnbound(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(v *vaultik.Vaultik) error
|
||||
}{
|
||||
{"prune", func(v *vaultik.Vaultik) error {
|
||||
return v.Prune(&vaultik.PruneOptions{Force: true})
|
||||
}},
|
||||
{"snapshot purge", func(v *vaultik.Vaultik) error {
|
||||
return v.PurgeSnapshotsWithOptions(
|
||||
&vaultik.SnapshotPurgeOptions{KeepLatest: true, Force: true})
|
||||
}},
|
||||
{"snapshot remove", func(v *vaultik.Vaultik) error {
|
||||
_, err := v.RemoveSnapshot("host_first_2026-01-02T03:04:05Z",
|
||||
&vaultik.RemoveOptions{Force: true})
|
||||
|
||||
return err
|
||||
}},
|
||||
{"snapshot remove --all", func(v *vaultik.Vaultik) error {
|
||||
_, err := v.RemoveAllSnapshots(&vaultik.RemoveOptions{Force: true})
|
||||
|
||||
return err
|
||||
}},
|
||||
{"remote nuke", func(v *vaultik.Vaultik) error {
|
||||
return v.NukeRemote(true)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos := newFileDestinationVaultik(ctx, t, t.TempDir())
|
||||
|
||||
_ = tt.run(v)
|
||||
|
||||
assert.Empty(t, storageBindingOf(ctx, t, repos))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// buildBindTestVaultik returns a minimal Vaultik wired with a real
|
||||
// in-memory DB and a config carrying the given StorageURL. Enough to
|
||||
// exercise EnsureStorageBinding without spinning up storage or fx.
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestUnknownSnapshotID checks that restore and verify, given an ID that
|
||||
// names no snapshot in the local index or on the destination store, say
|
||||
// so in the words `snapshot remove` uses and print nothing else.
|
||||
func TestUnknownSnapshotID(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
const unknownID = "host1_home_2026-01-01T00:00:00Z"
|
||||
|
||||
commands := map[string]func(v *vaultik.Vaultik) error{
|
||||
"restore": func(v *vaultik.Vaultik) error {
|
||||
return v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: unknownID,
|
||||
TargetDir: t.TempDir(),
|
||||
})
|
||||
},
|
||||
"verify": func(v *vaultik.Vaultik) error {
|
||||
return v.VerifySnapshotWithOptions(unknownID, &vaultik.VerifyOptions{})
|
||||
},
|
||||
"verify --deep": func(v *vaultik.Vaultik) error {
|
||||
return v.VerifySnapshotWithOptions(unknownID,
|
||||
&vaultik.VerifyOptions{Deep: true})
|
||||
},
|
||||
}
|
||||
|
||||
for name, run := range commands {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", nil)
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||
bytes.NewReader(nil)))
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Config = &config.Config{AgeSecretKey: testAgeSecretKey}
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||
|
||||
err := run(tv.Vaultik)
|
||||
|
||||
require.EqualError(t, err, "snapshot not found in the local "+
|
||||
"index or on the destination store: "+unknownID)
|
||||
assert.Empty(t, tv.Stdout.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package vaultik
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -184,6 +185,12 @@ func (v *Vaultik) printlnStdout(args ...any) {
|
||||
_, _ = fmt.Fprintln(v.Stdout, args...)
|
||||
}
|
||||
|
||||
// ErrNotConfirmed is returned when a confirmation prompt gets any answer
|
||||
// but the one it asks for (`y` or `Y`, or `yes` for `database delete`), or
|
||||
// stdin ends first. Nothing the prompt asked about is deleted, and the
|
||||
// command exits non-zero.
|
||||
var ErrNotConfirmed = errors.New("deletion not confirmed")
|
||||
|
||||
// scanStdin reads a line of input from stdin.
|
||||
func (v *Vaultik) scanStdin(a ...any) (int, error) {
|
||||
return fmt.Fscanln(v.Stdin, a...)
|
||||
|
||||
+21
-16
@@ -120,12 +120,17 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
|
||||
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
||||
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
|
||||
if !opts.JSON && !v.UI.Quiet() {
|
||||
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
|
||||
}
|
||||
|
||||
manifest, tempDB, dbBlobs, err := v.loadVerificationData(
|
||||
snapshotID, opts, result, identities)
|
||||
snapshotID, remoteKey, opts, result, identities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -170,21 +175,13 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadVerificationData downloads manifest, database, and blob list for verification
|
||||
// loadVerificationData downloads manifest, database, and blob list for
|
||||
// verification of the snapshot at remoteKey, which
|
||||
// resolveSnapshotRemoteKey returned for snapshotID.
|
||||
func (v *Vaultik) loadVerificationData(
|
||||
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
||||
snapshotID, remoteKey string, opts *VerifyOptions, result *VerifyResult,
|
||||
identities []age.Identity,
|
||||
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||
// hashed; a remote key (or its abbreviation, as printed for a
|
||||
// remote-only snapshot) is used as-is, so a host with no local index
|
||||
// can verify a snapshot it can only see on the store.
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
||||
fmt.Sprintf("resolving snapshot identifier: %v", err), err)
|
||||
}
|
||||
|
||||
// Download manifest. downloadManifestByKey is the single reader for
|
||||
// remote manifests; see its doc comment.
|
||||
log.Info("Downloading manifest", "remote_key", remoteKey)
|
||||
@@ -267,6 +264,10 @@ func (v *Vaultik) downloadVerifiedSnapshotDB(
|
||||
defer func() { _ = dbReader.Close() }()
|
||||
|
||||
tdb, err := v.decryptAndLoadDatabase(dbReader, identities)
|
||||
if errors.Is(err, errWrongPrivateKey) {
|
||||
return nil, v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, v.deepVerifyFailure(result, opts,
|
||||
fmt.Sprintf("failed to decrypt database: %v", err),
|
||||
@@ -351,9 +352,9 @@ func (v *Vaultik) decryptAndLoadDatabase(
|
||||
reader io.ReadCloser, identities []age.Identity,
|
||||
) (*tempDB, error) {
|
||||
// Decrypt and decompress through the shared blobgen reader.
|
||||
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||
blobReader, err := v.newSnapshotDBReader(reader, identities)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
@@ -426,7 +427,11 @@ func (v *Vaultik) verifyBlob(
|
||||
// the plaintext as it is read. A blob's hash — its remote name — is the
|
||||
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
||||
// encrypted bytes.
|
||||
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||
blobReader, err := v.newBlobReader(reader, identities)
|
||||
if errors.Is(err, errWrongPrivateKeyForBlob) {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create blob reader: %w", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user