Author SHA1 Message Date
sneak a23a8fffa3 Print a failed command's error once on stderr (closes #433)
check / check (push) Successful in 9m35s
A command that failed after startup wrote the error twice on stderr:
as an ERROR log record and as the final ERROR: line. The failure
handlers now log the failure at info level, so the record appears only
under --verbose or --debug, next to the records that led up to it.

A failed blob upload is logged at info level for the same reason. A
restore that stops on a file it cannot restore no longer logs that
file's error, which the final line carries; under --skip-errors it is
still logged.

Model: opus-5-5
2026-11-11 05:07:27 +00:00
20 changed files with 287 additions and 672 deletions

No files matched your search

-2
View File
@@ -448,8 +448,6 @@ remove the snapshot's metadata from it (`vaultik prune` does not). For a
snapshot that is only on the destination store, that failure is an error. To
wipe everything on the destination in one go, use `vaultik remote nuke --force`.
- Accepts a snapshot ID, or — for a snapshot only on the destination store — its
remote key (or an unambiguous leading part of it) as shown by `snapshot list`.
- `--local-only`: Skip remote cleanup; only touch the local index
- `--dry-run`: Show what would be deleted without deleting
- `--force`: Skip confirmation prompt
+1 -1
View File
@@ -337,7 +337,7 @@ func runVaultikApp(
},
Mode: mode,
}, op, func(err error) {
log.Error(failMsg, "error", err)
log.Info(failMsg, "error", err)
ReportErrorf("%s: %v", failMsg, err)
})
}
+1 -22
View File
@@ -49,14 +49,8 @@ var (
errWouldNotLoad = errors.New(
"vaultik would not load the config after this change, " +
"so the file was not changed")
errSetS3KeysFirst = errors.New(
"set s3.access_key_id and s3.secret_access_key first, " +
"then storage_url")
)
// storageURLKey is the config key that selects the storage backend.
const storageURLKey = "storage_url"
// macOS is the value of runtime.GOOS on macOS.
const macOS = "darwin"
@@ -519,7 +513,7 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
err = checkConfigChange(oldData, data, settingData)
if err != nil {
return wouldNotLoadError(key, err)
return fmt.Errorf("%w: %w", errWouldNotLoad, err)
}
err = os.WriteFile(path, data, configFileMode)
@@ -543,21 +537,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return nil
}
// wouldNotLoadError returns the refusal of a change to key that vaultik
// would not load for reason. When storage_url moves to s3:// before the S3
// credentials are set, it says to set them first: they can be set while
// the old storage_url is still in place.
func wouldNotLoadError(key string, reason error) error {
if key == storageURLKey &&
(errors.Is(reason, config.ErrS3KeyIDRequired) ||
errors.Is(reason, config.ErrS3SecretRequired)) {
return fmt.Errorf("%w: %w; %w", errWouldNotLoad, reason,
errSetS3KeysFirst)
}
return fmt.Errorf("%w: %w", errWouldNotLoad, reason)
}
// checkConfigChange returns the reason vaultik would not load newData, the
// config after a change, if the change brings that failure in. If oldData,
// the config before it, loads, the new one must load too. A config that
+1 -95
View File
@@ -2,7 +2,6 @@ package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet
import (
"bytes"
"errors"
"os"
"path/filepath"
"strconv"
@@ -153,21 +152,6 @@ snapshots:
- "~"
`
// testS3URL is an S3 storage_url, which needs credentials to load.
const testS3URL = "s3://bucket/prefix?endpoint=https://s3.example.com"
// configWithS3Keys loads: its storage_url is testS3URL and both S3 keys are
// set.
const configWithS3Keys = storageURLKey + ": " + testS3URL + `
s3:
access_key_id: AKIAEXAMPLE
secret_access_key: example-secret
snapshots:
home:
paths:
- "~"
`
// configWithUnsetReference loads only where VAULTIK_TEST_UNSET_SECRET is
// set, as a backup job's environment might set it. No test sets it.
const configWithUnsetReference = configWithoutRecipients + `s3:
@@ -268,20 +252,12 @@ func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
{configWithoutRecipients, "snapshots.home.paths", "/srv",
"into []string"},
{configWithoutRecipients, "chunk_size", "fast",
`chunk_size: "fast" is not a size`},
"invalid size format"},
// Values of the right type that fail validation.
{configWithoutRecipients, "compression_level", "99",
"compression_level must be between 1 and 19"},
{configWithoutRecipients, "chunk_size", "1KB",
"chunk_size must be at least 1MB"},
// An s3:// URL before its credentials names them.
{configWithoutRecipients, storageURLKey, testS3URL,
"s3.access_key_id is required for s3:// URLs; " +
"set s3.access_key_id and s3.secret_access_key first"},
{configWithoutRecipients + "s3:\n access_key_id: AKIAEXAMPLE\n",
storageURLKey, testS3URL,
"s3.secret_access_key is required for s3:// URLs; " +
"set s3.access_key_id and s3.secret_access_key first"},
// With its reference filled in, the config loads.
{levelReference, "blob_size_limit", "1MB",
"blob_size_limit must be at least"},
@@ -367,76 +343,6 @@ func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
}
}
// TestConfigSetS3KeysBeforeStorageURL checks that the order the refusal
// gives works on a config that loads with a file:// storage_url.
func TestConfigSetS3KeysBeforeStorageURL(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
for _, set := range [][2]string{
{"s3.access_key_id", "AKIAEXAMPLE"},
{"s3.secret_access_key", "example-secret"},
{storageURLKey, testS3URL},
} {
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, set[0], set[1])
if err != nil {
t.Fatalf("config set %s: %v", set[0], err)
}
}
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
if err != nil {
t.Fatalf("read config: %v", err)
}
if !strings.Contains(string(data), storageURLKey+": "+testS3URL) {
t.Errorf("config set storage_url wrote:\n%s", data)
}
}
// TestConfigSetRefusesClearingS3Key checks that clearing s3.access_key_id on
// a config with an s3:// storage_url is refused without saying to set the
// S3 keys first. That advice is only for a change to storage_url.
func TestConfigSetRefusesClearingS3Key(t *testing.T) {
t.Parallel()
const key = "s3.access_key_id"
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(configWithS3Keys), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false), path, key, "")
if !errors.Is(err, errWouldNotLoad) {
t.Errorf("clearing %s gives error %v, want one saying %q",
key, err, errWouldNotLoad)
}
if errors.Is(err, errSetS3KeysFirst) {
t.Errorf("clearing %s gives error %v, want one not saying %q",
key, err, errSetS3KeysFirst)
}
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
if err != nil {
t.Fatalf("read config: %v", err)
}
if string(data) != configWithS3Keys {
t.Errorf("clearing %s changed the file to:\n%s", key, data)
}
}
// TestConfigSetWritesEnvSection checks that config set writes a key under
// the top-level env section, whose keys are not settings, on a config that
// loads.
+4
View File
@@ -85,6 +85,10 @@ func emitStartupBanner(args []string, w io.Writer) {
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
// an error to cobra isn't an option) and anywhere else a CLI command
// must surface a failure outside the normal RunE return path.
//
// This line is the one report of the failure. A caller that also logs
// the failure logs it at info level, so the record appears only under
// --verbose or --debug and the error is not printed twice.
func ReportErrorf(format string, args ...any) {
ui.New(os.Stderr).Errorf(format, args...)
}
+205
View File
@@ -0,0 +1,205 @@
package cli //nolint:testpackage // shares the fixtures and capture helpers
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"filippo.io/age"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/ui"
)
const (
// errorLinePrefix starts the final line that reports a failed command.
errorLinePrefix = "ERROR: "
flagVerbose = "--verbose"
cmdRestore = "restore"
// blobsDir is the directory of a destination store that holds the
// blobs.
blobsDir = "blobs"
)
// TestEntryFailureIsPrintedOnce runs commands that fail after startup
// against a destination they cannot use. `info` is not among them: it
// cannot fail after startup.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntryFailureIsPrintedOnce(t *testing.T) {
for _, command := range [][]string{
{cmdSnapshot, cmdCreate},
{cmdSnapshot, cmdRemove, someSnapshotID},
{cmdSnapshot, cmdVerify, someSnapshotID},
{cmdRemote, cmdInfo},
{cmdPrune, flagJSON},
} {
for _, verbose := range []bool{false, true} {
name := fmt.Sprintf("%s verbose=%t", strings.Join(command, " "), verbose)
t.Run(name, func(t *testing.T) {
requireErrorPrintedOnce(t, verbose, append([]string{flagConfig,
writeUnusableDestinationConfig(t)}, command...)...)
})
}
}
}
// TestEntryBlobUploadFailureIsPrintedOnce backs up to a destination
// whose blobs directory is a file, so the blob upload fails.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntryBlobUploadFailureIsPrintedOnce(t *testing.T) {
for _, verbose := range []bool{false, true} {
t.Run(fmt.Sprintf("verbose=%t", verbose), func(t *testing.T) {
configPath, storeDir := writeBackupConfig(t)
require.NoError(t, os.WriteFile(
filepath.Join(storeDir, blobsDir), nil, 0o600))
requireErrorPrintedOnce(t, verbose,
flagConfig, configPath, cmdSnapshot, cmdCreate)
})
}
}
// TestEntryBlobDownloadFailureIsPrintedOnce backs up, deletes the blobs
// from the destination, and restores, so the blob download fails.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntryBlobDownloadFailureIsPrintedOnce(t *testing.T) {
for _, verbose := range []bool{false, true} {
t.Run(fmt.Sprintf("verbose=%t", verbose), func(t *testing.T) {
configPath, storeDir := writeBackupConfigWithPrivateKey(t)
code, _, _ := runEntry(t,
flagConfig, configPath, cmdSnapshot, cmdCreate)
require.Equal(t, 0, code)
require.NoError(t, os.RemoveAll(filepath.Join(storeDir, blobsDir)))
requireErrorPrintedOnce(t, verbose, flagConfig, configPath,
cmdSnapshot, cmdRestore, onlySnapshotID(t, configPath), t.TempDir())
})
}
}
// requireErrorPrintedOnce runs Entry with args, adding --verbose when
// verbose is set. The command must fail, and stderr must carry its error
// once, as the final ERROR: line. A log record carrying the same error
// appears only under --verbose.
func requireErrorPrintedOnce(t *testing.T, verbose bool, args ...string) {
t.Helper()
if verbose {
args = append(args, flagVerbose)
}
code, _, stderr := runEntry(t, args...)
require.Equal(t, 1, code)
errorLine, records := failureOutput(t, stderr)
if !verbose {
assert.Zero(t, records,
"without --verbose no log record repeats the error:\n%s", stderr)
return
}
assert.NotZero(t, records,
"--verbose must keep the log record of %q", errorLine)
}
// failureOutput requires stderr to hold exactly one ERROR: line and
// returns it, with the number of log records whose error attribute is
// the error that line reports. stderr is a pipe here, so every log
// record is a JSON object on its own line.
func failureOutput(t *testing.T, stderr string) (string, int) {
t.Helper()
var (
errorLines []string
records []map[string]any
)
for line := range strings.Lines(stderr) {
var record map[string]any
if json.Unmarshal([]byte(line), &record) == nil {
records = append(records, record)
continue
}
if strings.Contains(line, errorLinePrefix) {
errorLines = append(errorLines, strings.TrimSpace(line))
}
}
require.Len(t, errorLines, 1,
"the error must be printed once as an ERROR: line:\n%s", stderr)
// An error can run over several lines; the ERROR: line holds the
// first of them.
repeats := 0
for _, record := range records {
failure, ok := record["error"].(string)
firstLine, _, _ := strings.Cut(failure, "\n")
if ok && strings.Contains(errorLines[0], firstLine) {
repeats++
}
}
return errorLines[0], repeats
}
// writeBackupConfigWithPrivateKey builds what writeBackupConfig builds,
// with a generated key pair in place of its recipient. The config also
// holds the private key, so that restore can decrypt the backup. Returns
// the config path and the store directory.
func writeBackupConfigWithPrivateKey(t *testing.T) (string, string) {
t.Helper()
configPath, storeDir := writeBackupConfig(t)
identity, err := age.GenerateX25519Identity()
require.NoError(t, err)
out := ui.NewWithColor(io.Discard, false)
require.NoError(t, writeConfigSet(out, configPath,
"age_recipients.0", identity.Recipient().String()))
require.NoError(t, writeConfigSet(out, configPath,
"age_secret_key", identity.String()))
return configPath, storeDir
}
// onlySnapshotID returns the ID of the one snapshot in the local index
// that writeBackupConfig puts beside the config file.
func onlySnapshotID(t *testing.T, configPath string) string {
t.Helper()
ctx := context.Background()
db, err := database.New(ctx,
filepath.Join(filepath.Dir(configPath), "index.sqlite"))
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
backedUp, err := database.NewRepositories(db).Snapshots.GetOnlySnapshot(ctx)
require.NoError(t, err)
return backedUp.ID.String()
}
+1 -1
View File
@@ -39,7 +39,7 @@ func NewInfoCommand() *cobra.Command {
}, func(v *vaultik.Vaultik) error {
return v.ShowInfo()
}, func(err error) {
log.Error("Failed to show info", "error", err)
log.Info("Failed to show info", "error", err)
ReportErrorf("Failed to show info: %v", err)
})
},
+1 -1
View File
@@ -60,7 +60,7 @@ referenced.`,
}, func(v *vaultik.Vaultik) error {
return v.Prune(opts)
}, func(err error) {
log.Error("Prune operation failed", "error", err)
log.Info("Prune operation failed", "error", err)
ReportErrorf("Prune failed: %v", err)
})
},
+1 -1
View File
@@ -98,7 +98,7 @@ func newRemoteInfoCommand() *cobra.Command {
}, func(v *vaultik.Vaultik) error {
return v.RemoteInfo(jsonOutput)
}, func(err error) {
log.Error("Failed to get remote info", "error", err)
log.Info("Failed to get remote info", "error", err)
ReportErrorf("Failed to get remote info: %v", err)
})
},
+4 -8
View File
@@ -109,7 +109,7 @@ on Linux), then /etc/vaultik/config.yml.`,
return v.CreateSnapshot(opts)
}, func(err error) {
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
log.Error("Post-backup prune failed", "error", err)
log.Info("Post-backup prune failed", "error", err)
ReportErrorf("Backup completed, but %v", err)
return
@@ -118,13 +118,13 @@ on Linux), then /etc/vaultik/config.yml.`,
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
errors.Is(err, vaultik.ErrVolumeNotMounted) {
log.Error("Backup completed with an error", "error", err)
log.Info("Backup completed with an error", "error", err)
ReportErrorf("Backup completed, but %v", err)
return
}
log.Error("Snapshot creation failed", "error", err)
log.Info("Snapshot creation failed", "error", err)
ReportErrorf("Snapshot creation failed: %v", err)
})
},
@@ -258,7 +258,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
return
}
log.Error("Verification failed", "error", err)
log.Info("Verification failed", "error", err)
ReportErrorf("Verification failed: %v", err)
})
},
@@ -288,10 +288,6 @@ know about), which is what 'vaultik prune' does. The command prints the
'vaultik prune' invocation to run as a follow-up only when it removed the
snapshot's metadata from the destination store, and never under --json.
The snapshot may be named by its ID or, on a host with no local index, by
the remote key that 'snapshot list' prints for a remote-only snapshot (an
unambiguous leading part is enough).
An ID that names no snapshot in the local index or on the destination
store is an error.
+1 -1
View File
@@ -104,7 +104,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
SkipErrors: rootFlags.SkipErrors,
})
}, func(err error) {
log.Error("Restore operation failed", "error", err)
log.Info("Restore operation failed", "error", err)
ReportErrorf("Restore failed: %v", err)
})
}
+9 -16
View File
@@ -79,13 +79,8 @@ var (
"storage not configured; set storage_url or provide s3.endpoint + " +
"s3.bucket + credentials")
errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)")
)
// Validate returns these when S3 storage is configured without its
// credentials.
var (
ErrS3KeyIDRequired = errors.New("s3.access_key_id is required")
ErrS3SecretRequired = errors.New("s3.secret_access_key is required")
errS3KeyIDRequired = errors.New("s3.access_key_id is required")
errS3SecretRequired = errors.New("s3.secret_access_key is required")
)
// expandTilde expands ~ at the start of a path to the user's home directory.
@@ -295,8 +290,7 @@ func Load(path string) (*Config, error) {
}
// Parse reads YAML config data as Load does, without validating it. It
// refuses a key that no setting has and a size setting whose value is not a
// size, naming the setting. It applies default values for optional
// refuses a key that no setting has, applies default values for optional
// fields, fills in ${...} references using smartconfig, and expands ~ in
// paths.
func Parse(data []byte) (*Config, error) {
@@ -321,9 +315,8 @@ func Parse(data []byte) (*Config, error) {
return nil, fmt.Errorf("failed to load config file: %w", err)
}
// A ${...} reference can bring in keys that are not in the file, and
// is the value of a size setting only once it is filled in.
err = checkFilledConfig(sc.Data())
// A ${...} reference can bring in keys that are not in the file.
err = checkFilledKeys(sc.Data())
if err != nil {
return nil, err
}
@@ -529,11 +522,11 @@ func (c *Config) validateStorage() error {
}
if c.S3.AccessKeyID == "" {
return ErrS3KeyIDRequired
return errS3KeyIDRequired
}
if c.S3.SecretAccessKey == "" {
return ErrS3SecretRequired
return errS3SecretRequired
}
return nil
@@ -560,11 +553,11 @@ func (c *Config) validateStorageURL() error {
case strings.HasPrefix(c.StorageURL, "s3://"):
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3KeyIDRequired)
return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", ErrS3SecretRequired)
return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired)
}
return nil
+5 -11
View File
@@ -129,11 +129,10 @@ func checkKeys(node *yaml.Node, typ reflect.Type, path string,
return nil
}
// checkFilledConfig checks data, the config after smartconfig has filled in
// its ${...} references: its keys with checkKeys, then its size settings
// with checkSizes. A key that a reference brings in is not in the file, and
// the copy that Encode makes has no line numbers.
func checkFilledConfig(data map[string]any) error {
// checkFilledKeys is checkKeys for data, the config after smartconfig has
// filled in its ${...} references. A key that a reference brings in is not
// in the file, and the copy that Encode makes has no line numbers.
func checkFilledKeys(data map[string]any) error {
var filled yaml.Node
err := filled.Encode(data)
@@ -141,13 +140,8 @@ func checkFilledConfig(data map[string]any) error {
return fmt.Errorf("failed to marshal config data: %w", err)
}
err = checkKeys(&filled, reflect.TypeFor[Config](), "",
return checkKeys(&filled, reflect.TypeFor[Config](), "",
map[*yaml.Node]bool{}, map[nodeAndType]bool{})
if err != nil {
return err
}
return checkSizes(&filled, nil)
}
// checkMappingKeys is checkKeys for a map.
+11 -77
View File
@@ -4,20 +4,13 @@ import (
"errors"
"fmt"
"math"
"reflect"
"slices"
"strconv"
"strings"
"github.com/dustin/go-humanize"
"gopkg.in/yaml.v3"
)
var (
errSizeType = errors.New("size must be a number or string")
errSizeTooLarge = errors.New("size exceeds maximum supported value")
errNotASize = errors.New(
"is not a size; use a number of bytes or a unit such as 10MB")
)
// Size represents a byte size that can be specified in configuration files.
@@ -39,15 +32,6 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
return nil
}
// A number too large for an int64 decodes as a float64. Its text can be
// 1e+20, which is not a size string.
var floatVal float64
err = unmarshal(&floatVal)
if err == nil && floatVal > math.MaxInt64 {
return errSizeTooLarge
}
// Try to unmarshal as string
var strVal string
@@ -56,66 +40,17 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
return errSizeType
}
size, err := ParseSize(strVal)
// Parse the string using go-humanize
bytes, err := humanize.ParseBytes(strVal)
if err != nil {
return err
return fmt.Errorf("invalid size format: %w", err)
}
*s = size
return nil
}
// checkSizes returns an error naming the first size setting in node whose
// value is not a size. node is the config, or the part of it at the dotted
// keys, after smartconfig has filled in its ${...} references.
func checkSizes(node *yaml.Node, keys []string) error {
typ, err := SettingType(keys)
if err != nil {
// Unknown keys are refused before this check, so keys is in one of
// two places. The env section holds no settings. A value of the
// wrong shape, such as a map given for hostname or inside exclude,
// is reported later by the YAML decoder.
return nil //nolint:nilerr // not an error here
if bytes > math.MaxInt64 {
return fmt.Errorf("%w: %s", errSizeTooLarge, strVal)
}
if typ == reflect.TypeFor[Size]() {
var size Size
err = node.Decode(&size)
if err != nil {
return fmt.Errorf("%s: %w", strings.Join(keys, "."), err)
}
return nil
}
switch node.Kind {
case yaml.DocumentNode:
for _, child := range node.Content {
err = checkSizes(child, keys)
if err != nil {
return err
}
}
case yaml.MappingNode:
for i := 0; i+1 < len(node.Content); i += 2 {
err = checkSizes(node.Content[i+1],
slices.Concat(keys, []string{node.Content[i].Value}))
if err != nil {
return err
}
}
case yaml.SequenceNode:
for i, child := range node.Content {
err = checkSizes(child, slices.Concat(keys, []string{strconv.Itoa(i)}))
if err != nil {
return err
}
}
case yaml.ScalarNode, yaml.AliasNode:
// A scalar holds no settings, and the filled config has no aliases.
}
*s = Size(bytes)
return nil
}
@@ -135,17 +70,16 @@ func (s Size) String() string {
return humanize.Bytes(uint64(s))
}
// ParseSize parses a size string, such as "10MB", into a Size value. An
// error quotes s.
// ParseSize parses a size string into a Size value
func ParseSize(s string) (Size, error) {
bytes, err := humanize.ParseBigBytes(s)
bytes, err := humanize.ParseBytes(s)
if err != nil {
return 0, fmt.Errorf("%q %w", s, errNotASize)
return 0, fmt.Errorf("invalid size format: %w", err)
}
if !bytes.IsInt64() {
if bytes > math.MaxInt64 {
return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s)
}
return Size(bytes.Int64()), nil
return Size(bytes), nil
}
-120
View File
@@ -1,120 +0,0 @@
package config //nolint:testpackage // checks errNotASize and errSizeTooLarge
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
// TestParseRefusesValueThatIsNotASize checks that a size setting whose
// value is not a size fails the parse with an error naming the setting and
// quoting the value, also when a ${...} reference fills the value in.
func TestParseRefusesValueThatIsNotASize(t *testing.T) {
t.Parallel()
sizePath := filepath.Join(t.TempDir(), "size")
err := os.WriteFile(sizePath, []byte("lots"), 0o600)
if err != nil {
t.Fatalf("write size: %v", err)
}
tests := []struct {
name string
yaml string
want string
}{
{"chunk_size", keysTestConfig + "chunk_size: banana\n",
`chunk_size: "banana" is not a size; ` +
"use a number of bytes or a unit such as 10MB"},
{"blob_size_limit", keysTestConfig + "blob_size_limit: 10XB\n",
`blob_size_limit: "10XB" is not a size; ` +
"use a number of bytes or a unit such as 10MB"},
{"s3.part_size", strings.Replace(keysTestConfig, " region:",
" part_size: big\n region:", 1),
`s3.part_size: "big" is not a size; ` +
"use a number of bytes or a unit such as 10MB"},
{"from a reference",
keysTestConfig + "chunk_size: ${FILE:" + sizePath + "}\n",
`chunk_size: "lots" is not a size; ` +
"use a number of bytes or a unit such as 10MB"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
_, err := Parse([]byte(tt.yaml))
if !errors.Is(err, errNotASize) {
t.Fatalf("Parse() error = %v, want errNotASize", err)
}
if err.Error() != tt.want {
t.Errorf("Parse() error = %q, want %q", err.Error(), tt.want)
}
})
}
}
// TestParseRefusesSizeTooLarge checks that a size setting whose value is
// too large for a size fails the parse with an error naming the setting.
// YAML reads the second value as a float.
func TestParseRefusesSizeTooLarge(t *testing.T) {
t.Parallel()
tests := []struct {
value string
want string
}{
{"100EB", "chunk_size: size exceeds maximum supported value: 100EB"},
{"99999999999999999999",
"chunk_size: size exceeds maximum supported value"},
}
for _, tt := range tests {
t.Run(tt.value, func(t *testing.T) {
t.Parallel()
_, err := Parse([]byte(keysTestConfig + "chunk_size: " + tt.value + "\n"))
if !errors.Is(err, errSizeTooLarge) {
t.Fatalf("Parse() error = %v, want errSizeTooLarge", err)
}
if err.Error() != tt.want {
t.Errorf("Parse() error = %q, want %q", err.Error(), tt.want)
}
})
}
}
// TestParseAcceptsSizes checks that size settings given as a number of
// bytes, as a unit, and through a ${...} reference still parse.
func TestParseAcceptsSizes(t *testing.T) {
t.Parallel()
sizePath := filepath.Join(t.TempDir(), "size")
err := os.WriteFile(sizePath, []byte("20MB"), 0o600)
if err != nil {
t.Fatalf("write size: %v", err)
}
data := strings.Replace(keysTestConfig, " region:",
" part_size: 6000000\n region:", 1) +
"chunk_size: ${FILE:" + sizePath + "}\n" +
"blob_size_limit: 1GB\n"
cfg, err := Parse([]byte(data))
if err != nil {
t.Fatalf("Parse() unexpected error: %v", err)
}
if cfg.ChunkSize != 20_000_000 || cfg.BlobSizeLimit != 1_000_000_000 ||
cfg.S3.PartSize != 6_000_000 {
t.Errorf("chunk_size, blob_size_limit, s3.part_size = %d, %d, %d, "+
"want 20000000, 1000000000, 6000000",
cfg.ChunkSize, cfg.BlobSizeLimit, cfg.S3.PartSize)
}
}
+5 -3
View File
@@ -1875,10 +1875,12 @@ func (s *Scanner) uploadBlobIfNeeded(
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
finishedBlob.Compressed, progressCallback)
if err != nil {
// Only an interrupt cancels ctx, and the user is told about it,
// so an upload it stopped is not logged as a storage failure.
// The record is at info level: the returned error ends the backup
// and is printed as its ERROR: line. Only an interrupt cancels
// ctx, and the user is told about it, so an upload it stopped is
// not logged as a storage failure.
if ctx.Err() == nil {
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
log.Info("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
}
return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
+6 -6
View File
@@ -37,9 +37,9 @@ func uploadTestBlob(ctx context.Context, store storage.Storer) error {
}
// captureLog runs fn and returns what the logger wrote to stderr
// meanwhile. The logger writes to the os.Stderr it finds when it is
// initialized, so os.Stderr is pointed at a file first. Not
// parallel-safe: os.Stderr and the logger are process-global.
// meanwhile, at the --verbose level. The logger writes to the os.Stderr
// it finds when it is initialized, so os.Stderr is pointed at a file
// first. Not parallel-safe: os.Stderr and the logger are process-global.
func captureLog(t *testing.T, fn func()) string {
t.Helper()
@@ -53,7 +53,7 @@ func captureLog(t *testing.T, fn func()) string {
previous := os.Stderr
os.Stderr = stderrFile
log.Initialize(log.Config{})
log.Initialize(log.Config{Verbose: true})
fn()
@@ -107,10 +107,10 @@ func TestInterruptedUploadIsNotLoggedAsError(t *testing.T) {
}
}
// An upload that fails for any other reason is still logged as an error.
// An upload that fails for any other reason is still logged.
//
//nolint:paralleltest // captureLog replaces os.Stderr
func TestFailedUploadIsLoggedAsError(t *testing.T) {
func TestFailedUploadIsLogged(t *testing.T) {
inner, err := storage.NewFileStorer(t.TempDir())
if err != nil {
t.Fatal(err)
-198
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"database/sql"
"encoding/json"
"io"
"strings"
"sync"
@@ -473,171 +472,6 @@ func TestRemoveSnapshot_NotInLocalIndex(t *testing.T) {
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
}
// TestRemoveSnapshot_RemoteOnlyByRemoteKey checks that a snapshot only
// on the destination store, on a host with no local index, is removed by
// the remote key that 'snapshot list' prints for such a snapshot, and by
// an unambiguous leading part of it — the same identifiers verify and
// restore accept.
func TestRemoveSnapshot_RemoteOnlyByRemoteKey(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
identifiers := map[string]string{
"full key": remoteKey,
"leading part": remoteKey[:8],
}
for name, identifier := range identifiers {
t.Run(name, func(t *testing.T) {
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// No local index: a host that sees the snapshot only remotely.
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t)
result, err := tv.RemoveSnapshot(identifier,
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.True(t, result.RemoteRemoved)
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
out := tv.Stdout.String()
assert.NotContains(t, out, "from local database")
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
})
}
}
// TestRemoveSnapshot_LocalIndexByRemoteKey checks that a remote key (or
// its unambiguous leading part) that names a snapshot the local index
// holds removes that snapshot's local row and its remote metadata
// together. `snapshot list --json` prints a remote key for every
// snapshot, including local ones.
func TestRemoveSnapshot_LocalIndexByRemoteKey(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
identifiers := map[string]string{
"full key": remoteKey,
"leading part": remoteKey[:8],
}
for name, identifier := range identifiers {
t.Run(name, func(t *testing.T) {
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the
// marker, and the local index holds the snapshot.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
tv := vaultik.NewForTesting(store)
repos := newLocalIndex(t, "snapshot-001")
tv.Repositories = repos
result, err := tv.RemoveSnapshot(identifier,
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.True(t, result.RemoteRemoved)
assert.False(t, store.hasKey(remoteKeyPath("snapshot-001")))
row, err := repos.Snapshots.GetByID(
context.Background(), "snapshot-001")
require.NoError(t, err)
assert.Nil(t, row)
out := tv.Stdout.String()
assert.Contains(t, out,
"Removed snapshot 'snapshot-001' from local database")
assert.Contains(t, out, "Removed snapshot metadata from remote storage")
})
}
}
// TestRemoveSnapshot_DryRunByLeadingPart checks that a dry run by a
// leading part of a remote key names the snapshot that part resolved to:
// by its ID when the local index holds the snapshot, otherwise by its
// full remote key.
func TestRemoveSnapshot_DryRunByLeadingPart(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
for name, tc := range map[string]struct {
localIDs []string
shown string
}{
"in local index": {localIDs: []string{"snapshot-001"}, shown: "snapshot-001"},
"remote only": {localIDs: nil, shown: remoteKey},
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the marker.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t, tc.localIDs...)
_, err := tv.RemoveSnapshot(remoteKey[:8],
&vaultik.RemoveOptions{DryRun: true})
require.NoError(t, err)
assert.Contains(t, tv.Stdout.String(),
"Would remove snapshot: "+tc.shown+".")
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
})
}
}
// TestRemoveSnapshot_JSONDryRunByLeadingPart checks that a JSON dry run
// by a leading part of a remote key prints the ID and the full remote key
// of the snapshot that part resolved to.
func TestRemoveSnapshot_JSONDryRunByLeadingPart(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
remoteKey := snapshot.RemoteSnapshotKey("snapshot-001")
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the marker.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t, "snapshot-001")
_, err := tv.RemoveSnapshot(remoteKey[:8],
&vaultik.RemoveOptions{DryRun: true, JSON: true})
require.NoError(t, err)
var printed map[string]any
require.NoError(t, json.Unmarshal(tv.Stdout.Bytes(), &printed))
assert.Equal(t, "snapshot-001", printed["snapshot_id"])
assert.Equal(t, remoteKey, printed["remote_key"])
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
}
// TestRemoveSnapshot_NotInLocalIndexDeleteFails checks that removing a
// snapshot only the destination store holds fails when its metadata
// cannot be deleted there, under --force and under --json: nothing was
@@ -672,38 +506,6 @@ func TestRemoveSnapshot_NotInLocalIndexDeleteFails(t *testing.T) {
}
}
// TestRemoveSnapshot_LocalIndexByLeadingPartDeleteFails checks the
// warning when a snapshot the local index holds is removed by a leading
// part of its remote key and the delete on the destination store fails.
// The warning names the snapshot by its ID, not by the part given.
func TestRemoveSnapshot_LocalIndexByLeadingPartDeleteFails(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
// A backup wrote to this destination, so it carries the marker.
require.NoError(t, store.Put(context.Background(), markerFile,
bytes.NewReader(nil)))
faulty := faultstore.New(store)
faulty.OnDelete = func(string) bool { return true }
tv := vaultik.NewForTesting(faulty)
tv.Repositories = newLocalIndex(t, "snapshot-001")
result, err := tv.RemoveSnapshot(
snapshot.RemoteSnapshotKey("snapshot-001")[:8],
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.False(t, result.RemoteRemoved)
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
assert.Contains(t, tv.Stdout.String(),
"Run 'vaultik snapshot remove snapshot-001' again")
}
func TestRemoveAllSnapshots_RequiresForce(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
+3 -3
View File
@@ -667,19 +667,19 @@ func (v *Vaultik) restoreProgressTick(
// handleRestoreFileError records a per-file restore failure: fatal unless
// --skip-errors is set, in which case the file is counted as failed and
// dropped from the plan.
// dropped from the plan. A fatal failure is not logged here: the returned
// error is printed as the command's ERROR: line.
func (v *Vaultik) handleRestoreFileError(
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
file *database.File, fileID types.FileID, err error,
) error {
log.Error("Failed to restore file", "path", file.Path, "error", err)
if !opts.SkipErrors {
return fmt.Errorf(
"restoring %s: %w (pass --skip-errors to continue past "+
"restore failures)", file.Path, err)
}
log.Error("Failed to restore file", "path", file.Path, "error", err)
v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).",
v.UI.Path(file.Path.String()), err)
+28 -106
View File
@@ -1424,14 +1424,11 @@ type RemoveOptions struct {
LocalOnly bool // Skip remote cleanup; only touch the local index
}
// RemoveResult contains the result of a snapshot removal. For a single
// snapshot, SnapshotID is its ID, empty when the ID is not known, and
// RemoteKey is the full remote key of its metadata.
// RemoveResult contains the result of a snapshot removal
//
//nolint:tagliatelle // snake_case is the established JSON output format
type RemoveResult struct {
SnapshotID string `json:"snapshot_id,omitempty"`
RemoteKey string `json:"remote_key,omitempty"`
SnapshotsRemoved []string `json:"snapshots_removed,omitempty"`
RemoteRemoved bool `json:"remote_removed,omitempty"`
DryRun bool `json:"dry_run,omitempty"`
@@ -1461,35 +1458,25 @@ const snapshotRemoveCommandHint = "vaultik snapshot remove"
func (v *Vaultik) RemoveSnapshot(
snapshotID string, opts *RemoveOptions,
) (*RemoveResult, error) {
result := &RemoveResult{}
result := &RemoveResult{
SnapshotID: snapshotID,
}
err := v.checkStorageBinding()
if err != nil {
return result, err
}
inLocalIndex, resolvedID, remoteKey, err := v.findSnapshotToRemove(
snapshotID, opts.LocalOnly)
inLocalIndex, err := v.checkSnapshotExists(snapshotID, opts.LocalOnly)
if err != nil {
return result, err
}
result.SnapshotID = resolvedID
result.RemoteKey = remoteKey
// The dry run, the prompt and removeSnapshotRemote name the snapshot
// that snapshotID resolved to: by its ID, or by its full remote key
// when the ID is not known.
shown := resolvedID
if shown == "" {
shown = remoteKey
}
if opts.DryRun {
return v.removeSnapshotDryRun(result, shown, opts)
return v.removeSnapshotDryRun(result, snapshotID, opts)
}
if !v.confirmRemoveSnapshot(shown, opts) {
if !v.confirmRemoveSnapshot(snapshotID, opts) {
return result, ErrNotConfirmed
}
@@ -1502,16 +1489,16 @@ func (v *Vaultik) RemoveSnapshot(
}
if inLocalIndex {
log.Info("Removing snapshot from local database", "snapshot_id", resolvedID)
log.Info("Removing snapshot from local database", "snapshot_id", snapshotID)
err = v.deleteSnapshotFromLocalDB(resolvedID)
err = v.deleteSnapshotFromLocalDB(snapshotID)
if err != nil {
return result, fmt.Errorf("removing from local database: %w", err)
}
}
result.RemoteRemoved, err = v.removeSnapshotRemote(
shown, remoteKey, inLocalIndex, markerErr, opts)
snapshotID, inLocalIndex, markerErr, opts)
if err != nil {
return result, err
}
@@ -1530,7 +1517,7 @@ func (v *Vaultik) RemoveSnapshot(
if inLocalIndex {
v.UI.Infof("Removed snapshot '%s' from local database.",
v.UI.Snapshot(resolvedID))
v.UI.Snapshot(snapshotID))
}
if result.RemoteRemoved {
@@ -1543,52 +1530,6 @@ func (v *Vaultik) RemoveSnapshot(
return result, nil
}
// findSnapshotToRemove finds the snapshot that identifier names. It
// returns whether the local index holds that snapshot, its snapshot ID,
// and the remote key of its metadata on the destination store. The ID is
// known when identifier is a snapshot ID or the local index holds the
// snapshot; otherwise it is empty.
//
// A snapshot ID is looked up by checkSnapshotExists. Under localOnly the
// store is not consulted, so every identifier is looked up that way and
// a remote key is not found. Otherwise a remote key, or a leading part
// of one, is resolved by resolveSnapshotRemoteKey, as restore and verify
// do. That fails with errSnapshotKeyNotFound or errSnapshotKeyAmbiguous
// when the identifier matches no snapshot on the store, or more than
// one. The store is listed only after checkDestinationMarker passes, as
// in checkSnapshotExists. The local row is the one whose snapshot ID
// hashes to the resolved key, as in RemoveAllSnapshots, so removing by
// remote key removes that row too.
func (v *Vaultik) findSnapshotToRemove(
identifier string, localOnly bool,
) (bool, string, string, error) {
if localOnly || !isRemoteKeyOrPrefix(identifier) {
inLocalIndex, err := v.checkSnapshotExists(identifier, localOnly)
if err != nil {
return false, "", "", err
}
return inLocalIndex, identifier, snapshot.RemoteSnapshotKey(identifier), nil
}
err := v.checkDestinationMarker()
if err != nil {
return false, "", "", err
}
remoteKey, err := v.resolveSnapshotRemoteKey(identifier)
if err != nil {
return false, "", "", err
}
localID, err := v.localSnapshotIDForRemoteKey(remoteKey)
if err != nil {
return false, "", "", err
}
return localID != "", localID, remoteKey, nil
}
// checkSnapshotExists returns errSnapshotNotFound when the local index
// has no row for snapshotID and, unless localOnly is set, the
// destination store has no metadata for it either. inLocalIndex reports
@@ -1698,22 +1639,19 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
}
// removeSnapshotRemote strips the snapshot's metadata from the
// destination store. remoteKey is the resolved metadata directory name
// to delete, as returned by findSnapshotToRemove. shown names the
// snapshot in the log records and the warning: its ID, or its full
// remote key when the ID is not known. It returns true only when the
// delete finished without an error and removed at least one object;
// under LocalOnly it does nothing. markerErr is the result of
// checkDestinationMarker; when it is set, nothing is deleted and it is
// the failure. When the snapshot is only on the destination store, a
// failure is returned as an error, because that removal is all the
// command does. When the snapshot was in the local index, its removal
// there has already happened, so a failure is only a warning telling
// the user to run `vaultik snapshot remove` for the snapshot again once
// the destination store is reachable (`vaultik prune` never removes
// snapshot metadata that has its manifest).
// destination store. It returns true only when the delete finished
// without an error and removed at least one object; under LocalOnly it
// does nothing. markerErr is the result of checkDestinationMarker; when
// it is set, nothing is deleted and it is the failure. When the snapshot
// is only on the destination store, a failure is returned as an error,
// because that removal is all the command does. When the snapshot was in
// the local index, its removal there has already happened, so a failure
// is only a warning telling the user to run `vaultik snapshot remove`
// for the snapshot again once the destination store is reachable
// (`vaultik prune` never removes snapshot metadata that has its
// manifest).
func (v *Vaultik) removeSnapshotRemote(
shown, remoteKey string, inLocalIndex bool, markerErr error,
snapshotID string, inLocalIndex bool, markerErr error,
opts *RemoveOptions,
) (bool, error) {
if opts.LocalOnly {
@@ -1721,13 +1659,14 @@ func (v *Vaultik) removeSnapshotRemote(
}
log.Info("Removing snapshot metadata from remote storage",
"snapshot_id", shown)
"snapshot_id", snapshotID)
var deleted int
err := markerErr
if err == nil {
deleted, err = v.deleteRemoteSnapshotByKey(remoteKey)
deleted, err = v.deleteRemoteSnapshotByKey(
snapshot.RemoteSnapshotKey(snapshotID))
}
if err != nil && !inLocalIndex {
@@ -1739,14 +1678,14 @@ func (v *Vaultik) removeSnapshotRemote(
log.Warn("Could not remove snapshot metadata from remote storage; "+
"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
"again once the remote is reachable",
"snapshot_id", shown, "error", err)
"snapshot_id", snapshotID, "error", err)
// The UI writes to stdout, which under --json holds only the
// document; the log record above is the warning on stderr.
if v.UI != nil && !opts.JSON {
v.UI.Warningf("Could not remove snapshot metadata from remote: "+
"%v. Run '%s %s' again once the remote is reachable.",
err, snapshotRemoveCommandHint, shown)
err, snapshotRemoveCommandHint, snapshotID)
}
return false, nil
@@ -1841,23 +1780,6 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) {
return ids, nil
}
// localSnapshotIDForRemoteKey returns the human snapshot ID of the local
// index row whose RemoteSnapshotKey is key, or "" when no row has it.
func (v *Vaultik) localSnapshotIDForRemoteKey(key string) (string, error) {
ids, err := v.localSnapshotIDs()
if err != nil {
return "", fmt.Errorf("listing local snapshots: %w", err)
}
for _, id := range ids {
if snapshot.RemoteSnapshotKey(id) == key {
return id, nil
}
}
return "", nil
}
// listAllRemoteSnapshotKeys collects the hashed remote keys
// (subdirectories under metadata/) currently present in the
// destination store. Returns (nil, err) when the store cannot be