Refuse credentials in a storage URL's endpoint or rclone remote name (closes #322)

storage_url refused only the outer URL's userinfo and unknown query
parameters. Userinfo inside the s3 endpoint= value, and rclone's
remote,param=value parameters in the remote name, were accepted and
then printed by info, remote info and snapshot create. Both are now
refused without echoing the value: a host name never contains "@", and
an rclone remote name never contains a comma. An s3 or rclone query
that cannot be decoded (a ";" or a bad "%" escape) is refused too,
because decoding drops that part before it can be checked. The comma
is checked before url.Parse, whose error for such a name prints part of
it. The rclone remote name case has its own error pointing at rclone
config.

Model: opus-5-5

README.md

@@ -271,8 +271,17 @@ existing file. Created with mode `0600` since it will contain credentials.
 **`config set`**: Set a scalar config value by dotted YAML path
 (e.g. `vaultik config set compression_level 9`,
 `vaultik config set storage_url "file:///mnt/backups"`). Comments and
-formatting in the file are preserved; intermediate maps are created as
+formatting in the file are preserved. Missing keys along the path are
-needed.
+created: as a list when the next path element is a number (so
+`vaultik config set age_recipients.0 age1...` works on a file without
+`age_recipients`), otherwise as a map. If the config loads, a change
+after which it would not, such as `vaultik config set compression_level 99`,
+is refused and the file is left unchanged. A config that does not load,
+such as the one `config init` writes before `storage_url` is set or one
+with a `${ENV:...}` reference to a variable not set in this shell, is
+checked one setting at a time: only a change after which the changed
+setting cannot be read, such as one giving it the wrong type (a single
+value where a list belongs or a word where a number belongs), is refused.
 
 **`snapshot create`**: Perform incremental backup of configured snapshots.
 * Optional snapshot names argument to create specific snapshots (default: all)
@@ -373,7 +382,9 @@ on the destination in one go, use `vaultik remote nuke --force`.
   store — its remote key (or an unambiguous leading part of it) as shown
   by `snapshot list`. See
   [restoring on another machine](#restoring-on-another-machine).
-* Optional path arguments to restore specific files/directories (default: all)
+* Optional path arguments to restore specific files/directories (default: all).
+  A path that matches nothing in the snapshot is named in a warning, and the
+  command exits non-zero after restoring the paths that match.
 * Preserves file permissions, timestamps, ownership (ownership requires root),
   symlinks, and empty directories
 * `--verify`: After restoring, verify every restored file's chunk hashes
@@ -413,11 +424,12 @@ under `blobs/` whose name is not a blob hash is not counted as a blob,
 since `prune` never deletes it.
 * `--json`: Output as JSON
 
-**`remote nuke`**: Delete every snapshot's metadata and every blob from the
+**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on
-backup destination store, leaving the bucket prefix empty. Destructive and
+the backup destination store, whatever its name, leaving the bucket prefix
-irreversible. This is the single supported way to wipe the entire
+empty. Destructive and irreversible. This is the single supported way to
-destination store. If anything cannot be deleted, the command exits
+wipe the entire destination store. If a listing afterwards still finds
-non-zero; running it again deletes what is left.
+anything there, the command names each object left and exits non-zero;
+running it again deletes what is left.
 * `--force`: Required to confirm destruction.
 
 **`database delete`**: Delete the local SQLite state database file

config.example.yml

@@ -302,7 +302,8 @@ storage_url: "rclone://myremote/path/to/backups"
 
 # Average chunk size for content-defined chunking
 # Smaller chunks = better deduplication but more metadata
-# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
+# Minimum 1MB
+# Supports: 10MB, 5M, 1GB, 64MiB, etc.
 # Default: 10MB
 #chunk_size: 10MB
 

internal/cli/config.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -44,6 +45,9 @@ var (
 	errNeedNumericIndex = errors.New("key is a list; use a numeric index")
 	errIndexOutOfRange  = errors.New("index out of range")
 	errNotMapOrList     = errors.New("key is not a map or list")
+	errWouldNotLoad     = errors.New(
+		"vaultik would not load the config after this change, " +
+			"so the file was not changed")
 )
 
 const defaultConfigTemplate = `# vaultik configuration
@@ -220,7 +224,7 @@ storage_url: ""
 
 # Average chunk size for content-defined chunking (FastCDC).
 # Smaller = better deduplication but more metadata overhead.
-# Accepts: 1MB, 10M, 64KB, etc.
+# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
 # Default: 10MB
 # chunk_size: 10MB
 
@@ -391,8 +395,17 @@ func newConfigSetCommand() *cobra.Command {
 		Use:   "set <key> <value>",
 		Short: "Set a config value by dotted path (e.g. compression_level 5)",
 		Long: `Sets a scalar config value addressed by dotted YAML path and writes
-the file back, preserving comments and formatting. Intermediate maps
+the file back, preserving comments and formatting. Missing keys along
-are created as needed.
+the path are created: as a list when the next path element is a
+number, otherwise as a map.
+
+If the config loads, a change after which it would not, such as
+compression_level 99, is refused and the file is left unchanged. A
+config that does not load, such as the one config init writes before
+storage_url is set or one with a ${ENV:...} reference to a variable
+not set here, is checked one setting at a time: only a change after
+which the changed setting cannot be read, such as one giving it the
+wrong type, is refused.
 
 Examples:
   vaultik config set storage_url "file:///mnt/backups"
@@ -422,7 +435,14 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 		return err
 	}
 
-	err = yamlPathSet(root, strings.Split(key, "."), value)
+	oldData, err := marshalConfigYAML(root)
+	if err != nil {
+		return fmt.Errorf("marshaling config: %w", err)
+	}
+
+	keys := strings.Split(key, ".")
+
+	err = yamlPathSet(root, keys, value)
 	if err != nil {
 		return err
 	}
@@ -432,6 +452,21 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 		return fmt.Errorf("marshaling config: %w", err)
 	}
 
+	setting, err := yamlPathOnly(root, keys)
+	if err != nil {
+		return err
+	}
+
+	settingData, err := marshalConfigYAML(setting)
+	if err != nil {
+		return fmt.Errorf("marshaling config: %w", err)
+	}
+
+	err = checkConfigChange(oldData, data, settingData)
+	if err != nil {
+		return fmt.Errorf("%w: %w", errWouldNotLoad, err)
+	}
+
 	err = os.WriteFile(path, data, configFileMode)
 	if err != nil {
 		return fmt.Errorf("writing config file: %w", err)
@@ -453,6 +488,58 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 	return nil
 }
 
+// checkConfigChange returns the reason vaultik would not load newData, the
+// config after a change, if the change brings that failure in. If oldData,
+// the config before it, loads, the new one must load too. A config that
+// does not load, such as one still being filled in or one with a ${...}
+// reference that cannot be filled in here, is checked one setting at a
+// time: settingData, the changed setting on its own, must decode into the
+// config type.
+func checkConfigChange(oldData, newData, settingData []byte) error {
+	oldCfg, err := config.Parse(oldData)
+	if err == nil && oldCfg.Validate() == nil {
+		newCfg, err := config.Parse(newData)
+		if err != nil {
+			return err
+		}
+
+		return newCfg.Validate()
+	}
+
+	_, err = config.Parse(settingData)
+
+	return err
+}
+
+// yamlPathOnly returns a document holding only the value at the dotted key
+// path in root, inside the maps and lists that lead to it there.
+func yamlPathOnly(root *yaml.Node, keys []string) (*yaml.Node, error) {
+	node, err := yamlPathGet(root, keys)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, key := range slices.Backward(keys) {
+		parent, err := yamlPathGet(root, keys[:i])
+		if err != nil {
+			return nil, err
+		}
+
+		if parent.Kind == yaml.SequenceNode {
+			node = &yaml.Node{
+				Kind: yaml.SequenceNode, Content: []*yaml.Node{node},
+			}
+		} else {
+			keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
+			node = &yaml.Node{
+				Kind: yaml.MappingNode, Content: []*yaml.Node{keyNode, node},
+			}
+		}
+	}
+
+	return &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{node}}, nil
+}
+
 // marshalConfigYAML renders a config document tree with 2-space indentation,
 // matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
 // would reindent the whole file on the first `config set` despite the promise
@@ -559,8 +646,8 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
 	return node, nil
 }
 
-// yamlPathSet navigates a dotted key path, creating intermediate maps as
+// yamlPathSet navigates a dotted key path, creating missing keys as needed,
-// needed, and sets the final key to the given scalar value. Numeric path
+// and sets the final key to the given scalar value. Numeric path
 // components index into sequences; an index equal to the sequence length
 // appends a new element (e.g. "age_recipients.1" on a 1-element list).
 func yamlPathSet(root *yaml.Node, keys []string, value string) error {
@@ -573,12 +660,12 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
 		node = node.Content[0]
 	}
 
-	for i, key := range keys {
+	for i := range keys {
 		last := i == len(keys)-1
 
 		switch node.Kind {
 		case yaml.MappingNode:
-			node = yamlSetInMapping(node, key, value, last)
+			node = yamlSetInMapping(node, keys, i, value, last)
 		case yaml.SequenceNode:
 			next, err := yamlSetInSequence(node, keys, i, value, last)
 			if err != nil {
@@ -647,14 +734,16 @@ func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
 	return reflect.StructField{}, false
 }
 
-// yamlSetInMapping resolves (creating if needed) the value node for key
+// yamlSetInMapping resolves (creating if needed) the value node for the
-// within a mapping node, setting it to value when it is the final path
+// path element keys[i] within a mapping node, setting it to value when it
-// element, and returns the node to descend into.
+// is the final path element, and returns the node to descend into.
-func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
+func yamlSetInMapping(
+	node *yaml.Node, keys []string, i int, value string, last bool,
+) *yaml.Node {
 	var valueNode *yaml.Node
 
 	for j := 0; j+1 < len(node.Content); j += 2 {
-		if node.Content[j].Value == key {
+		if node.Content[j].Value == keys[i] {
 			valueNode = node.Content[j+1]
 
 			break
@@ -662,13 +751,8 @@ func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node
 	}
 
 	if valueNode == nil {
-		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
+		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: keys[i]}
-
+		valueNode = newPathNode(keys, i, value)
-		valueNode = &yaml.Node{Kind: yaml.MappingNode}
-		if last {
-			valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
-		}
-
 		node.Content = append(node.Content, keyNode, valueNode)
 	} else if last {
 		setScalar(valueNode, value)
@@ -696,12 +780,7 @@ func yamlSetInSequence(
 	}
 
 	if idx == len(node.Content) {
-		newNode := &yaml.Node{Kind: yaml.MappingNode}
+		node.Content = append(node.Content, newPathNode(keys, i, value))
-		if last {
-			newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
-		}
-
-		node.Content = append(node.Content, newNode)
 	} else if last {
 		setScalar(node.Content[idx], value)
 	}
@@ -709,6 +788,22 @@ func yamlSetInSequence(
 	return node.Content[idx], nil
 }
 
+// newPathNode returns the node to create for the missing path element
+// keys[i]: the value when it is the final path element, otherwise an empty
+// list when the next path element is a number and an empty map when not.
+func newPathNode(keys []string, i int, value string) *yaml.Node {
+	if i == len(keys)-1 {
+		return &yaml.Node{Kind: yaml.ScalarNode, Value: value}
+	}
+
+	_, err := strconv.Atoi(keys[i+1])
+	if err == nil {
+		return &yaml.Node{Kind: yaml.SequenceNode}
+	}
+
+	return &yaml.Node{Kind: yaml.MappingNode}
+}
+
 // setScalar overwrites a node in place with a plain scalar value.
 func setScalar(n *yaml.Node, value string) {
 	n.Kind = yaml.ScalarNode

internal/cli/config_test.go

@@ -58,14 +58,17 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
 	}
 }
 
+// testRecipient is a valid age public key.
+const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
+
 // TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
 // file `config init` writes, `config set age_recipients.0` and
 // `config set storage_url` give a config that loads with that recipient.
+// Neither is refused, although the config does not load until both are set;
+// once it loads, a recipient that is not a valid key is.
 func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 	t.Parallel()
 
-	const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
-
 	path := filepath.Join(t.TempDir(), "config.yml")
 
 	err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
@@ -75,7 +78,7 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 
 	out := ui.NewWithColor(&bytes.Buffer{}, false)
 
-	err = writeConfigSet(out, path, "age_recipients.0", recipient)
+	err = writeConfigSet(out, path, "age_recipients.0", testRecipient)
 	if err != nil {
 		t.Fatalf("config set age_recipients.0: %v", err)
 	}
@@ -90,8 +93,211 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 		t.Fatalf("config.Load: %v", err)
 	}
 
-	if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
+	if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != testRecipient {
-		t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
+		t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, testRecipient)
+	}
+
+	err = writeConfigSet(out, path, "age_recipients.0", "notakey")
+	if err == nil || !strings.Contains(err.Error(), "not a valid recipient") {
+		t.Errorf("config set age_recipients.0 notakey gives error %v, "+
+			"want one saying it is not a valid recipient", err)
+	}
+
+	_, err = config.Load(path)
+	if err != nil {
+		t.Errorf("config.Load after the refused change: %v", err)
+	}
+}
+
+// configWithoutRecipients loads, but has no age_recipients key and no
+// snapshot named work.
+const configWithoutRecipients = `storage_url: file:///mnt/backups
+snapshots:
+  home:
+    paths:
+      - "~"
+`
+
+// configWithUnsetReference loads only where VAULTIK_TEST_UNSET_SECRET is
+// set, as a backup job's environment might set it. No test sets it.
+const configWithUnsetReference = configWithoutRecipients + `s3:
+  secret_access_key: ${ENV:VAULTIK_TEST_UNSET_SECRET}
+`
+
+// configWithLevelReference returns configWithoutRecipients with a ${FILE:...}
+// reference on compression_level to a file holding 9, so that the config
+// loads once the reference is filled in.
+func configWithLevelReference(t *testing.T) string {
+	t.Helper()
+
+	levelPath := filepath.Join(t.TempDir(), "level")
+
+	err := os.WriteFile(levelPath, []byte("9"), configFileMode)
+	if err != nil {
+		t.Fatalf("write level: %v", err)
+	}
+
+	return configWithoutRecipients +
+		"compression_level: ${FILE:" + levelPath + "}\n"
+}
+
+// TestConfigSetIndexUnderMissingKey checks that a list index under a key
+// the file does not have creates a list, and that the result loads.
+func TestConfigSetIndexUnderMissingKey(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		key   string
+		value string
+		field func(cfg *config.Config) []string
+	}{
+		{"age_recipients.0", testRecipient,
+			func(cfg *config.Config) []string { return cfg.AgeRecipients }},
+		{"snapshots.work.paths.0", "/srv/work",
+			func(cfg *config.Config) []string { return cfg.Snapshots["work"].Paths }},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.key, func(t *testing.T) {
+			t.Parallel()
+
+			path := filepath.Join(t.TempDir(), "config.yml")
+
+			err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
+			if err != nil {
+				t.Fatalf("write config: %v", err)
+			}
+
+			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
+				path, tt.key, tt.value)
+			if err != nil {
+				t.Fatalf("config set %s: %v", tt.key, err)
+			}
+
+			cfg, err := config.Load(path)
+			if err != nil {
+				t.Fatalf("config.Load: %v", err)
+			}
+
+			got := tt.field(cfg)
+			if len(got) != 1 || got[0] != tt.value {
+				t.Errorf("%s gives %v, want [%s]", tt.key, got, tt.value)
+			}
+		})
+	}
+}
+
+// TestConfigSetKeepsFileThatWouldNotLoad checks that config set refuses a
+// change after which the config would not load, with an error giving the
+// reason, and leaves the file as it was. On the file `config init` writes,
+// which does not load until storage_url is set, and on one that cannot be
+// read here, only a value of the wrong type is refused.
+func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
+	t.Parallel()
+
+	levelReference := configWithLevelReference(t)
+
+	tests := []struct {
+		config string
+		key    string
+		value  string
+		reason string
+	}{
+		// Index 1 of the empty list the missing key gives.
+		{configWithoutRecipients, "age_recipients.1", testRecipient,
+			"index out of range"},
+		// A list set to a single value.
+		{configWithoutRecipients, "snapshots.home.paths", "/srv",
+			"into []string"},
+		{configWithoutRecipients, "chunk_size", "fast",
+			"invalid size format"},
+		// Values of the right type that fail validation.
+		{configWithoutRecipients, "compression_level", "99",
+			"compression_level must be between 1 and 19"},
+		{configWithoutRecipients, "chunk_size", "1KB",
+			"chunk_size must be at least 1MB"},
+		// With its reference filled in, the config loads.
+		{levelReference, "blob_size_limit", "1MB",
+			"blob_size_limit must be at least"},
+		// The type check on a config that does not load yet.
+		{defaultConfigTemplate, "snapshots.apps.paths", "/srv",
+			"into []string"},
+		// The type check on a config that cannot be read here.
+		{configWithUnsetReference, "snapshots.home.paths", "/x",
+			"cannot unmarshal !!str `/x`"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.key+" "+tt.value, func(t *testing.T) {
+			t.Parallel()
+
+			path := filepath.Join(t.TempDir(), "config.yml")
+
+			err := os.WriteFile(path, []byte(tt.config), configFileMode)
+			if err != nil {
+				t.Fatalf("write config: %v", err)
+			}
+
+			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
+				path, tt.key, tt.value)
+			if err == nil || !strings.Contains(err.Error(), tt.reason) {
+				t.Errorf("config set %s %s gives error %v, want one saying %q",
+					tt.key, tt.value, err, tt.reason)
+			}
+
+			data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
+			if err != nil {
+				t.Fatalf("read config: %v", err)
+			}
+
+			if string(data) != tt.config {
+				t.Errorf("config set %s changed the file to:\n%s",
+					tt.key, data)
+			}
+		})
+	}
+}
+
+// TestConfigSetRefusesOnlyItsOwnFailure checks that setting another key
+// works on a config with a ${...} reference on compression_level, which
+// loads because loading fills the reference in, on one whose
+// compression_level already fails to load, and on one with a reference
+// that cannot be filled in here.
+func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
+	t.Parallel()
+
+	configs := map[string]string{
+		"reference":       configWithLevelReference(t),
+		"failing":         configWithoutRecipients + "compression_level: fast\n",
+		"unset reference": configWithUnsetReference,
+	}
+
+	for name, configYAML := range configs {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			path := filepath.Join(t.TempDir(), "config.yml")
+
+			err := os.WriteFile(path, []byte(configYAML), configFileMode)
+			if err != nil {
+				t.Fatalf("write config: %v", err)
+			}
+
+			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
+				path, "hostname", "myhost")
+			if err != nil {
+				t.Fatalf("config set hostname: %v", err)
+			}
+
+			data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
+			if err != nil {
+				t.Fatalf("read config: %v", err)
+			}
+
+			if !strings.Contains(string(data), "hostname: myhost") {
+				t.Errorf("config set hostname wrote:\n%s", data)
+			}
+		})
 	}
 }
 

internal/cli/entry_snapshot_purge_test.go

@@ -0,0 +1,68 @@
+package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/snapshot"
+)
+
+// Words and flags of `snapshot purge` used to build argument vectors
+// below.
+const (
+	cmdPurge      = "purge"
+	flagOlderThan = "--older-than"
+	flagForce     = "--force"
+)
+
+// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
+// the local index refuse to delete an old snapshot and checks that
+// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
+// the destination store. It used to log the error, delete the metadata
+// and exit 0.
+//
+// `snapshot create --prune` runs the same purge, but cannot be driven
+// this way: the refusal also breaks the backup's metadata export, which
+// deletes the other snapshots from its copy of the local index.
+//
+//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
+func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
+	configPath := writeHermeticPruneConfig(t, false)
+	dir := filepath.Dir(configPath)
+	indexPath := filepath.Join(dir, "index.sqlite")
+
+	// seedStaleSnapshotRecord writes an old snapshot to the local index
+	// only; the manifest planted here puts it on the destination store
+	// too, so the purge selects it rather than dropping it as stale.
+	seedStaleSnapshotRecord(t, indexPath)
+
+	manifestDir := filepath.Join(dir, "store", "metadata",
+		snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
+	require.NoError(t, os.MkdirAll(manifestDir, 0o750))
+
+	manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
+	require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
+
+	ctx := context.Background()
+
+	db, err := database.New(ctx, indexPath)
+	require.NoError(t, err)
+
+	_, err = db.Conn().ExecContext(ctx, `
+		CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
+		BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
+	require.NoError(t, err)
+	require.NoError(t, db.Close())
+
+	code, _, _ := runEntry(t, flagConfig, configPath,
+		cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
+
+	assert.Equal(t, 1, code)
+	assert.FileExists(t, manifestPath,
+		"the snapshot is still in the local index, so its metadata must stay")
+}

internal/cli/snapshot_restore.go

@@ -30,6 +30,8 @@ This command will restore files from the specified snapshot to the
 target directory.
 If no paths are specified, all files are restored.
 If paths are specified, only matching files/directories are restored.
+A path that matches nothing in the snapshot is named in a warning, and
+the command exits non-zero after restoring the paths that match.
 
 The snapshot may be named by its ID or, when restoring on a host with no
 local index, by the remote key that 'snapshot list' prints for a

internal/config/config.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -29,13 +30,15 @@ const appName = "vaultik"
 //nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
 const secretKeyPrefix = "AGE-SECRET-KEY-"
 
-// Defaults and validation bounds for tunable settings.
+// Defaults and validation bounds for tunable settings. MB and GB are
+// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
+// limits, which are 1024-based (MiB, GiB).
 const (
-	defaultBlobSizeLimit    = Size(10 * 1024 * 1024 * 1024) // 10GB
+	defaultBlobSizeLimit    = Size(10 * 1000 * 1000 * 1000) // 10GB
-	defaultChunkSize        = Size(10 * 1024 * 1024)        // 10MB
+	defaultChunkSize        = Size(10 * 1000 * 1000)        // 10MB
 	defaultS3PartSize       = Size(5 * 1024 * 1024)         // 5MiB
 	defaultCompressionLevel = 3
-	minChunkSize            = 1024 * 1024 // 1MB
+	minChunkSize            = 1000 * 1000 // 1MB
 	minCompressionLevel     = 1
 	maxCompressionLevel     = 19
 	// S3 accepts a multipart upload part from 5MiB to 5GiB.
@@ -233,14 +236,48 @@ func New(path Path) (*Config, error) {
 	return cfg, nil
 }
 
-// Load reads and parses the configuration file from the specified path.
+// Load reads the configuration file at path, parses it with Parse, warns
-// It applies default values for optional fields, performs environment variable
+// when others can read the file, and validates the configuration.
-// substitution using smartconfig, and validates the configuration.
+// Returns an error if the file cannot be read, parsed, or if validation
-// The configuration file should be in YAML format. Returns an error if the file
+// fails.
-// cannot be read, parsed, or if validation fails.
 func Load(path string) (*Config, error) {
+	data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config file: %w", err)
+	}
+
+	cfg, err := Parse(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check config file permissions (warn if world or group readable)
+	//nolint:gosec // G703: config path is operator-supplied by design
+	info, statErr := os.Stat(path)
+	if statErr == nil {
+		mode := info.Mode().Perm()
+		if mode&0044 != 0 { // group or world readable
+			log.Warn(cfg.readableByOthersWarning(),
+				"path", path,
+				"mode", fmt.Sprintf("%04o", mode),
+				"recommendation", "chmod 600 "+path)
+		}
+	}
+
+	err = cfg.Validate()
+	if err != nil {
+		return nil, fmt.Errorf("invalid config: %w", err)
+	}
+
+	return cfg, nil
+}
+
+// Parse reads YAML config data as Load does, without validating it. It
+// applies default values for optional fields, fills in ${...} references
+// using smartconfig, and expands ~ in paths.
+func Parse(data []byte) (*Config, error) {
 	// Load config using smartconfig for interpolation
-	sc, err := smartconfig.NewFromConfigPath(path)
+	sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
 	if err != nil {
 		return nil, fmt.Errorf("failed to load config file: %w", err)
 	}
@@ -302,24 +339,6 @@ func Load(path string) (*Config, error) {
 		cfg.S3.Region = "us-east-1"
 	}
 
-	// Check config file permissions (warn if world or group readable)
-	//nolint:gosec // G703: config path is operator-supplied by design
-	info, statErr := os.Stat(path)
-	if statErr == nil {
-		mode := info.Mode().Perm()
-		if mode&0044 != 0 { // group or world readable
-			log.Warn(cfg.readableByOthersWarning(),
-				"path", path,
-				"mode", fmt.Sprintf("%04o", mode),
-				"recommendation", "chmod 600 "+path)
-		}
-	}
-
-	err = cfg.Validate()
-	if err != nil {
-		return nil, fmt.Errorf("invalid config: %w", err)
-	}
-
 	return cfg, nil
 }
 

internal/config/config_test.go

@@ -337,6 +337,81 @@ func TestLoadS3PartSize(t *testing.T) {
 	})
 }
 
+// TestDocumentedSizeDefaults checks that the defaults are the documented
+// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
+// (1000-based), so writing a default out in the config file changes nothing.
+func TestDocumentedSizeDefaults(t *testing.T) {
+	t.Parallel()
+
+	cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
+	if err != nil {
+		t.Fatalf("Parse() unexpected error: %v", err)
+	}
+
+	if cfg.ChunkSize != defaultChunkSize {
+		t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
+			cfg.ChunkSize, defaultChunkSize)
+	}
+
+	if cfg.BlobSizeLimit != defaultBlobSizeLimit {
+		t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
+			cfg.BlobSizeLimit, defaultBlobSizeLimit)
+	}
+}
+
+// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
+// every size the README, config.example.yml and the config init template
+// list as an example is accepted.
+func TestDocumentedSizeExamples(t *testing.T) {
+	t.Parallel()
+
+	const withoutSizes = "snapshots:\n" +
+		"  test:\n" +
+		"    paths: [/tmp/vaultik-test-source]\n" +
+		"storage_url: file:///tmp/vaultik-test-storage\n"
+
+	tests := []struct {
+		setting string
+		wantErr error
+	}{
+		{"chunk_size: 1MB", nil},
+		{"chunk_size: 999999", errChunkSizeTooSmall},
+		{"chunk_size: 5M", nil},
+		{"chunk_size: 10M", nil},
+		{"chunk_size: 10MB", nil},
+		{"chunk_size: 64MiB", nil},
+		{"chunk_size: 1GB", nil},
+		// Four times the default chunk_size, the documented minimum.
+		{"blob_size_limit: 40MB", nil},
+		{"blob_size_limit: 500MB", nil},
+		{"blob_size_limit: 1GB", nil},
+		{"blob_size_limit: 1GiB", nil},
+		{"blob_size_limit: 10G", nil},
+		{"blob_size_limit: 10GB", nil},
+		{"s3: {part_size: 5MiB}", nil},
+		{"s3: {part_size: 10MB}", nil},
+		{"s3: {part_size: 16MiB}", nil},
+		{"s3: {part_size: 100MiB}", nil},
+		{"s3: {part_size: 5GiB}", nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.setting, func(t *testing.T) {
+			t.Parallel()
+
+			cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
+			if err != nil {
+				t.Fatalf("Parse() unexpected error: %v", err)
+			}
+
+			err = cfg.Validate()
+			if !errors.Is(err, tt.wantErr) {
+				t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 // TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
 // relative path fails at load. Accepted, it would resolve against the
 // working directory, so a backup run from another directory would write

internal/snapshot/exclude_test.go

@@ -578,3 +578,54 @@ func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
 	require.Equal(t, 1, result.FilesScanned,
 		"**/ file pattern should match at the top level and nested")
 }
+
+func TestExcludePatterns_AnchoredDoubleStarAtEveryDepth(t *testing.T) {
+	t.Parallel()
+
+	// As in gitignore, /**/ also matches directly under the source dir
+	fs := setupAnchoredTestFS(t)
+
+	scanner, repos, cleanup := createTestScanner(t, fs, []string{"/**/projectname"})
+	defer cleanup()
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
+	require.NoError(t, err)
+
+	// /**/projectname should exclude BOTH:
+	// - /backup/projectname/file.txt
+	// - /backup/otherproject/projectname/file.txt
+	// Total files: 4, excluded: 2
+	require.Equal(t, 2, result.FilesScanned,
+		"/**/ pattern should match at the top level and nested")
+}
+
+func TestExcludePatterns_DoubleStarInMiddleMatchesZeroDirectories(t *testing.T) {
+	t.Parallel()
+
+	// As in gitignore, a/**/b matches a/b as well as a/c/b. The last part
+	// is a glob: with a plain name the glob library already matched a/b.
+	fs := &afero.MemMapFs{}
+	files := []string{"/backup/a/x.log", "/backup/a/c/y.log", "/backup/a/c/kept.txt"}
+
+	for _, path := range files {
+		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0755))
+		require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0644))
+	}
+
+	scanner, repos, cleanup := createTestScanner(t, fs, []string{"a/**/*.log"})
+	defer cleanup()
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
+	require.NoError(t, err)
+
+	// a/**/*.log should exclude /backup/a/x.log and /backup/a/c/y.log
+	// Total files: 3, excluded: 2
+	require.Equal(t, 1, result.FilesScanned,
+		"a/**/*.log should match a/x.log as well as a/c/y.log")
+}

internal/snapshot/scanner.go

@@ -2035,33 +2035,60 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
 		pattern = strings.TrimSuffix(pattern, "/")
 
 		// A leading "**/" demands a "/" before the name and so misses an
-		// entry directly under the source path. An unanchored pattern
+		// entry directly under the source path. It means "at every depth",
-		// already matches at every depth, so drop the prefix: as in
+		// which is what an unanchored pattern already does, so drop the
-		// gitignore, "**/x" then also matches a top-level "x".
+		// prefix and match unanchored: as in gitignore, "**/x" and
-		if !anchored {
+		// "/**/x" then also match a top-level "x".
-			pattern = strings.TrimPrefix(pattern, "**/")
+		if rest, found := strings.CutPrefix(pattern, "**/"); found {
+			pattern = rest
+			anchored = false
 		}
 
-		// Compile the glob pattern
+		// Likewise "/**/" inside a pattern demands two slashes, so
-		// For patterns without path separators, we need to match them as components
+		// "a/**/*.log" misses "a/x.log". As in gitignore it stands for
-		// e.g., ".git" should match ".git" anywhere in the path
+		// zero or more directories, so also compile the pattern with it
-		g, err := glob.Compile(pattern, '/')
+		// made a single "/". The glob library's {a,b} is no help: with a
-		if err != nil {
+		// "**" inside it in mid-pattern, it misses paths it should match.
-			log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
+		for _, variant := range doubleStarVariants(pattern) {
+			// For patterns without path separators, we need to match them as components
+			// e.g., ".git" should match ".git" anywhere in the path
+			g, err := glob.Compile(variant, '/')
+			if err != nil {
+				log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
 
-			continue
+				break
+			}
+
+			compiled = append(compiled, compiledPattern{
+				pattern:  g,
+				anchored: anchored,
+				original: p,
+			})
 		}
-
-		compiled = append(compiled, compiledPattern{
-			pattern:  g,
-			anchored: anchored,
-			original: p,
-		})
 	}
 
 	return compiled
 }
 
+// doubleStarVariants returns pattern once for every way of keeping each
+// "/**/" in it or making it a single "/".
+func doubleStarVariants(pattern string) []string {
+	const doubleStar = "/**/"
+
+	before, after, found := strings.Cut(pattern, doubleStar)
+	if !found {
+		return []string{pattern}
+	}
+
+	var variants []string
+
+	for _, rest := range doubleStarVariants(after) {
+		variants = append(variants, before+doubleStar+rest, before+"/"+rest)
+	}
+
+	return variants
+}
+
 // shouldExclude checks if a path should be excluded based on exclude patterns
 // filePath is the full path to the file
 // rootPath is the root of the backup source directory

internal/storage/file.go

@@ -120,9 +120,10 @@ func (f *FileStorer) Delete(_ context.Context, key string) error {
 	return nil
 }
 
-// List returns all keys with the given prefix. It fails when the
+// List returns all keys with the given prefix, reading through symlinked
-// destination directory is missing; a missing prefix under it is an
+// directories. It fails when the destination directory is missing, or
-// empty listing.
+// when the prefix or a directory above it is a symlink whose target is
+// missing; a prefix that is truly missing is an empty listing.
 func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
 	var keys []string
 
@@ -133,8 +134,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 
 	basePath := f.fullPath(prefix)
 
-	// Check if the prefix exists
+	exists, err := f.prefixExists(prefix)
-	exists, err := afero.Exists(f.fs, basePath)
 	if err != nil {
 		return nil, fmt.Errorf("checking path: %w", err)
 	}
@@ -143,7 +143,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 		return keys, nil // Empty list for non-existent prefix
 	}
 
-	err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
+	err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -176,7 +176,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 }
 
 // ListStream returns a channel of ObjectInfo for large result sets. Like
-// List, it sends an error when the destination directory is missing.
+// List, it reads through symlinked directories and sends an error when
+// the destination directory is missing, or when the prefix or a directory
+// above it is a symlink whose target is missing.
 func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo {
 	ch := make(chan ObjectInfo)
 
@@ -192,8 +194,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 
 		basePath := f.fullPath(prefix)
 
-		// Check if the prefix exists
+		exists, err := f.prefixExists(prefix)
-		exists, err := afero.Exists(f.fs, basePath)
 		if err != nil {
 			ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
 
@@ -204,7 +205,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 			return // Empty channel for non-existent prefix
 		}
 
-		_ = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
+		_ = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
 			// Check context cancellation
 			select {
 			case <-ctx.Done():
@@ -243,11 +244,12 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 }
 
 // DeletePartialUploads removes every file under prefix whose name ends in
-// tempSuffix. A missing prefix has none to remove.
+// tempSuffix. Like List, it reads through symlinked directories and fails
+// on one whose target is missing. A missing prefix has none to remove.
 func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
 	basePath := f.fullPath(prefix)
 
-	exists, err := afero.Exists(f.fs, basePath)
+	exists, err := f.prefixExists(prefix)
 	if err != nil {
 		return fmt.Errorf("checking path: %w", err)
 	}
@@ -256,7 +258,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
 		return nil
 	}
 
-	err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
+	err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -352,6 +354,54 @@ func (f *FileStorer) writeAtomic(
 	return nil
 }
 
+// walk is afero.Walk, except that it reads through a symlinked directory
+// the way Get and Stat do, where afero.Walk reports the link as one file.
+// afero.Walk uses Lstat, which does not follow a link, whenever the
+// filesystem has one; the struct below carries only the afero.Fs methods,
+// so the walk uses Stat instead. A link back up the tree is followed until
+// the operating system refuses the path for too many levels of links, and
+// the walk then fails.
+func (f *FileStorer) walk(root string, fn filepath.WalkFunc) error {
+	return afero.Walk(struct{ afero.Fs }{f.fs}, root, fn)
+}
+
+// prefixExists reports whether prefix exists under the destination
+// directory. It checks each directory on the way down to prefix in turn,
+// and the first one Stat cannot find decides: if it is not there at all,
+// prefix is missing; if it is a symlink whose target is missing, such as
+// a link to a volume that is not mounted, prefixExists fails, since
+// listing it as empty would let prune delete every blob.
+func (f *FileStorer) prefixExists(prefix string) (bool, error) {
+	lstater, canLstat := f.fs.(afero.Lstater)
+	path := f.basePath
+
+	for name := range strings.SplitSeq(prefix, "/") {
+		path = filepath.Join(path, name)
+
+		_, err := f.fs.Stat(path)
+		if err == nil {
+			continue
+		}
+
+		if !os.IsNotExist(err) {
+			return false, err
+		}
+
+		// Lstat does not follow a symlink at path, so it finds the link
+		// that Stat could not follow.
+		if canLstat {
+			_, _, lstatErr := lstater.LstatIfPossible(path)
+			if lstatErr == nil {
+				return false, err
+			}
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
 // fullPath returns the full filesystem path for a key.
 func (f *FileStorer) fullPath(key string) string {
 	return filepath.Join(f.basePath, key)

internal/storage/file_backend_test.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"errors"
 	"io/fs"
+	"os"
 	"path/filepath"
+	"reflect"
+	"strings"
 	"testing"
 
 	"sneak.berlin/go/vaultik/internal/storage"
@@ -60,3 +63,232 @@ func TestFileStorerListMissingDestination(t *testing.T) {
 		t.Errorf("ListStream error = %v, want a not-exist error", streamErr)
 	}
 }
+
+// The two directories at the top of a destination, either of which the
+// tests below replace with a symlink.
+const (
+	metadataDir = "metadata"
+	blobsDir    = "blobs"
+)
+
+// moveBehindSymlink moves base/dir to a directory elsewhere and puts a
+// symlink to it in its place.
+func moveBehindSymlink(t *testing.T, base, dir string) {
+	t.Helper()
+
+	linkPath := filepath.Join(base, dir)
+	movedPath := filepath.Join(t.TempDir(), dir)
+
+	err := os.Rename(linkPath, movedPath)
+	if err != nil {
+		t.Fatalf("moving %s away: %v", dir, err)
+	}
+
+	err = os.Symlink(movedPath, linkPath)
+	if err != nil {
+		t.Fatalf("linking %s: %v", dir, err)
+	}
+}
+
+// TestFileStorerListsThroughSymlinkedDirectory checks that List and
+// ListStream read through a symlinked metadata/ or blobs/ directory, as Get
+// and Stat do, instead of listing the link as one key. Listing the whole
+// destination meets the link part-way through the walk; listing its prefix
+// starts the walk at the link. A prefix missing behind the link lists as
+// empty.
+func TestFileStorerListsThroughSymlinkedDirectory(t *testing.T) {
+	t.Parallel()
+
+	for _, dir := range []string{metadataDir, blobsDir} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+			key := dir + "/aa/bb/object"
+
+			s, err := storage.NewFileStorer(base)
+			if err != nil {
+				t.Fatalf("NewFileStorer: %v", err)
+			}
+
+			err = s.Put(context.Background(), key, strings.NewReader("data"))
+			if err != nil {
+				t.Fatalf("Put: %v", err)
+			}
+
+			moveBehindSymlink(t, base, dir)
+
+			want := []string{key}
+
+			for _, prefix := range []string{"", dir + "/"} {
+				got := listSorted(t, s, prefix)
+				if !reflect.DeepEqual(got, want) {
+					t.Errorf("List(%q) = %v, want %v", prefix, got, want)
+				}
+
+				got = listStreamKeys(t, s, prefix)
+				if !reflect.DeepEqual(got, want) {
+					t.Errorf("ListStream(%q) = %v, want %v", prefix, got, want)
+				}
+			}
+
+			absent := dir + "/absent/"
+
+			if got := listSorted(t, s, absent); len(got) != 0 {
+				t.Errorf("List(%q) = %v, want empty", absent, got)
+			}
+
+			if got := listStreamKeys(t, s, absent); len(got) != 0 {
+				t.Errorf("ListStream(%q) = %v, want empty", absent, got)
+			}
+		})
+	}
+}
+
+// TestFileStorerListFailsOnSymlinkWithMissingTarget checks that List and
+// ListStream fail on a symlinked metadata/ or blobs/ directory whose target
+// is missing, such as a link to a volume that is not mounted, as they do on
+// a missing destination directory, instead of listing it as empty. That
+// holds for a prefix below the link too.
+func TestFileStorerListFailsOnSymlinkWithMissingTarget(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	for _, dir := range []string{metadataDir, blobsDir} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+
+			s, err := storage.NewFileStorer(base)
+			if err != nil {
+				t.Fatalf("NewFileStorer: %v", err)
+			}
+
+			unmounted := filepath.Join(t.TempDir(), "unmounted")
+
+			err = os.Symlink(unmounted, filepath.Join(base, dir))
+			if err != nil {
+				t.Fatalf("Symlink: %v", err)
+			}
+
+			for _, prefix := range []string{"", dir + "/", dir + "/aa/"} {
+				keys, err := s.List(ctx, prefix)
+				if !errors.Is(err, fs.ErrNotExist) {
+					t.Errorf("List(%q) = %v, %v; want a not-exist error",
+						prefix, keys, err)
+				}
+
+				var streamErr error
+
+				for object := range s.ListStream(ctx, prefix) {
+					if object.Err != nil {
+						streamErr = object.Err
+					}
+				}
+
+				if !errors.Is(streamErr, fs.ErrNotExist) {
+					t.Errorf("ListStream(%q) error = %v, want a not-exist error",
+						prefix, streamErr)
+				}
+			}
+		})
+	}
+}
+
+// TestFileStorerListFailsOnSymlinkLoop checks that a symlink pointing back
+// at the destination makes List and ListStream fail instead of walking it
+// forever.
+func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	base := t.TempDir()
+
+	s, err := storage.NewFileStorer(base)
+	if err != nil {
+		t.Fatalf("NewFileStorer: %v", err)
+	}
+
+	err = s.Put(ctx, testBlobKey, strings.NewReader("data"))
+	if err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	err = os.Symlink(base, filepath.Join(base, "blobs", "loop"))
+	if err != nil {
+		t.Fatalf("Symlink: %v", err)
+	}
+
+	keys, err := s.List(ctx, "blobs/")
+	if err == nil {
+		t.Errorf("List = %d keys and no error, want an error", len(keys))
+	}
+
+	var streamErr error
+
+	for object := range s.ListStream(ctx, "blobs/") {
+		if object.Err != nil {
+			streamErr = object.Err
+		}
+	}
+
+	if streamErr == nil {
+		t.Error("ListStream sent no error, want one")
+	}
+}
+
+// TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory checks that
+// DeletePartialUploads, which remote nuke uses, removes a leftover temp
+// file behind a symlinked metadata/ or blobs/ directory and keeps the
+// object at the real key.
+func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	for _, dir := range []string{metadataDir, blobsDir} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+			key := dir + "/aa/bb/object"
+
+			s, err := storage.NewFileStorer(base)
+			if err != nil {
+				t.Fatalf("NewFileStorer: %v", err)
+			}
+
+			err = s.Put(ctx, key, strings.NewReader("data"))
+			if err != nil {
+				t.Fatalf("Put: %v", err)
+			}
+
+			leftover := filepath.Join(base, key+"-123456.partial")
+
+			err = os.WriteFile(leftover, []byte("half"), 0o600)
+			if err != nil {
+				t.Fatalf("writing leftover temp file: %v", err)
+			}
+
+			moveBehindSymlink(t, base, dir)
+
+			err = s.DeletePartialUploads(ctx, dir+"/")
+			if err != nil {
+				t.Fatalf("DeletePartialUploads: %v", err)
+			}
+
+			// leftover runs through the link.
+			_, err = os.Stat(leftover)
+			if !os.IsNotExist(err) {
+				t.Errorf("leftover temp file was not removed: %v", err)
+			}
+
+			_, err = s.Stat(ctx, key)
+			if err != nil {
+				t.Errorf("Stat of the real key: %v", err)
+			}
+		})
+	}
+}

internal/storage/rclone.go

@@ -7,10 +7,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/operations"
 
@@ -33,6 +35,15 @@ type RcloneStorer struct {
 // The remote parameter is the rclone remote name (as configured via `rclone config`).
 // The path parameter is the path within the remote.
 func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, error) {
+	// rclone reads RCLONE_CONFIG only as the value of its --config flag,
+	// which vaultik does not have, so it is applied here as that flag would.
+	if configPath, ok := os.LookupEnv("RCLONE_CONFIG"); ok {
+		err := config.SetConfigPath(configPath)
+		if err != nil {
+			return nil, fmt.Errorf("using RCLONE_CONFIG %q: %w", configPath, err)
+		}
+	}
+
 	// Install the default config file handler
 	configfile.Install()
 

internal/storage/rclone_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"sneak.berlin/go/vaultik/internal/storage"
 )
@@ -390,3 +391,41 @@ func TestNewRcloneStorerUnknownRemote(t *testing.T) {
 		t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
 	}
 }
+
+// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only
+// in the file RCLONE_CONFIG names is found and written to, as rclone itself
+// would.
+func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
+	// rclone keeps the config file path in a process-global variable, which
+	// NewRcloneStorer leaves pointing at this test's file.
+	previous := config.GetConfigPath()
+
+	t.Cleanup(func() { _ = config.SetConfigPath(previous) })
+
+	confFile := filepath.Join(t.TempDir(), "rclone.conf")
+
+	err := os.WriteFile(confFile, []byte("[loc]\ntype = local\n"), 0o600)
+	if err != nil {
+		t.Fatalf("writing rclone config: %v", err)
+	}
+
+	t.Setenv("RCLONE_CONFIG", confFile)
+
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	s, err := storage.NewRcloneStorer(ctx, "loc", dir)
+	if err != nil {
+		t.Fatalf("NewRcloneStorer: %v", err)
+	}
+
+	err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
+	if err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	_, err = os.Stat(filepath.Join(dir, testBlobKey))
+	if err != nil {
+		t.Errorf("object is not in the remote's directory: %v", err)
+	}
+}

internal/storage/url.go

@@ -30,6 +30,12 @@ var (
 		"storage URL must not carry credentials; " +
 			"set s3.access_key_id and s3.secret_access_key in the config instead")
 	ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
+	ErrURLBadQuery     = errors.New(
+		"storage URL query cannot be decoded; " +
+			"check it for a ; or a % not followed by two hex digits")
+	ErrRcloneRemoteParams = errors.New(
+		"rclone URL must not carry parameters in the remote name; " +
+			"set them on the remote with rclone config instead")
 )
 
 // URL represents a parsed storage URL.
@@ -90,8 +96,9 @@ func parseFileURL(rawPath string) (*URL, error) {
 }
 
 // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
-// the userinfo and any query parameter other than endpoint, region and
+// the userinfo or in the endpoint parameter's userinfo, a query that
-// ssl, so a credential-bearing URL is never stored or echoed.
+// cannot be decoded, and any query parameter other than endpoint, region
+// and ssl, so a credential-bearing URL is never stored or echoed.
 func parseS3URL(rawURL string) (*URL, error) {
 	u, err := url.Parse(rawURL)
 	if err != nil {
@@ -107,13 +114,26 @@ func parseS3URL(rawURL string) (*URL, error) {
 		return nil, ErrMissingBucket
 	}
 
-	query := u.Query()
+	// u.Query() would silently drop a part it cannot decode, endpoint
+	// included. The decode error is not wrapped: it can quote the value.
+	query, err := url.ParseQuery(u.RawQuery)
+	if err != nil {
+		return nil, ErrURLBadQuery
+	}
 
 	err = rejectUnknownParams(query, "endpoint", "region", "ssl")
 	if err != nil {
 		return nil, err
 	}
 
+	// A host name never contains "@", so one in an endpoint, with or
+	// without a scheme, can only introduce userinfo.
+	for _, endpoint := range query["endpoint"] {
+		if strings.Contains(endpoint, "@") {
+			return nil, ErrURLCredentials
+		}
+	}
+
 	return &URL{
 		Scheme:   schemeS3,
 		Bucket:   bucket,
@@ -125,9 +145,23 @@ func parseS3URL(rawURL string) (*URL, error) {
 }
 
 // parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
-// query parameters, so credentials in the userinfo and any parameter at
+// query parameters, so credentials in the userinfo, parameters in the
+// remote name, a query that cannot be decoded and any query parameter at
 // all are rejected rather than silently ignored.
 func parseRcloneURL(rawURL string) (*URL, error) {
+	// The remote name runs up to the first "/", "?" or "#". It never
+	// contains a comma; one starts rclone's remote,param=value form,
+	// whose values can be credentials. This is checked before url.Parse,
+	// whose error for such a name can print part of it.
+	remote := strings.TrimPrefix(rawURL, "rclone://")
+	if end := strings.IndexAny(remote, "/?#"); end >= 0 {
+		remote = remote[:end]
+	}
+
+	if strings.Contains(remote, ",") {
+		return nil, ErrRcloneRemoteParams
+	}
+
 	u, err := url.Parse(rawURL)
 	if err != nil {
 		return nil, wrapParseError(err)
@@ -137,12 +171,19 @@ func parseRcloneURL(rawURL string) (*URL, error) {
 		return nil, ErrURLCredentials
 	}
 
-	remote := u.Host
+	remote = u.Host
 	if remote == "" {
 		return nil, ErrMissingRemote
 	}
 
-	err = rejectUnknownParams(u.Query())
+	// As in parseS3URL: u.Query() would drop a part it cannot decode, and
+	// the decode error can quote the value.
+	query, err := url.ParseQuery(u.RawQuery)
+	if err != nil {
+		return nil, ErrURLBadQuery
+	}
+
+	err = rejectUnknownParams(query)
 	if err != nil {
 		return nil, err
 	}

internal/storage/url_parse_test.go

@@ -150,6 +150,19 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
 			wantErr: storage.ErrURLCredentials,
 			secrets: []string{key, secret, "mybucket"},
 		},
+		{
+			name: "s3 userinfo in endpoint",
+			raw: "s3://mybucket?endpoint=https://" +
+				key + ":" + secret + "@minio.example.com",
+			wantErr: storage.ErrURLCredentials,
+			secrets: []string{key, secret},
+		},
+		{
+			name:    "s3 userinfo in endpoint without scheme",
+			raw:     "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
+			wantErr: storage.ErrURLCredentials,
+			secrets: []string{key, secret},
+		},
 		{
 			name:    "s3 unknown query param",
 			raw:     "s3://mybucket?access_key=" + key + "&secret=" + secret,
@@ -203,6 +216,112 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
 	}
 }
 
+// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
+// remote name carrying parameters (rclone's "remote,param=value" form,
+// which can hold a credential) is rejected without echoing the value,
+// including the URLs that url.Parse itself refuses.
+func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name    string
+		raw     string
+		secrets []string // must not appear in the error message
+	}{
+		{
+			name:    "configured remote",
+			raw:     "rclone://gdrive,client_secret=topsecret/backups",
+			secrets: []string{"topsecret"},
+		},
+		{
+			name:    "remote defined inline",
+			raw:     "rclone://:s3,access_key_id=KEY,secret_access_key=SECRET/bucket",
+			secrets: []string{"KEY", "SECRET"},
+		},
+		{
+			name:    "colon in a parameter value",
+			raw:     "rclone://gdrive,client_secret=ab:cd/backups",
+			secrets: []string{"cd"},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := storage.ParseStorageURL(tc.raw)
+			if !errors.Is(err, storage.ErrRcloneRemoteParams) {
+				t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
+					tc.raw, err, storage.ErrRcloneRemoteParams)
+			}
+
+			for _, secret := range tc.secrets {
+				if strings.Contains(err.Error(), secret) {
+					t.Errorf("error message leaked %q: %v", secret, err.Error())
+				}
+			}
+		})
+	}
+}
+
+// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
+// URL whose query cannot be decoded is rejected, not read with that part
+// of the query left out, and that the error does not echo it.
+func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
+	t.Parallel()
+
+	const (
+		key    = "AKIAKEY"
+		secret = "topsecret"
+	)
+
+	cases := []struct {
+		name string
+		raw  string
+	}{
+		{
+			name: "semicolon",
+			raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
+				"@minio.example.com:9000;region=us-east-1",
+		},
+		{
+			name: "bad percent escape",
+			raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
+				"%zz@minio.example.com:9000",
+		},
+		{
+			name: "rclone semicolon",
+			raw:  "rclone://gdrive/backups?token=" + secret + ";x=1",
+		},
+		{
+			name: "rclone bad percent escape",
+			raw:  "rclone://gdrive/backups?token=" + secret + "%zz",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := storage.ParseStorageURL(tc.raw)
+			if err == nil {
+				t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
+			}
+
+			if !errors.Is(err, storage.ErrURLBadQuery) {
+				t.Errorf("ParseStorageURL(%q) error = %v, want %v",
+					tc.raw, err, storage.ErrURLBadQuery)
+			}
+
+			for _, s := range []string{key, secret, "minio.example.com"} {
+				if strings.Contains(err.Error(), s) {
+					t.Errorf("error message leaked %q: %v", s, err.Error())
+				}
+			}
+		})
+	}
+}
+
 // TestParseStorageURLParseFailureHidesURL checks that when url.Parse
 // itself fails, the wrapped error carries only the inner cause, not the
 // *url.Error whose text embeds the raw (possibly credential-bearing) URL.

internal/vaultik/nuke_remote_test.go

@@ -18,7 +18,8 @@ import (
 // TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file
 // under a file:// destination, including the `.partial` files uploads
 // killed part-way leave next to a snapshot's metadata and next to where a
-// blob would have been.
+// blob would have been, a snapshot's database whose manifest was never
+// uploaded, and a file under blobs/ whose name is not a blob hash.
 func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
@@ -38,6 +39,9 @@ func TestNukeRemoteLeavesNoFiles(t *testing.T) {
 			"db.zst.age-123456.partial"),
 		filepath.Join(storeDir, "blobs", hash[:2], hash[2:4],
 			hash+"-123456.partial"),
+		filepath.Join(storeDir, "metadata",
+			snapshot.RemoteSnapshotKey("no-manifest"), "db.zst.age"),
+		filepath.Join(storeDir, "blobs", "0b", "01", ".DS_Store"),
 	}
 
 	for _, leftover := range leftovers {
@@ -100,6 +104,34 @@ func TestNukeRemoteFailsWhenADeleteFails(t *testing.T) {
 	}
 }
 
+// TestNukeRemoteNamesWhatIsLeft checks that remote nuke, when an object
+// it does not recognise cannot be deleted, does not report the store
+// empty but names that object and fails.
+func TestNukeRemoteNamesWhatIsLeft(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := filepath.Join(t.TempDir(), "store")
+	v, _, out := backUpToFileDestination(ctx, t, storeDir)
+
+	const leftKey = "blobs/0b/01/.DS_Store"
+
+	leftPath := filepath.Join(storeDir, filepath.FromSlash(leftKey))
+	require.NoError(t, os.MkdirAll(filepath.Dir(leftPath), 0o750))
+	require.NoError(t, os.WriteFile(leftPath, []byte("finder"), 0o600))
+
+	store := faultstore.New(v.Storage)
+	store.OnDelete = func(key string) bool { return key == leftKey }
+	v.Storage = store
+
+	require.Error(t, v.NukeRemote(true))
+
+	assert.Contains(t, out.String(), leftKey)
+	assert.NotContains(t, out.String(), "now empty")
+	assert.Contains(t, out.String(), "vaultik remote nuke --force")
+}
+
 // TestNukeRemoteRefusesAnotherDestinationWithoutRetryAdvice checks that
 // remote nuke, when the local index belongs to a different destination
 // than the configured one, fails without advising to run nuke again:

internal/vaultik/prune.go

@@ -22,6 +22,11 @@ var errNukeRequiresForce = errors.New(
 // errBlobDeleteFailed reports unreferenced blobs prune could not delete.
 var errBlobDeleteFailed = errors.New("blob(s) could not be deleted")
 
+// errObjectsLeft reports objects remote nuke still finds on the backup
+// destination store after deleting.
+var errObjectsLeft = errors.New(
+	"object(s) still on the backup destination store")
+
 // nukeCommandHint is the command suggested when a remote nuke could not
 // delete everything: running it again deletes what is left, which
 // `vaultik prune` does not do for snapshot metadata.
@@ -35,10 +40,11 @@ const metadataDirName = "metadata"
 // (blobs/<aa>/<bb>/<hash>).
 const blobKeyParts = 4
 
-// NukeRemote deletes every snapshot's metadata and every blob from remote
+// NukeRemote deletes every object under metadata/ and blobs/ in remote
-// storage, along with any object an upload cut off part-way left under a
+// storage, whatever its name, along with any object an upload cut off
-// temporary `.partial` name. After this returns successfully the bucket
+// part-way left under a temporary `.partial` name. It returns
-// prefix is empty and the next backup starts from scratch.
+// successfully only when a listing afterwards finds nothing left there,
+// and the next backup then starts from scratch.
 //
 // Refuses to run unless force is true. The caller is responsible for
 // confirming with the user.
@@ -68,7 +74,8 @@ func (v *Vaultik) NukeRemote(force bool) error {
 }
 
 // emptyDestinationStore does the deleting for NukeRemote and stops at
-// the first step that fails.
+// the first step that fails. It also fails when objects are still
+// listed at the end, naming each one.
 func (v *Vaultik) emptyDestinationStore() error {
 	v.UI.Beginf("Removing all snapshot metadata from backup destination store.")
 
@@ -77,25 +84,62 @@ func (v *Vaultik) emptyDestinationStore() error {
 		return fmt.Errorf("removing all snapshots: %w", err)
 	}
 
-	v.UI.Beginf("Removing any blobs still present in backup destination store.")
+	v.UI.Beginf("Removing everything still present in backup destination store.")
 
-	err = v.PruneBlobs(&PruneOptions{Force: true})
+	var left []string
-	if err != nil {
+
-		return fmt.Errorf("pruning blobs: %w", err)
+	for _, prefix := range []string{"metadata/", "blobs/"} {
+		keys, err := v.deleteEverythingUnder(prefix)
+		if err != nil {
+			return err
+		}
+
+		left = append(left, keys...)
 	}
 
-	// The file and rclone listings skip `.partial` objects, so the two
+	for _, key := range left {
-	// steps above never delete them.
+		v.UI.Warningf("Still on the backup destination store: %s", key)
-	for _, prefix := range []string{"metadata/", "blobs/"} {
+	}
-		err = v.Storage.DeletePartialUploads(v.ctx, prefix)
+
-		if err != nil {
+	if len(left) > 0 {
-			return fmt.Errorf("deleting partial uploads: %w", err)
+		return fmt.Errorf("%d %w", len(left), errObjectsLeft)
-		}
 	}
 
 	return nil
 }
 
+// deleteEverythingUnder deletes every object under prefix, whatever its
+// name, and returns the keys a listing afterwards still finds. A delete
+// that fails is logged and the others still run; that object then
+// shows up in the listing afterwards.
+func (v *Vaultik) deleteEverythingUnder(prefix string) ([]string, error) {
+	keys, err := v.Storage.List(v.ctx, prefix)
+	if err != nil {
+		return nil, fmt.Errorf("listing %s: %w", prefix, err)
+	}
+
+	for _, key := range keys {
+		err = v.Storage.Delete(v.ctx, key)
+		if err != nil {
+			log.Error("Failed to delete object", "key", key, "error", err)
+		}
+	}
+
+	// The file and rclone listings skip `.partial` objects, so the
+	// deletes above never reach them.
+	err = v.Storage.DeletePartialUploads(v.ctx, prefix)
+	if err != nil {
+		return nil, fmt.Errorf("deleting partial uploads: %w", err)
+	}
+
+	left, err := v.Storage.List(v.ctx, prefix)
+	if err != nil {
+		return nil, fmt.Errorf("listing %s: %w", prefix, err)
+	}
+
+	return left, nil
+}
+
 // PruneBlobsResult contains the result of a blob prune operation
 //
 //nolint:tagliatelle // snake_case is the established JSON output format

internal/vaultik/restore.go

@@ -27,6 +27,7 @@ import (
 // Sentinel errors for restore failures.
 var (
 	errFilesFailedRestore    = errors.New("file(s) failed to restore")
+	errPathsNotInSnapshot    = errors.New("requested path(s) not in the snapshot")
 	errFilesFailedVerify     = errors.New("files failed verification")
 	errDecryptionKeyRequired = errors.New(
 		"decryption key required for restore\n\n" +
@@ -150,16 +151,20 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 	repos := database.NewRepositories(tempDB)
 
 	// Step 2: Get list of files to restore
-	files, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
+	files, unmatched, err := v.getFilesToRestore(v.ctx, repos, opts.Paths)
 	if err != nil {
 		return fmt.Errorf("getting files to restore: %w", err)
 	}
 
+	// Named before anything is restored, so the names are shown even
+	// when the restore stops on an error.
+	unmatchedErr := v.reportUnmatchedPaths(unmatched)
+
 	if len(files) == 0 {
 		log.Warn("No files found to restore")
 		v.UI.Warningf("No files found to restore.")
 
-		return nil
+		return unmatchedErr
 	}
 
 	log.Info("Found files to restore", "count", len(files))
@@ -185,6 +190,20 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 
 	result.Duration = time.Since(startTime)
 
+	return v.finishRestore(repos, files, unmatchedErr, opts, result)
+}
+
+// finishRestore prints the restore summary and the post-restore
+// warnings, runs optional verification, and converts any failed-file
+// count into an error. Otherwise it returns unmatchedErr, the error for
+// the requested paths that matched nothing (nil when there are none).
+func (v *Vaultik) finishRestore(
+	repos *database.Repositories,
+	files []*database.File,
+	unmatchedErr error,
+	opts *RestoreOptions,
+	result *RestoreResult,
+) error {
 	log.Info("Restore complete",
 		"files_restored", result.FilesRestored,
 		"bytes_restored", ubytes(result.BytesRestored),
@@ -199,17 +218,6 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
 		v.UI.Duration(result.Duration),
 	)
 
-	return v.finishRestore(repos, files, opts, result)
-}
-
-// finishRestore emits the post-restore warnings, runs optional
-// verification, and converts any failed-file count into an error.
-func (v *Vaultik) finishRestore(
-	repos *database.Repositories,
-	files []*database.File,
-	opts *RestoreOptions,
-	result *RestoreResult,
-) error {
 	if os.Geteuid() != 0 {
 		v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
 			"requires root. Re-run as root (e.g. with sudo) if you need " +
@@ -236,7 +244,23 @@ func (v *Vaultik) finishRestore(
 		return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
 	}
 
-	return nil
+	return unmatchedErr
+}
+
+// reportUnmatchedPaths warns, by name, about each requested path that
+// matched nothing in the snapshot, and returns an error counting them,
+// or nil when there are none.
+func (v *Vaultik) reportUnmatchedPaths(unmatched []string) error {
+	if len(unmatched) == 0 {
+		return nil
+	}
+
+	for _, path := range unmatched {
+		log.Warn("Requested path matches nothing in the snapshot", "path", path)
+		v.UI.Warningf("%s matches nothing in the snapshot.", v.UI.Path(path))
+	}
+
+	return fmt.Errorf("%d %w", len(unmatched), errPathsNotInSnapshot)
 }
 
 // restoreIdentities parses the configured age secret key once into every
@@ -840,27 +864,34 @@ func (v *Vaultik) materializeSnapshotDB(
 	return db, tempDir, nil
 }
 
-// getFilesToRestore returns the list of files to restore based on path filters
+// getFilesToRestore returns the list of files to restore based on path
+// filters, and the filters, as given, that matched no file.
 func (v *Vaultik) getFilesToRestore(
 	ctx context.Context, repos *database.Repositories, pathFilters []string,
-) ([]*database.File, error) {
+) ([]*database.File, []string, error) {
 	// If no filters, get all files
 	if len(pathFilters) == 0 {
-		return repos.Files.ListAll(ctx)
+		files, err := repos.Files.ListAll(ctx)
+
+		return files, nil, err
 	}
 
 	// Get files matching the path filters
-	var result []*database.File
+	var (
+		result    []*database.File
+		unmatched []string
+	)
 
 	seen := make(map[string]bool)
 
 	for _, filter := range pathFilters {
-		// Normalize the filter path
+		files, err := repos.Files.ListUnderPath(ctx, filepath.Clean(filter))
-		filter = filepath.Clean(filter)
-
-		files, err := repos.Files.ListUnderPath(ctx, filter)
 		if err != nil {
-			return nil, fmt.Errorf("listing files under %s: %w", filter, err)
+			return nil, nil, fmt.Errorf("listing files under %s: %w", filter, err)
+		}
+
+		if len(files) == 0 {
+			unmatched = append(unmatched, filter)
 		}
 
 		for _, file := range files {
@@ -871,7 +902,7 @@ func (v *Vaultik) getFilesToRestore(
 		}
 	}
 
-	return result, nil
+	return result, unmatched, nil
 }
 
 // buildChunkToBlobMap creates a mapping from chunk hash to blob information

internal/vaultik/restore_unmatched_path_test.go

@@ -0,0 +1,202 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/storage/faultstore"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// These tests cover https://git.eeqj.de/sneak/vaultik/issues/309: a path
+// given to restore that matches nothing in the snapshot is named in a
+// warning and makes the restore fail, after the paths that do match are
+// restored.
+
+func TestRestoreNamesUnmatchedPathAndRestoresTheMatchingOne(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	dataDir := filepath.Join(tempDir, "src")
+	storeDir := filepath.Join(tempDir, "remote")
+	restoreDir := filepath.Join(tempDir, "restored")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	ctx := context.Background()
+	cfg := faultTestConfig()
+	files := writeFaultSourceTree(t, fs, dataDir)
+
+	store, err := storage.NewFileStorer(storeDir)
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, dbPath)
+	require.NoError(t, err)
+
+	id := fullFaultBackup(ctx, t, fs, store, cfg,
+		database.NewRepositories(db), dataDir, dbPath, "unmatched")
+	require.NoError(t, db.Close())
+
+	matched := filepath.Join(dataDir, "small.txt")
+	unmatched := filepath.Join(dataDir, "typo.txt")
+
+	var out bytes.Buffer
+
+	v := newReaderVaultik(ctx, cfg, store, nil, fs)
+	v.UI = ui.NewWithColor(&out, false)
+
+	err = v.Restore(&vaultik.RestoreOptions{
+		SnapshotID: id,
+		TargetDir:  restoreDir,
+		Paths:      []string{matched, unmatched},
+	})
+
+	require.Error(t, err, "a path that matches nothing must fail the restore")
+	assert.Contains(t, out.String(), unmatched,
+		"the path that matches nothing must be named")
+	assert.NotContains(t, out.String(), matched,
+		"the path that matches must not be reported")
+
+	got, err := afero.ReadFile(fs, filepath.Join(restoreDir, matched))
+	require.NoError(t, err, "the path that matches must still be restored")
+	assert.Equal(t, files[matched], got)
+}
+
+// When no requested path matches, nothing is restored and each path is
+// named. One path is relative. The other goes through the symlink that
+// is the snapshot's configured path, while the entries are stored under
+// the directory the symlink points to.
+func TestRestoreNamesEveryPathWhenNothingMatches(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	const snapshotName = "linked"
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	dataDir := filepath.Join(tempDir, "src")
+	linkDir := filepath.Join(tempDir, "srclink")
+	storeDir := filepath.Join(tempDir, "remote")
+	restoreDir := filepath.Join(tempDir, "restored")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	ctx := context.Background()
+
+	writeFaultSourceTree(t, fs, dataDir)
+	require.NoError(t, os.Symlink(dataDir, linkDir))
+
+	cfg := faultTestConfig()
+	cfg.IndexPath = dbPath
+	cfg.ChunkSize = config.Size(faultChunkSize)
+	cfg.Snapshots = map[string]config.SnapshotConfig{
+		snapshotName: {Paths: []string{linkDir}},
+	}
+
+	store, err := storage.NewFileStorer(storeDir)
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, dbPath)
+	require.NoError(t, err)
+
+	repos := database.NewRepositories(db)
+	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+
+	require.NoError(t, backUp(v, snapshotName))
+
+	id := localSnapshotID(ctx, t, repos, snapshotName)
+	require.NoError(t, db.Close())
+
+	throughLink := filepath.Join(linkDir, "sub")
+	relative := strings.TrimPrefix(filepath.Join(dataDir, "small.txt"), "/")
+
+	var out bytes.Buffer
+
+	reader := newReaderVaultik(ctx, cfg, store, nil, fs)
+	reader.UI = ui.NewWithColor(&out, false)
+
+	err = reader.Restore(&vaultik.RestoreOptions{
+		SnapshotID: id,
+		TargetDir:  restoreDir,
+		Paths:      []string{throughLink, relative},
+	})
+
+	require.Error(t, err, "a restore that matches nothing must fail")
+	assert.Contains(t, out.String(), throughLink,
+		"the path through the symlink must be named")
+	assert.Contains(t, out.String(), relative,
+		"the relative path must be named")
+
+	_, err = fs.Stat(restoreDir)
+	assert.True(t, os.IsNotExist(err), "nothing must be restored")
+}
+
+// Without --skip-errors a damaged blob stops the restore part way. The
+// path that matches nothing must still be named.
+func TestRestoreNamesUnmatchedPathWhenADamagedBlobStopsIt(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	dataDir := filepath.Join(tempDir, "src")
+	storeDir := filepath.Join(tempDir, "remote")
+	restoreDir := filepath.Join(tempDir, "restored")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	ctx := context.Background()
+	cfg := faultTestConfig()
+
+	writeFaultSourceTree(t, fs, dataDir)
+
+	inner, err := storage.NewFileStorer(storeDir)
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, dbPath)
+	require.NoError(t, err)
+
+	id := fullFaultBackup(ctx, t, fs, inner, cfg,
+		database.NewRepositories(db), dataDir, dbPath, "damaged")
+	require.NoError(t, db.Close())
+
+	damaged := faultstore.New(inner)
+	damaged.OnGet = func(key string) faultstore.GetFault {
+		if strings.HasPrefix(key, "blobs/") {
+			return faultstore.GetCorrupt
+		}
+
+		return faultstore.GetNormal
+	}
+
+	matched := filepath.Join(dataDir, "small.txt")
+	unmatched := filepath.Join(dataDir, "typo.txt")
+
+	var out bytes.Buffer
+
+	v := newReaderVaultik(ctx, cfg, damaged, nil, fs)
+	v.UI = ui.NewWithColor(&out, false)
+
+	err = v.Restore(&vaultik.RestoreOptions{
+		SnapshotID: id,
+		TargetDir:  restoreDir,
+		Paths:      []string{matched, unmatched},
+	})
+
+	require.Error(t, err, "a damaged blob must stop the restore")
+	assert.Contains(t, err.Error(), "blob",
+		"the restore must stop on the damaged blob")
+	assert.Contains(t, out.String(), unmatched,
+		"the path that matches nothing must be named")
+}

internal/vaultik/snapshot.go

@@ -669,10 +669,11 @@ func (v *Vaultik) confirmAndExecutePurge(
 		snapshotID := snap.ID.String()
 		log.Info("Deleting snapshot", "id", snapshotID)
 
+		// Stop before the remote delete, so a snapshot still in the local
+		// index keeps its metadata on the destination store.
 		err := v.deleteSnapshotFromLocalDB(snapshotID)
 		if err != nil {
-			log.Error("Failed to delete from local database",
+			return fmt.Errorf("deleting snapshot %s from local index: %w", snapshotID, err)
-				"snapshot_id", snapshotID, "error", err)
 		}
 
 		_, err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID))

internal/vaultik/symlinked_destination_test.go

@@ -0,0 +1,147 @@
+package vaultik_test
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestPruneDeletesNothingThroughSymlinkedDirectory checks that prune keeps
+// every blob and the local snapshot record when the metadata/ or blobs/
+// directory of a file:// destination is a symlink to a directory elsewhere
+// (https://git.eeqj.de/sneak/vaultik/issues/296). A backup writes through
+// such a link, so prune has to list through it too.
+func TestPruneDeletesNothingThroughSymlinkedDirectory(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	for _, dir := range []string{"metadata", "blobs"} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			storeDir := filepath.Join(t.TempDir(), "store")
+			v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
+			blobFiles := blobFilesIn(t, storeDir)
+			moveBehindSymlink(t, storeDir, dir)
+
+			require.NoError(t, v.Prune(&vaultik.PruneOptions{Force: true}))
+
+			// When blobs/ is the link, these paths run through it.
+			for _, blobFile := range blobFiles {
+				assert.FileExists(t, blobFile, "prune must delete no blob")
+			}
+
+			snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
+			require.NoError(t, err)
+			assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
+		})
+	}
+}
+
+// TestPruneFailsOnSymlinkedDirectoryWithMissingTarget checks that prune
+// fails, deleting no blob and keeping the local snapshot record, when the
+// metadata/ or blobs/ directory of a file:// destination is a symlink
+// whose target is missing, as a link to a volume that is not mounted is.
+func TestPruneFailsOnSymlinkedDirectoryWithMissingTarget(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	for _, dir := range []string{"metadata", "blobs"} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			storeDir := filepath.Join(t.TempDir(), "store")
+			v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
+			blobFiles := blobFilesIn(t, storeDir)
+			targetPath := moveBehindSymlink(t, storeDir, dir)
+
+			// Unmount the volume the link points to, and mount it again
+			// once prune has run.
+			unmountedPath := targetPath + "-unmounted"
+			require.NoError(t, os.Rename(targetPath, unmountedPath))
+
+			err := v.Prune(&vaultik.PruneOptions{Force: true})
+			require.ErrorIs(t, err, fs.ErrNotExist)
+
+			require.NoError(t, os.Rename(unmountedPath, targetPath))
+
+			// When blobs/ is the link, these paths run through it.
+			for _, blobFile := range blobFiles {
+				assert.FileExists(t, blobFile, "prune must delete no blob")
+			}
+
+			snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
+			require.NoError(t, err)
+			assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
+		})
+	}
+}
+
+// TestRemoveSnapshotWarnsOnSymlinkedMetadataWithMissingTarget checks that
+// snapshot remove warns that the metadata could not be removed from the
+// destination when metadata/ is a symlink whose target is missing, as it
+// does when the destination directory is missing, instead of finding no
+// metadata to remove.
+func TestRemoveSnapshotWarnsOnSymlinkedMetadataWithMissingTarget(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	storeDir := filepath.Join(t.TempDir(), "store")
+	v, repos, out := backUpToFileDestination(ctx, t, storeDir)
+	targetPath := moveBehindSymlink(t, storeDir, "metadata")
+	require.NoError(t, os.Rename(targetPath, targetPath+"-unmounted"))
+
+	result, err := v.RemoveSnapshot(localSnapshotID(ctx, t, repos, "first"),
+		&vaultik.RemoveOptions{Force: true})
+	require.NoError(t, err)
+
+	assert.False(t, result.RemoteRemoved)
+	assert.Contains(t, out.String(),
+		"Could not remove snapshot metadata from remote")
+	assert.NotContains(t, out.String(),
+		"Removed snapshot metadata from remote storage")
+}
+
+// blobFilesIn returns the paths of the blob files under storeDir/blobs, and
+// fails the test when there are none.
+func blobFilesIn(t *testing.T, storeDir string) []string {
+	t.Helper()
+
+	var blobFiles []string
+
+	err := filepath.WalkDir(filepath.Join(storeDir, "blobs"),
+		func(path string, entry fs.DirEntry, err error) error {
+			if err == nil && !entry.IsDir() {
+				blobFiles = append(blobFiles, path)
+			}
+
+			return err
+		})
+	require.NoError(t, err)
+	require.NotEmpty(t, blobFiles)
+
+	return blobFiles
+}
+
+// moveBehindSymlink moves storeDir/dir to a directory elsewhere, puts a
+// symlink to it in its place, and returns where it went.
+func moveBehindSymlink(t *testing.T, storeDir, dir string) string {
+	t.Helper()
+
+	linkPath := filepath.Join(storeDir, dir)
+	targetPath := filepath.Join(t.TempDir(), dir)
+	require.NoError(t, os.Rename(linkPath, targetPath))
+	require.NoError(t, os.Symlink(targetPath, linkPath))
+
+	return targetPath
+}