2 Commits
Author SHA1 Message Date
sneak 89ce0f36f7 Make snapshot list report a changed destination instead of advising prune (closes #317)
check / check (push) Successful in 15m50s
When the local index is bound to a different destination than the
configured one, snapshot list now gives the warning prune, snapshot
purge and snapshot remove refuse with: both destinations and the two
ways out (revert the configuration, or run vaultik database delete).
Below the table it replaces the local-only note and its prune advice.
With --json it goes to stderr through the logger, so stdout stays the
document, whose local rows show remote_present false.

Reading the binding moves out of EnsureStorageBinding into
storageBinding, which records nothing, so the read-only list can call
it without binding a fresh index.

Model: opus-5-5
2026-10-14 01:50:52 +00:00
clawbot aeac897805 Refuse credentials in a storage URL's endpoint or rclone remote name (closes #322)
check / check (push) Successful in 11m34s
storage_url refused only the outer URL's userinfo and unknown query
parameters. Userinfo inside the s3 endpoint= value, and rclone's
remote,param=value parameters in the remote name, were accepted and
then printed by info, remote info and snapshot create. Both are now
refused without echoing the value: a host name never contains "@", and
an rclone remote name never contains a comma. An s3 or rclone query
that cannot be decoded (a ";" or a bad "%" escape) is refused too,
because decoding drops that part before it can be checked. The comma
is checked before url.Parse, whose error for such a name prints part of
it. The rclone remote name case has its own error pointing at rclone
config.

Model: opus-5-5
2026-10-14 03:26:02 +02:00
6 changed files with 300 additions and 24 deletions
+10 -4
View File
@@ -323,7 +323,11 @@ read from the manifest.
Snapshots in the local index with no counterpart on the destination Snapshots in the local index with no counterpart on the destination
store are reported below the table as drift, with the `vaultik prune` store are reported below the table as drift, with the `vaultik prune`
invocation that reconciles them. invocation that reconciles them. If the local index is bound to a
different destination than the configured one (after a change to
`storage_url`, or `s3.endpoint`, `s3.bucket` or `s3.prefix`), the
command says so instead and gives the same two ways out that `prune`
gives: revert the configuration, or run `vaultik database delete`.
If the destination store cannot be listed (unmounted volume, If the destination store cannot be listed (unmounted volume,
permission denied, network down), the command warns, falls back to the permission denied, network down), the command warns, falls back to the
@@ -332,9 +336,11 @@ local index alone, and still exits zero.
(whether the snapshot is in the local index), `remote_key` (the full (whether the snapshot is in the local index), `remote_key` (the full
64-character storage key), and `remote_present` (whether it was seen 64-character storage key), and `remote_present` (whether it was seen
on the destination store, or `null` if the destination could not be on the destination store, or `null` if the destination could not be
listed). Warnings about an unlistable destination, unreadable listed). Warnings about an unlistable destination, a local index
manifests, and a truncated listing all go to stderr through the bound to a different destination (with both destinations and the
logger, so stdout stays a single parseable document. same two ways out), unreadable manifests, and a truncated listing all
go to stderr through the logger, so stdout stays a single parseable
document.
**`snapshot verify`**: Verify snapshot integrity. **`snapshot verify`**: Verify snapshot integrity.
* Default (shallow): checks that every blob the manifest lists is present in * Default (shallow): checks that every blob the manifest lists is present in
+47 -6
View File
@@ -30,6 +30,12 @@ var (
"storage URL must not carry credentials; " + "storage URL must not carry credentials; " +
"set s3.access_key_id and s3.secret_access_key in the config instead") "set s3.access_key_id and s3.secret_access_key in the config instead")
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL") ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
ErrURLBadQuery = errors.New(
"storage URL query cannot be decoded; " +
"check it for a ; or a % not followed by two hex digits")
ErrRcloneRemoteParams = errors.New(
"rclone URL must not carry parameters in the remote name; " +
"set them on the remote with rclone config instead")
) )
// URL represents a parsed storage URL. // URL represents a parsed storage URL.
@@ -90,8 +96,9 @@ func parseFileURL(rawPath string) (*URL, error) {
} }
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
// the userinfo and any query parameter other than endpoint, region and // the userinfo or in the endpoint parameter's userinfo, a query that
// ssl, so a credential-bearing URL is never stored or echoed. // cannot be decoded, and any query parameter other than endpoint, region
// and ssl, so a credential-bearing URL is never stored or echoed.
func parseS3URL(rawURL string) (*URL, error) { func parseS3URL(rawURL string) (*URL, error) {
u, err := url.Parse(rawURL) u, err := url.Parse(rawURL)
if err != nil { if err != nil {
@@ -107,13 +114,26 @@ func parseS3URL(rawURL string) (*URL, error) {
return nil, ErrMissingBucket return nil, ErrMissingBucket
} }
query := u.Query() // u.Query() would silently drop a part it cannot decode, endpoint
// included. The decode error is not wrapped: it can quote the value.
query, err := url.ParseQuery(u.RawQuery)
if err != nil {
return nil, ErrURLBadQuery
}
err = rejectUnknownParams(query, "endpoint", "region", "ssl") err = rejectUnknownParams(query, "endpoint", "region", "ssl")
if err != nil { if err != nil {
return nil, err return nil, err
} }
// A host name never contains "@", so one in an endpoint, with or
// without a scheme, can only introduce userinfo.
for _, endpoint := range query["endpoint"] {
if strings.Contains(endpoint, "@") {
return nil, ErrURLCredentials
}
}
return &URL{ return &URL{
Scheme: schemeS3, Scheme: schemeS3,
Bucket: bucket, Bucket: bucket,
@@ -125,9 +145,23 @@ func parseS3URL(rawURL string) (*URL, error) {
} }
// parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no // parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
// query parameters, so credentials in the userinfo and any parameter at // query parameters, so credentials in the userinfo, parameters in the
// remote name, a query that cannot be decoded and any query parameter at
// all are rejected rather than silently ignored. // all are rejected rather than silently ignored.
func parseRcloneURL(rawURL string) (*URL, error) { func parseRcloneURL(rawURL string) (*URL, error) {
// The remote name runs up to the first "/", "?" or "#". It never
// contains a comma; one starts rclone's remote,param=value form,
// whose values can be credentials. This is checked before url.Parse,
// whose error for such a name can print part of it.
remote := strings.TrimPrefix(rawURL, "rclone://")
if end := strings.IndexAny(remote, "/?#"); end >= 0 {
remote = remote[:end]
}
if strings.Contains(remote, ",") {
return nil, ErrRcloneRemoteParams
}
u, err := url.Parse(rawURL) u, err := url.Parse(rawURL)
if err != nil { if err != nil {
return nil, wrapParseError(err) return nil, wrapParseError(err)
@@ -137,12 +171,19 @@ func parseRcloneURL(rawURL string) (*URL, error) {
return nil, ErrURLCredentials return nil, ErrURLCredentials
} }
remote := u.Host remote = u.Host
if remote == "" { if remote == "" {
return nil, ErrMissingRemote return nil, ErrMissingRemote
} }
err = rejectUnknownParams(u.Query()) // As in parseS3URL: u.Query() would drop a part it cannot decode, and
// the decode error can quote the value.
query, err := url.ParseQuery(u.RawQuery)
if err != nil {
return nil, ErrURLBadQuery
}
err = rejectUnknownParams(query)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+119
View File
@@ -150,6 +150,19 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
wantErr: storage.ErrURLCredentials, wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret, "mybucket"}, secrets: []string{key, secret, "mybucket"},
}, },
{
name: "s3 userinfo in endpoint",
raw: "s3://mybucket?endpoint=https://" +
key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret},
},
{
name: "s3 userinfo in endpoint without scheme",
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret},
},
{ {
name: "s3 unknown query param", name: "s3 unknown query param",
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret, raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
@@ -203,6 +216,112 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
} }
} }
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
// remote name carrying parameters (rclone's "remote,param=value" form,
// which can hold a credential) is rejected without echoing the value,
// including the URLs that url.Parse itself refuses.
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
t.Parallel()
cases := []struct {
name string
raw string
secrets []string // must not appear in the error message
}{
{
name: "configured remote",
raw: "rclone://gdrive,client_secret=topsecret/backups",
secrets: []string{"topsecret"},
},
{
name: "remote defined inline",
raw: "rclone://:s3,access_key_id=KEY,secret_access_key=SECRET/bucket",
secrets: []string{"KEY", "SECRET"},
},
{
name: "colon in a parameter value",
raw: "rclone://gdrive,client_secret=ab:cd/backups",
secrets: []string{"cd"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, storage.ErrRcloneRemoteParams)
}
for _, secret := range tc.secrets {
if strings.Contains(err.Error(), secret) {
t.Errorf("error message leaked %q: %v", secret, err.Error())
}
}
})
}
}
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
// URL whose query cannot be decoded is rejected, not read with that part
// of the query left out, and that the error does not echo it.
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
t.Parallel()
const (
key = "AKIAKEY"
secret = "topsecret"
)
cases := []struct {
name string
raw string
}{
{
name: "semicolon",
raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
"@minio.example.com:9000;region=us-east-1",
},
{
name: "bad percent escape",
raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
"%zz@minio.example.com:9000",
},
{
name: "rclone semicolon",
raw: "rclone://gdrive/backups?token=" + secret + ";x=1",
},
{
name: "rclone bad percent escape",
raw: "rclone://gdrive/backups?token=" + secret + "%zz",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
}
if !errors.Is(err, storage.ErrURLBadQuery) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, storage.ErrURLBadQuery)
}
for _, s := range []string{key, secret, "minio.example.com"} {
if strings.Contains(err.Error(), s) {
t.Errorf("error message leaked %q: %v", s, err.Error())
}
}
})
}
}
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse // TestParseStorageURLParseFailureHidesURL checks that when url.Parse
// itself fails, the wrapped error carries only the inner cause, not the // itself fails, the wrapped error carries only the inner cause, not the
// *url.Error whose text embeds the raw (possibly credential-bearing) URL. // *url.Error whose text embeds the raw (possibly credential-bearing) URL.
+33 -4
View File
@@ -61,7 +61,9 @@ const tabPadding = 3
// attempt is made to recover or fabricate the human ID. // attempt is made to recover or fabricate the human ID.
// //
// Snapshots in the local index with no counterpart on the destination // Snapshots in the local index with no counterpart on the destination
// store are reported as drift below the table. // store are reported as drift below the table, unless the local index
// is bound to a different destination than the configured one: that is
// reported instead.
// //
// If remote listing fails (unmounted volume, permission denied, // If remote listing fails (unmounted volume, permission denied,
// network), we degrade to local-only with a warning. List never fails // network), we degrade to local-only with a warning. List never fails
@@ -103,7 +105,26 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
return snapshots[i].Timestamp.After(snapshots[j].Timestamp) return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
}) })
stored, configured, err := v.storageBinding()
if err != nil {
return err
}
// A local index bound to another destination describes that one, so
// its records are expected to be missing here, and prune refuses to
// run. The binding is reported instead of advising prune.
boundElsewhere := stored != "" && stored != configured
if jsonOutput { if jsonOutput {
// stdout holds only the document, so this goes to the logger.
if boundElsewhere {
log.Warn("Local index is bound to a different backup "+
"destination; to proceed, revert storage_url (or the s3.* "+
"endpoint, bucket and prefix) to the bound destination, "+
"or run 'vaultik database delete'",
"bound_to", stored, "configured", configured)
}
if remoteErr == nil { if remoteErr == nil {
v.reportJSONListingLimits(listing) v.reportJSONListingLimits(listing)
} }
@@ -125,8 +146,13 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
} }
} }
if boundElsewhere {
v.UI.Warningf("%v\n%s", errStorageBindingMismatch,
buildBindingMismatchMessage(stored, configured))
}
if remoteErr == nil { if remoteErr == nil {
v.reportListDrift(snapshots, listing) v.reportListDrift(snapshots, listing, boundElsewhere)
} }
return nil return nil
@@ -432,7 +458,9 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo {
// reportListDrift prints the reconciliation notes the merged table // reportListDrift prints the reconciliation notes the merged table
// cannot express on its own: local records with no counterpart on the // cannot express on its own: local records with no counterpart on the
// destination store, plus counts of remote snapshots that were // destination store, plus counts of remote snapshots that were
// unreadable or omitted. // unreadable or omitted. Local records are not reported when
// boundElsewhere is set: ListSnapshots has already said why they are
// missing.
// //
// This is what remains of the old reportRemoteDrift, and it no longer // This is what remains of the old reportRemoteDrift, and it no longer
// touches the destination store. Its remote-only half collapsed into // touches the destination store. Its remote-only half collapsed into
@@ -441,6 +469,7 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo {
// so `snapshot list` lists the destination exactly once per invocation. // so `snapshot list` lists the destination exactly once per invocation.
func (v *Vaultik) reportListDrift( func (v *Vaultik) reportListDrift(
snapshots []SnapshotInfo, listing *remoteSnapshotListing, snapshots []SnapshotInfo, listing *remoteSnapshotListing,
boundElsewhere bool,
) { ) {
var localOnly []string var localOnly []string
@@ -450,7 +479,7 @@ func (v *Vaultik) reportListDrift(
} }
} }
if len(localOnly) > 0 { if len(localOnly) > 0 && !boundElsewhere {
v.UI.Warningf("%d local snapshot record(s) not found in backup "+ v.UI.Warningf("%d local snapshot record(s) not found in backup "+
"destination store:", len(localOnly)) "destination store:", len(localOnly))
+68
View File
@@ -421,6 +421,74 @@ func TestListSnapshots_LocalOnlyReportedAsDrift(t *testing.T) {
assert.NotContains(t, out, "snapshot cleanup") assert.NotContains(t, out, "snapshot cleanup")
} }
// TestListSnapshots_BoundElsewhereDoesNotAdvisePrune covers a changed
// storage_url: the local index is still bound to the old destination,
// so its records are missing from the new one. prune refuses to run in
// that state, so the listing must give the choices prune gives instead
// of advising it.
func TestListSnapshots_BoundElsewhereDoesNotAdvisePrune(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const (
oldURL = "file:///mnt/backups/old"
newURL = "file:///mnt/backups/new"
)
env := newListEnv(t)
env.v.Config.StorageURL = newURL
require.NoError(t, env.v.Repositories.LocalMeta.Set(context.Background(),
database.LocalMetaKeyStorageURL, oldURL))
env.addLocal(t, listLocalID, time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC))
err := env.v.ListSnapshots(false)
require.NoError(t, err)
out := env.stdout.String()
assert.Contains(t, out, listLocalID)
assert.Contains(t, out,
"local index is bound to a different backup destination")
assert.Contains(t, out, oldURL)
assert.Contains(t, out, newURL)
assert.Contains(t, out, "revert storage_url")
assert.Contains(t, out, "vaultik database delete")
assert.NotContains(t, out, "vaultik prune")
}
// TestListSnapshots_JSONReportsBoundElsewhere is the --json side of the
// test above. Every local row comes out with remote_present false, which
// a script reads as stale local records unless it is told, on stderr,
// that the local index is bound to a different destination.
//
//nolint:paralleltest // captureProcessStderr replaces os.Stderr
func TestListSnapshots_JSONReportsBoundElsewhere(t *testing.T) {
const (
oldURL = "file:///mnt/backups/old"
newURL = "file:///mnt/backups/new"
)
env := newListEnv(t)
env.v.Config.StorageURL = newURL
require.NoError(t, env.v.Repositories.LocalMeta.Set(context.Background(),
database.LocalMetaKeyStorageURL, oldURL))
env.addLocal(t, listLocalID, time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC))
stderr := captureProcessStderr(t, func() {
require.NoError(t, env.v.ListSnapshots(true))
})
rows := decodeListJSON(t, env.stdout.String())
require.Len(t, rows, 1)
assert.Equal(t, listLocalID, rows[0].ID)
assert.Contains(t, stderr, "bound to a different backup destination")
assert.Contains(t, stderr, oldURL)
assert.Contains(t, stderr, newURL)
assert.Contains(t, stderr, "revert storage_url")
assert.Contains(t, stderr, "vaultik database delete")
}
// TestListSnapshots_UnreachableRemoteDegrades covers the promise in the // TestListSnapshots_UnreachableRemoteDegrades covers the promise in the
// doc comment: an unreachable destination is a warning plus local-only // doc comment: an unreachable destination is a warning plus local-only
// output, never a failure. // output, never a failure.
+23 -10
View File
@@ -45,17 +45,9 @@ func (v *Vaultik) EnsureStorageBinding() error {
return nil return nil
} }
configured := v.Config.StorageURL stored, configured, err := v.storageBinding()
if configured == "" {
// The s3.* form: bind on its endpoint, bucket and prefix,
// written the way storage_url would name the same destination.
s3 := v.Config.S3
configured = "s3://" + s3.Bucket + "/" + s3.Prefix + "?endpoint=" + s3.Endpoint
}
stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
if err != nil { if err != nil {
return fmt.Errorf("reading local storage binding: %w", err) return err
} }
if stored == "" { if stored == "" {
@@ -78,6 +70,27 @@ func (v *Vaultik) EnsureStorageBinding() error {
errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured)) errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured))
} }
// storageBinding returns the destination the local index is bound to
// ("" when no binding is recorded yet), then the configured destination
// written the same way. It records nothing, so a read-only command can
// call it.
func (v *Vaultik) storageBinding() (string, string, error) {
configured := v.Config.StorageURL
if configured == "" {
// The s3.* form: bind on its endpoint, bucket and prefix,
// written the way storage_url would name the same destination.
s3 := v.Config.S3
configured = "s3://" + s3.Bucket + "/" + s3.Prefix + "?endpoint=" + s3.Endpoint
}
stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
if err != nil {
return "", "", fmt.Errorf("reading local storage binding: %w", err)
}
return stored, configured, nil
}
// buildBindingMismatchMessage assembles the multi-line explanation // buildBindingMismatchMessage assembles the multi-line explanation
// shown when the local index is bound to a different destination than // shown when the local index is bound to a different destination than
// the currently-configured one (the first line lives in the // the currently-configured one (the first line lives in the