Check storage_url with the storage parser at config load (closes #468)
check / check (push) Waiting to run

config set accepted rclone:// values that every command then refused:
no remote name, parameters in the remote name, a query parameter. The
config only checked the URL's prefix, because config could not import
the parser from storage, which imports config.

The parser moves from internal/storage to a new package,
internal/storageurl, that both import. ParseStorageURL becomes
storageurl.Parse. Loading the config now runs it on every storage_url,
so config set refuses such a value and leaves the file unchanged.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-19 07:24:47 +00:00
1 parent d747ddef3a
commit 4e084a2746
8 files changed
+133 -129

No files matched your search

+2 -1
View File
@@ -9,6 +9,7 @@ import (
"strings"
"github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/storageurl"
)
// FileStorer implements Storer using the local filesystem.
@@ -288,7 +289,7 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
// Info returns human-readable storage location information.
func (f *FileStorer) Info() Info {
return Info{
Type: schemeFile,
Type: storageurl.SchemeFile,
Location: f.basePath,
}
}
+11 -5
View File
@@ -2,14 +2,20 @@ package storage
import (
"context"
"errors"
"fmt"
"strings"
"go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storageurl"
)
// ErrUnsupportedStorage is returned for a parsed storage URL whose scheme
// has no storage backend.
var ErrUnsupportedStorage = errors.New("unsupported storage scheme")
// defaultS3Region is used when neither the URL nor the config specify one.
const defaultS3Region = "us-east-1"
@@ -40,19 +46,19 @@ func NewStorer(cfg *config.Config) (Storer, error) {
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
parsed, err := ParseStorageURL(rawURL)
parsed, err := storageurl.Parse(rawURL)
if err != nil {
return nil, fmt.Errorf("parsing storage URL: %w", err)
}
switch parsed.Scheme {
case schemeFile:
case storageurl.SchemeFile:
return NewFileStorer(parsed.Prefix)
case schemeS3:
case storageurl.SchemeS3:
return storerFromParsedS3URL(parsed, cfg)
case schemeRclone:
case storageurl.SchemeRclone:
return NewRcloneStorer(
context.Background(), parsed.RcloneRemote, parsed.Prefix)
@@ -65,7 +71,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
// filling endpoint protocol and region defaults from the config.
//
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
func storerFromParsedS3URL(parsed *storageurl.URL, cfg *config.Config) (Storer, error) {
// Build endpoint URL
endpoint := parsed.Endpoint
if endpoint == "" {
+2 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/operations"
"sneak.berlin/go/vaultik/internal/storageurl"
// Import all rclone backends
_ "github.com/rclone/rclone/backend/all"
@@ -266,7 +267,7 @@ func (r *RcloneStorer) Info() Info {
}
return Info{
Type: schemeRclone,
Type: storageurl.SchemeRclone,
Location: location,
}
}
-251
View File
@@ -1,251 +0,0 @@
package storage
import (
"errors"
"fmt"
"net/url"
"slices"
"strings"
)
// Storage URL scheme names.
const (
schemeFile = "file"
schemeS3 = "s3"
schemeRclone = "rclone"
)
// Sentinel errors for storage URL parsing.
var (
ErrEmptyStorageURL = errors.New("storage URL is empty")
ErrEmptyFilePath = errors.New("file URL path is empty")
ErrRelativeFilePath = errors.New(
"file URL needs an absolute path, as in file:///path/to/backup")
ErrMissingBucket = errors.New("s3 URL missing bucket name")
ErrMissingRemote = errors.New("rclone URL missing remote name")
ErrUnsupportedScheme = errors.New(
"unsupported URL scheme: must start with s3://, file://, or rclone://")
ErrUnsupportedStorage = errors.New("unsupported storage scheme")
ErrURLCredentials = errors.New(
"storage URL must not carry credentials; " +
"set s3.access_key_id and s3.secret_access_key in the config instead")
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
ErrURLBadQuery = errors.New(
"storage URL query cannot be decoded; " +
"check it for a ; or a % not followed by two hex digits")
ErrRcloneRemoteParams = errors.New(
"rclone URL must not carry parameters in the remote name; " +
"set them on the remote with rclone config instead")
)
// URL represents a parsed storage URL.
type URL struct {
Scheme string // "s3", "file", or "rclone"
Bucket string // S3 bucket name (empty for file/rclone)
Prefix string // Path within bucket or filesystem base path
Endpoint string // S3 endpoint (optional, default AWS)
Region string // S3 region (optional)
UseSSL bool // Use HTTPS for S3 (default true)
RcloneRemote string // rclone remote name (for rclone:// URLs)
}
// ParseStorageURL parses a storage URL string.
// Supported formats:
// - s3://bucket/prefix?endpoint=host&region=us-east-1&ssl=true
// - file:///absolute/path/to/backup
// - rclone://remote/path/to/backups
func ParseStorageURL(rawURL string) (*URL, error) {
if rawURL == "" {
return nil, ErrEmptyStorageURL
}
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
return parseFileURL(after)
}
if strings.HasPrefix(rawURL, "s3://") {
return parseS3URL(rawURL)
}
if strings.HasPrefix(rawURL, "rclone://") {
return parseRcloneURL(rawURL)
}
return nil, ErrUnsupportedScheme
}
// parseFileURL parses the path that follows file://. The path must be
// absolute: a relative one would resolve against the working directory,
// so the destination would move with it. Percent-escapes are decoded, so
// my%20stick names the directory "my stick"; a literal space is kept.
func parseFileURL(rawPath string) (*URL, error) {
if rawPath == "" {
return nil, ErrEmptyFilePath
}
if !strings.HasPrefix(rawPath, "/") {
return nil, ErrRelativeFilePath
}
path, err := url.PathUnescape(rawPath)
if err != nil {
return nil, fmt.Errorf("invalid file URL path: %w", err)
}
return &URL{Scheme: schemeFile, Prefix: path}, nil
}
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
// the userinfo or in the endpoint parameter's userinfo, a query that
// cannot be decoded, and any query parameter other than endpoint, region
// and ssl, so a credential-bearing URL is never stored or echoed.
func parseS3URL(rawURL string) (*URL, error) {
u, err := url.Parse(rawURL)
if err != nil {
return nil, wrapParseError(err)
}
if u.User != nil {
return nil, ErrURLCredentials
}
bucket := u.Host
if bucket == "" {
return nil, ErrMissingBucket
}
// u.Query() would silently drop a part it cannot decode, endpoint
// included. The decode error is not wrapped: it can quote the value.
query, err := url.ParseQuery(u.RawQuery)
if err != nil {
return nil, ErrURLBadQuery
}
err = rejectUnknownParams(query, "endpoint", "region", "ssl")
if err != nil {
return nil, err
}
// A host name never contains "@", so one in an endpoint, with or
// without a scheme, can only introduce userinfo.
for _, endpoint := range query["endpoint"] {
if strings.Contains(endpoint, "@") {
return nil, ErrURLCredentials
}
}
return &URL{
Scheme: schemeS3,
Bucket: bucket,
Prefix: strings.TrimPrefix(u.Path, "/"),
Endpoint: query.Get("endpoint"),
Region: query.Get("region"),
UseSSL: query.Get("ssl") != "false",
}, nil
}
// parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
// query parameters, so credentials in the userinfo, parameters in the
// remote name, a query that cannot be decoded and any query parameter at
// all are rejected rather than silently ignored.
func parseRcloneURL(rawURL string) (*URL, error) {
// The remote name runs up to the first "/", "?" or "#". It never
// contains a comma; one starts rclone's remote,param=value form,
// whose values can be credentials. This is checked before url.Parse,
// whose error for such a name can print part of it.
remote := strings.TrimPrefix(rawURL, "rclone://")
if end := strings.IndexAny(remote, "/?#"); end >= 0 {
remote = remote[:end]
}
if strings.Contains(remote, ",") {
return nil, ErrRcloneRemoteParams
}
u, err := url.Parse(rawURL)
if err != nil {
return nil, wrapParseError(err)
}
if u.User != nil {
return nil, ErrURLCredentials
}
remote = u.Host
if remote == "" {
return nil, ErrMissingRemote
}
// As in parseS3URL: u.Query() would drop a part it cannot decode, and
// the decode error can quote the value.
query, err := url.ParseQuery(u.RawQuery)
if err != nil {
return nil, ErrURLBadQuery
}
err = rejectUnknownParams(query)
if err != nil {
return nil, err
}
return &URL{
Scheme: schemeRclone,
Prefix: strings.TrimPrefix(u.Path, "/"),
RcloneRemote: remote,
}, nil
}
// rejectUnknownParams returns an error naming the first query parameter
// not in allowed. The parameter's name is included (so a misspelt
// endpoint= is caught), but never its value, which could be a secret,
// and never the whole URL.
func rejectUnknownParams(query url.Values, allowed ...string) error {
for name := range query {
if !slices.Contains(allowed, name) {
return fmt.Errorf(
"%w: %q; put credentials in s3.access_key_id and "+
"s3.secret_access_key, not the URL",
ErrURLUnknownParam, name)
}
}
return nil
}
// wrapParseError wraps only the inner cause of a url.Parse failure. The
// *url.Error that url.Parse returns embeds the raw URL in its message, so
// wrapping it directly would echo a credential-bearing URL into logs.
func wrapParseError(err error) error {
if uerr, ok := errors.AsType[*url.Error](err); ok {
return fmt.Errorf("invalid URL: %w", uerr.Err)
}
return fmt.Errorf("invalid URL: %w", err)
}
// String returns a human-readable representation of the storage URL.
func (u *URL) String() string {
switch u.Scheme {
case schemeFile:
return "file://" + u.Prefix
case schemeS3:
endpoint := u.Endpoint
if endpoint == "" {
endpoint = "s3.amazonaws.com"
}
if u.Prefix != "" {
return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint)
}
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
case schemeRclone:
if u.Prefix != "" {
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
}
return "rclone://" + u.RcloneRemote
default:
return u.Scheme + "://?"
}
}
-341
View File
@@ -1,341 +0,0 @@
package storage_test
import (
"errors"
"reflect"
"strings"
"testing"
"sneak.berlin/go/vaultik/internal/storage"
)
// TestParseStorageURLValid checks that each supported scheme parses into
// the expected fields, since those fields decide which backend is built.
func TestParseStorageURLValid(t *testing.T) {
t.Parallel()
const (
bucket = "mybucket"
fileScheme = "file"
)
cases := []struct {
name string
raw string
want *storage.URL
}{
{
name: "file absolute path",
raw: "file:///var/backups/vaultik",
want: &storage.URL{Scheme: fileScheme, Prefix: "/var/backups/vaultik"},
},
{
name: "file percent-escaped path is decoded",
raw: "file:///Volumes/my%20stick/backup",
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
},
{
name: "file path with a literal space",
raw: "file:///Volumes/my stick/backup",
want: &storage.URL{Scheme: fileScheme, Prefix: "/Volumes/my stick/backup"},
},
{
name: "s3 bucket and prefix, ssl defaults on",
raw: "s3://mybucket/backups/host",
want: &storage.URL{
Scheme: "s3", Bucket: bucket,
Prefix: "backups/host", UseSSL: true,
},
},
{
name: "s3 bucket only",
raw: "s3://mybucket",
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
},
{
name: "s3 with endpoint, region, ssl off",
raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false",
want: &storage.URL{
Scheme: "s3", Bucket: bucket,
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
},
},
{
name: "rclone remote and path",
raw: "rclone://gdrive/backups/host",
want: &storage.URL{
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
},
},
{
name: "rclone remote only",
raw: "rclone://gdrive",
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := storage.ParseStorageURL(tc.raw)
if err != nil {
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
}
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
}
})
}
}
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
// inputs fail with the documented sentinel errors instead of parsing to a
// wrong destination.
func TestParseStorageURLErrors(t *testing.T) {
t.Parallel()
cases := []struct {
name string
raw string
wantErr error
}{
{"empty url", "", storage.ErrEmptyStorageURL},
{"file empty path", "file://", storage.ErrEmptyFilePath},
{"file relative path", "file://dest", storage.ErrRelativeFilePath},
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, tc.wantErr) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, tc.wantErr)
}
})
}
}
// TestParseStorageURLRejectsCredentials checks that a URL carrying
// credentials in its userinfo or in an unknown query parameter is
// rejected, and that the error never echoes the secret-bearing URL back
// into logs or output.
func TestParseStorageURLRejectsCredentials(t *testing.T) {
t.Parallel()
// Split so the literals never form a "user:pass@" URL pattern that
// tooling would flag as a real hardcoded credential.
const (
key = "AKIAKEY"
secret = "topsecret"
)
cases := []struct {
name string
raw string
wantErr error
secrets []string // must not appear in the error message
}{
{
name: "s3 userinfo",
raw: "s3://" + key + ":" + secret + "@mybucket/prefix",
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret, "mybucket"},
},
{
name: "s3 userinfo in endpoint",
raw: "s3://mybucket?endpoint=https://" +
key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret},
},
{
name: "s3 userinfo in endpoint without scheme",
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
wantErr: storage.ErrURLCredentials,
secrets: []string{key, secret},
},
{
name: "s3 unknown query param",
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
wantErr: storage.ErrURLUnknownParam,
secrets: []string{key, secret},
},
{
name: "s3 misspelt endpoint",
raw: "s3://mybucket?endpiont=minio.example.com",
wantErr: storage.ErrURLUnknownParam,
secrets: nil,
},
{
name: "rclone userinfo",
raw: "rclone://user:" + secret + "@gdrive/backups",
wantErr: storage.ErrURLCredentials,
secrets: []string{secret},
},
{
name: "rclone query param",
raw: "rclone://gdrive/backups?token=" + secret,
wantErr: storage.ErrURLUnknownParam,
secrets: []string{secret},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, tc.wantErr) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, tc.wantErr)
}
// The rejection must name the proper config keys so the
// operator knows where credentials belong.
for _, key := range []string{"s3.access_key_id", "s3.secret_access_key"} {
if !strings.Contains(err.Error(), key) {
t.Errorf("error %q does not name %q", err.Error(), key)
}
}
for _, secret := range tc.secrets {
if strings.Contains(err.Error(), secret) {
t.Errorf("error message leaked %q: %v", secret, err.Error())
}
}
})
}
}
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
// remote name carrying parameters (rclone's "remote,param=value" form,
// which can hold a credential) is rejected without echoing the value,
// including the URLs that url.Parse itself refuses.
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
t.Parallel()
cases := []struct {
name string
raw string
secrets []string // must not appear in the error message
}{
{
name: "configured remote",
raw: "rclone://gdrive,client_secret=topsecret/backups",
secrets: []string{"topsecret"},
},
{
name: "remote defined inline",
raw: "rclone://:s3,access_key_id=KEY,secret_access_key=SECRET/bucket",
secrets: []string{"KEY", "SECRET"},
},
{
name: "colon in a parameter value",
raw: "rclone://gdrive,client_secret=ab:cd/backups",
secrets: []string{"cd"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, storage.ErrRcloneRemoteParams)
}
for _, secret := range tc.secrets {
if strings.Contains(err.Error(), secret) {
t.Errorf("error message leaked %q: %v", secret, err.Error())
}
}
})
}
}
// TestParseStorageURLRejectsUndecodableQuery checks that an s3 or rclone
// URL whose query cannot be decoded is rejected, not read with that part
// of the query left out, and that the error does not echo it.
func TestParseStorageURLRejectsUndecodableQuery(t *testing.T) {
t.Parallel()
const (
key = "AKIAKEY"
secret = "topsecret"
)
cases := []struct {
name string
raw string
}{
{
name: "semicolon",
raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
"@minio.example.com:9000;region=us-east-1",
},
{
name: "bad percent escape",
raw: "s3://mybucket/prefix?endpoint=https://" + key + ":" + secret +
"%zz@minio.example.com:9000",
},
{
name: "rclone semicolon",
raw: "rclone://gdrive/backups?token=" + secret + ";x=1",
},
{
name: "rclone bad percent escape",
raw: "rclone://gdrive/backups?token=" + secret + "%zz",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", tc.raw)
}
if !errors.Is(err, storage.ErrURLBadQuery) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, storage.ErrURLBadQuery)
}
for _, s := range []string{key, secret, "minio.example.com"} {
if strings.Contains(err.Error(), s) {
t.Errorf("error message leaked %q: %v", s, err.Error())
}
}
})
}
}
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
// itself fails, the wrapped error carries only the inner cause, not the
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
func TestParseStorageURLParseFailureHidesURL(t *testing.T) {
t.Parallel()
const raw = "s3://mybucket/%zz"
_, err := storage.ParseStorageURL(raw)
if err == nil {
t.Fatalf("ParseStorageURL(%q) returned no error", raw)
}
if strings.Contains(err.Error(), "mybucket") {
t.Errorf("error message echoed the raw URL: %v", err.Error())
}
}