Check storage_url with the storage parser at config load (closes #468)
check / check (push) Waiting to run
check / check (push) Waiting to run
config set accepted rclone:// values that every command then refused: no remote name, parameters in the remote name, a query parameter. The config only checked the URL's prefix, because config could not import the parser from storage, which imports config. The parser moves from internal/storage to a new package, internal/storageurl, that both import. ParseStorageURL becomes storageurl.Parse. Loading the config now runs it on every storage_url, so config set refuses such a value and leaves the file unchanged. Model: opus-5-5
This commit was merged in pull request #472.
This commit is contained in:
1 parent
be362080d3
commit
57b5cb327b
8 files changed
+133
-129
No files matched your search
@@ -0,0 +1,253 @@
|
||||
// Package storageurl parses the storage_url config value. The config
|
||||
// package uses it to refuse a bad value when the config loads; the storage
|
||||
// package uses it to open the destination.
|
||||
package storageurl
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Storage URL scheme names.
|
||||
const (
|
||||
SchemeFile = "file"
|
||||
SchemeS3 = "s3"
|
||||
SchemeRclone = "rclone"
|
||||
)
|
||||
|
||||
// Sentinel errors for storage URL parsing.
|
||||
var (
|
||||
ErrEmptyStorageURL = errors.New("storage URL is empty")
|
||||
ErrEmptyFilePath = errors.New("file URL path is empty")
|
||||
ErrRelativeFilePath = errors.New(
|
||||
"file URL needs an absolute path, as in file:///path/to/backup")
|
||||
ErrMissingBucket = errors.New("s3 URL missing bucket name")
|
||||
ErrMissingRemote = errors.New("rclone URL missing remote name")
|
||||
ErrUnsupportedScheme = errors.New(
|
||||
"unsupported URL scheme: must start with s3://, file://, or rclone://")
|
||||
ErrURLCredentials = errors.New(
|
||||
"storage URL must not carry credentials; " +
|
||||
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
||||
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
||||
ErrURLBadQuery = errors.New(
|
||||
"storage URL query cannot be decoded; " +
|
||||
"check it for a ; or a % not followed by two hex digits")
|
||||
ErrRcloneRemoteParams = errors.New(
|
||||
"rclone URL must not carry parameters in the remote name; " +
|
||||
"set them on the remote with rclone config instead")
|
||||
)
|
||||
|
||||
// URL represents a parsed storage URL.
|
||||
type URL struct {
|
||||
Scheme string // "s3", "file", or "rclone"
|
||||
Bucket string // S3 bucket name (empty for file/rclone)
|
||||
Prefix string // Path within bucket or filesystem base path
|
||||
Endpoint string // S3 endpoint (optional, default AWS)
|
||||
Region string // S3 region (optional)
|
||||
UseSSL bool // Use HTTPS for S3 (default true)
|
||||
RcloneRemote string // rclone remote name (for rclone:// URLs)
|
||||
}
|
||||
|
||||
// Parse parses a storage URL string.
|
||||
// Supported formats:
|
||||
// - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true
|
||||
// - file:///absolute/path/to/backup
|
||||
// - rclone://remote/path/to/backups
|
||||
func Parse(rawURL string) (*URL, error) {
|
||||
if rawURL == "" {
|
||||
return nil, ErrEmptyStorageURL
|
||||
}
|
||||
|
||||
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
||||
return parseFileURL(after)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(rawURL, "s3://") {
|
||||
return parseS3URL(rawURL)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(rawURL, "rclone://") {
|
||||
return parseRcloneURL(rawURL)
|
||||
}
|
||||
|
||||
return nil, ErrUnsupportedScheme
|
||||
}
|
||||
|
||||
// parseFileURL parses the path that follows file://. The path must be
|
||||
// absolute: a relative one would resolve against the working directory,
|
||||
// so the destination would move with it. Percent-escapes are decoded, so
|
||||
// my%20stick names the directory "my stick"; a literal space is kept.
|
||||
func parseFileURL(rawPath string) (*URL, error) {
|
||||
if rawPath == "" {
|
||||
return nil, ErrEmptyFilePath
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(rawPath, "/") {
|
||||
return nil, ErrRelativeFilePath
|
||||
}
|
||||
|
||||
path, err := url.PathUnescape(rawPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid file URL path: %w", err)
|
||||
}
|
||||
|
||||
return &URL{Scheme: SchemeFile, Prefix: path}, nil
|
||||
}
|
||||
|
||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||
// the userinfo or in the endpoint parameter's userinfo, a query that
|
||||
// cannot be decoded, and any query parameter other than endpoint, region
|
||||
// and ssl, so a credential-bearing URL is never stored or echoed.
|
||||
func parseS3URL(rawURL string) (*URL, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return nil, wrapParseError(err)
|
||||
}
|
||||
|
||||
if u.User != nil {
|
||||
return nil, ErrURLCredentials
|
||||
}
|
||||
|
||||
bucket := u.Host
|
||||
if bucket == "" {
|
||||
return nil, ErrMissingBucket
|
||||
}
|
||||
|
||||
// u.Query() would silently drop a part it cannot decode, endpoint
|
||||
// included. The decode error is not wrapped: it can quote the value.
|
||||
query, err := url.ParseQuery(u.RawQuery)
|
||||
if err != nil {
|
||||
return nil, ErrURLBadQuery
|
||||
}
|
||||
|
||||
err = rejectUnknownParams(query, "endpoint", "region", "ssl")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// A host name never contains "@", so one in an endpoint, with or
|
||||
// without a scheme, can only introduce userinfo.
|
||||
for _, endpoint := range query["endpoint"] {
|
||||
if strings.Contains(endpoint, "@") {
|
||||
return nil, ErrURLCredentials
|
||||
}
|
||||
}
|
||||
|
||||
return &URL{
|
||||
Scheme: SchemeS3,
|
||||
Bucket: bucket,
|
||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||
Endpoint: query.Get("endpoint"),
|
||||
Region: query.Get("region"),
|
||||
UseSSL: query.Get("ssl") != "false",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
|
||||
// query parameters, so credentials in the userinfo, parameters in the
|
||||
// remote name, a query that cannot be decoded and any query parameter at
|
||||
// all are rejected rather than silently ignored.
|
||||
func parseRcloneURL(rawURL string) (*URL, error) {
|
||||
// The remote name runs up to the first "/", "?" or "#". It never
|
||||
// contains a comma; one starts rclone's remote,param=value form,
|
||||
// whose values can be credentials. This is checked before url.Parse,
|
||||
// whose error for such a name can print part of it.
|
||||
remote := strings.TrimPrefix(rawURL, "rclone://")
|
||||
if end := strings.IndexAny(remote, "/?#"); end >= 0 {
|
||||
remote = remote[:end]
|
||||
}
|
||||
|
||||
if strings.Contains(remote, ",") {
|
||||
return nil, ErrRcloneRemoteParams
|
||||
}
|
||||
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return nil, wrapParseError(err)
|
||||
}
|
||||
|
||||
if u.User != nil {
|
||||
return nil, ErrURLCredentials
|
||||
}
|
||||
|
||||
remote = u.Host
|
||||
if remote == "" {
|
||||
return nil, ErrMissingRemote
|
||||
}
|
||||
|
||||
// As in parseS3URL: u.Query() would drop a part it cannot decode, and
|
||||
// the decode error can quote the value.
|
||||
query, err := url.ParseQuery(u.RawQuery)
|
||||
if err != nil {
|
||||
return nil, ErrURLBadQuery
|
||||
}
|
||||
|
||||
err = rejectUnknownParams(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &URL{
|
||||
Scheme: SchemeRclone,
|
||||
Prefix: strings.TrimPrefix(u.Path, "/"),
|
||||
RcloneRemote: remote,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// rejectUnknownParams returns an error naming the first query parameter
|
||||
// not in allowed. The parameter's name is included (so a misspelt
|
||||
// endpoint= is caught), but never its value, which could be a secret,
|
||||
// and never the whole URL.
|
||||
func rejectUnknownParams(query url.Values, allowed ...string) error {
|
||||
for name := range query {
|
||||
if !slices.Contains(allowed, name) {
|
||||
return fmt.Errorf(
|
||||
"%w: %q; put credentials in s3.access_key_id and "+
|
||||
"s3.secret_access_key, not the URL",
|
||||
ErrURLUnknownParam, name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// wrapParseError wraps only the inner cause of a url.Parse failure. The
|
||||
// *url.Error that url.Parse returns embeds the raw URL in its message, so
|
||||
// wrapping it directly would echo a credential-bearing URL into logs.
|
||||
func wrapParseError(err error) error {
|
||||
if uerr, ok := errors.AsType[*url.Error](err); ok {
|
||||
return fmt.Errorf("invalid URL: %w", uerr.Err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
// String returns a human-readable representation of the storage URL.
|
||||
func (u *URL) String() string {
|
||||
switch u.Scheme {
|
||||
case SchemeFile:
|
||||
return "file://" + u.Prefix
|
||||
case SchemeS3:
|
||||
endpoint := u.Endpoint
|
||||
if endpoint == "" {
|
||||
endpoint = "s3.amazonaws.com"
|
||||
}
|
||||
|
||||
if u.Prefix != "" {
|
||||
return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
||||
case SchemeRclone:
|
||||
if u.Prefix != "" {
|
||||
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
||||
}
|
||||
|
||||
return "rclone://" + u.RcloneRemote
|
||||
default:
|
||||
return u.Scheme + "://?"
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user