package storage import ( "errors" "fmt" "net/url" "slices" "strings" ) // Storage URL scheme names. const ( schemeFile = "file" schemeS3 = "s3" schemeRclone = "rclone" ) // Sentinel errors for storage URL parsing. var ( ErrEmptyStorageURL = errors.New("storage URL is empty") ErrEmptyFilePath = errors.New("file URL path is empty") ErrRelativeFilePath = errors.New( "file URL needs an absolute path, as in file:///path/to/backup") ErrMissingBucket = errors.New("s3 URL missing bucket name") ErrMissingRemote = errors.New("rclone URL missing remote name") ErrUnsupportedScheme = errors.New( "unsupported URL scheme: must start with s3://, file://, or rclone://") ErrUnsupportedStorage = errors.New("unsupported storage scheme") ErrURLCredentials = errors.New( "storage URL must not carry credentials; " + "set s3.access_key_id and s3.secret_access_key in the config instead") ErrURLUnknownParam = errors.New("unknown query parameter in storage URL") ErrRcloneRemoteParams = errors.New( "rclone URL must not carry parameters in the remote name; " + "set them on the remote with rclone config instead") ) // URL represents a parsed storage URL. type URL struct { Scheme string // "s3", "file", or "rclone" Bucket string // S3 bucket name (empty for file/rclone) Prefix string // Path within bucket or filesystem base path Endpoint string // S3 endpoint (optional, default AWS) Region string // S3 region (optional) UseSSL bool // Use HTTPS for S3 (default true) RcloneRemote string // rclone remote name (for rclone:// URLs) } // ParseStorageURL parses a storage URL string. // Supported formats: // - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true // - file:///absolute/path/to/backup // - rclone://remote/path/to/backups func ParseStorageURL(rawURL string) (*URL, error) { if rawURL == "" { return nil, ErrEmptyStorageURL } if after, ok := strings.CutPrefix(rawURL, "file://"); ok { return parseFileURL(after) } if strings.HasPrefix(rawURL, "s3://") { return parseS3URL(rawURL) } if strings.HasPrefix(rawURL, "rclone://") { return parseRcloneURL(rawURL) } return nil, ErrUnsupportedScheme } // parseFileURL parses the path that follows file://. The path must be // absolute: a relative one would resolve against the working directory, // so the destination would move with it. Percent-escapes are decoded, so // my%20stick names the directory "my stick"; a literal space is kept. func parseFileURL(rawPath string) (*URL, error) { if rawPath == "" { return nil, ErrEmptyFilePath } if !strings.HasPrefix(rawPath, "/") { return nil, ErrRelativeFilePath } path, err := url.PathUnescape(rawPath) if err != nil { return nil, fmt.Errorf("invalid file URL path: %w", err) } return &URL{Scheme: schemeFile, Prefix: path}, nil } // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in // the userinfo or in the endpoint parameter's userinfo, and any query // parameter other than endpoint, region and ssl, so a credential-bearing // URL is never stored or echoed. func parseS3URL(rawURL string) (*URL, error) { u, err := url.Parse(rawURL) if err != nil { return nil, wrapParseError(err) } if u.User != nil { return nil, ErrURLCredentials } bucket := u.Host if bucket == "" { return nil, ErrMissingBucket } query := u.Query() err = rejectUnknownParams(query, "endpoint", "region", "ssl") if err != nil { return nil, err } // A host name never contains "@", so one in an endpoint, with or // without a scheme, can only introduce userinfo. for _, endpoint := range query["endpoint"] { if strings.Contains(endpoint, "@") { return nil, ErrURLCredentials } } return &URL{ Scheme: schemeS3, Bucket: bucket, Prefix: strings.TrimPrefix(u.Path, "/"), Endpoint: query.Get("endpoint"), Region: query.Get("region"), UseSSL: query.Get("ssl") != "false", }, nil } // parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no // query parameters, so credentials in the userinfo, parameters in the // remote name and any query parameter at all are rejected rather than // silently ignored. func parseRcloneURL(rawURL string) (*URL, error) { // The remote name runs up to the first "/", "?" or "#". It never // contains a comma; one starts rclone's remote,param=value form, // whose values can be credentials. This is checked before url.Parse, // whose error for such a name can print part of it. remote := strings.TrimPrefix(rawURL, "rclone://") if end := strings.IndexAny(remote, "/?#"); end >= 0 { remote = remote[:end] } if strings.Contains(remote, ",") { return nil, ErrRcloneRemoteParams } u, err := url.Parse(rawURL) if err != nil { return nil, wrapParseError(err) } if u.User != nil { return nil, ErrURLCredentials } remote = u.Host if remote == "" { return nil, ErrMissingRemote } err = rejectUnknownParams(u.Query()) if err != nil { return nil, err } return &URL{ Scheme: schemeRclone, Prefix: strings.TrimPrefix(u.Path, "/"), RcloneRemote: remote, }, nil } // rejectUnknownParams returns an error naming the first query parameter // not in allowed. The parameter's name is included (so a misspelt // endpoint= is caught), but never its value, which could be a secret, // and never the whole URL. func rejectUnknownParams(query url.Values, allowed ...string) error { for name := range query { if !slices.Contains(allowed, name) { return fmt.Errorf( "%w: %q; put credentials in s3.access_key_id and "+ "s3.secret_access_key, not the URL", ErrURLUnknownParam, name) } } return nil } // wrapParseError wraps only the inner cause of a url.Parse failure. The // *url.Error that url.Parse returns embeds the raw URL in its message, so // wrapping it directly would echo a credential-bearing URL into logs. func wrapParseError(err error) error { if uerr, ok := errors.AsType[*url.Error](err); ok { return fmt.Errorf("invalid URL: %w", uerr.Err) } return fmt.Errorf("invalid URL: %w", err) } // String returns a human-readable representation of the storage URL. func (u *URL) String() string { switch u.Scheme { case schemeFile: return "file://" + u.Prefix case schemeS3: endpoint := u.Endpoint if endpoint == "" { endpoint = "s3.amazonaws.com" } if u.Prefix != "" { return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint) } return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) case schemeRclone: if u.Prefix != "" { return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) } return "rclone://" + u.RcloneRemote default: return u.Scheme + "://?" } }