package storage import ( "errors" "fmt" "net/url" "slices" "strings" ) // Storage URL scheme names. const ( schemeFile = "file" schemeS3 = "s3" schemeRclone = "rclone" ) // Sentinel errors for storage URL parsing. var ( ErrEmptyStorageURL = errors.New("storage URL is empty") ErrEmptyFilePath = errors.New("file URL path is empty") ErrMissingBucket = errors.New("s3 URL missing bucket name") ErrMissingRemote = errors.New("rclone URL missing remote name") ErrUnsupportedScheme = errors.New( "unsupported URL scheme: must start with s3://, file://, or rclone://") ErrUnsupportedStorage = errors.New("unsupported storage scheme") ErrURLCredentials = errors.New( "storage URL must not carry credentials; " + "set s3.access_key_id and s3.secret_access_key in the config instead") ErrURLUnknownParam = errors.New("unknown query parameter in storage URL") ) // URL represents a parsed storage URL. type URL struct { Scheme string // "s3", "file", or "rclone" Bucket string // S3 bucket name (empty for file/rclone) Prefix string // Path within bucket or filesystem base path Endpoint string // S3 endpoint (optional, default AWS) Region string // S3 region (optional) UseSSL bool // Use HTTPS for S3 (default true) RcloneRemote string // rclone remote name (for rclone:// URLs) } // ParseStorageURL parses a storage URL string. // Supported formats: // - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true // - file:///absolute/path/to/backup // - rclone://remote/path/to/backups func ParseStorageURL(rawURL string) (*URL, error) { if rawURL == "" { return nil, ErrEmptyStorageURL } // Handle file:// URLs if after, ok := strings.CutPrefix(rawURL, "file://"); ok { path := after if path == "" { return nil, ErrEmptyFilePath } return &URL{ Scheme: schemeFile, Prefix: path, }, nil } if strings.HasPrefix(rawURL, "s3://") { return parseS3URL(rawURL) } if strings.HasPrefix(rawURL, "rclone://") { return parseRcloneURL(rawURL) } return nil, ErrUnsupportedScheme } // parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in // the userinfo and any query parameter other than endpoint, region and // ssl, so a credential-bearing URL is never stored or echoed. func parseS3URL(rawURL string) (*URL, error) { u, err := url.Parse(rawURL) if err != nil { return nil, wrapParseError(err) } if u.User != nil { return nil, ErrURLCredentials } bucket := u.Host if bucket == "" { return nil, ErrMissingBucket } query := u.Query() err = rejectUnknownParams(query, "endpoint", "region", "ssl") if err != nil { return nil, err } return &URL{ Scheme: schemeS3, Bucket: bucket, Prefix: strings.TrimPrefix(u.Path, "/"), Endpoint: query.Get("endpoint"), Region: query.Get("region"), UseSSL: query.Get("ssl") != "false", }, nil } // parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no // query parameters, so credentials in the userinfo and any parameter at // all are rejected rather than silently ignored. func parseRcloneURL(rawURL string) (*URL, error) { u, err := url.Parse(rawURL) if err != nil { return nil, wrapParseError(err) } if u.User != nil { return nil, ErrURLCredentials } remote := u.Host if remote == "" { return nil, ErrMissingRemote } err = rejectUnknownParams(u.Query()) if err != nil { return nil, err } return &URL{ Scheme: schemeRclone, Prefix: strings.TrimPrefix(u.Path, "/"), RcloneRemote: remote, }, nil } // rejectUnknownParams returns an error naming the first query parameter // not in allowed. The parameter's name is included (so a misspelt // endpoint= is caught), but never its value, which could be a secret, // and never the whole URL. func rejectUnknownParams(query url.Values, allowed ...string) error { for name := range query { if !slices.Contains(allowed, name) { return fmt.Errorf( "%w: %q; put credentials in s3.access_key_id and "+ "s3.secret_access_key, not the URL", ErrURLUnknownParam, name) } } return nil } // wrapParseError wraps only the inner cause of a url.Parse failure. The // *url.Error that url.Parse returns embeds the raw URL in its message, so // wrapping it directly would echo a credential-bearing URL into logs. func wrapParseError(err error) error { var uerr *url.Error if errors.As(err, &uerr) { return fmt.Errorf("invalid URL: %w", uerr.Err) } return fmt.Errorf("invalid URL: %w", err) } // String returns a human-readable representation of the storage URL. func (u *URL) String() string { switch u.Scheme { case schemeFile: return "file://" + u.Prefix case schemeS3: endpoint := u.Endpoint if endpoint == "" { endpoint = "s3.amazonaws.com" } if u.Prefix != "" { return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint) } return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) case schemeRclone: if u.Prefix != "" { return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) } return "rclone://" + u.RcloneRemote default: return u.Scheme + "://?" } }