check / check (push) Waiting to run
s3.part_size was loaded and defaulted but never reached the S3 client, whose uploader used a fixed 10MiB part. The client now takes the part size from the config, for storage_url and for the s3.* fields, and config load rejects a value below 5MiB or above 5GiB, an explicit 0 included. A blob too large for S3's limit of 10,000 parts at that size is uploaded in larger parts, since the uploader cannot learn the size of the reader it is given. The docs gave the default as 5MB, which the config file reads as 5,000,000 bytes, below the minimum; they now say 5MiB. Judgement call: the 5GiB maximum is enforced along with the 5MiB minimum the issue names. Model: opus-5-5
146 lines
3.9 KiB
Go
146 lines
3.9 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"go.uber.org/fx"
|
|
"sneak.berlin/go/vaultik/internal/config"
|
|
"sneak.berlin/go/vaultik/internal/s3"
|
|
)
|
|
|
|
// defaultS3Region is used when neither the URL nor the config specify one.
|
|
const defaultS3Region = "us-east-1"
|
|
|
|
// defaultS3Endpoint is the AWS endpoint used when none is configured.
|
|
const defaultS3Endpoint = "s3.amazonaws.com"
|
|
|
|
// Module exports storage functionality as an fx module.
|
|
// It provides a Storer implementation based on the configured storage URL
|
|
// or falls back to legacy S3 configuration.
|
|
//
|
|
//nolint:gochecknoglobals // fx module definitions are package globals
|
|
var Module = fx.Module("storage",
|
|
fx.Provide(NewStorer),
|
|
)
|
|
|
|
// NewStorer creates a Storer based on configuration.
|
|
// If StorageURL is set, it uses URL-based configuration.
|
|
// Otherwise, it falls back to legacy S3 configuration.
|
|
//
|
|
//nolint:ireturn // fx provider intentionally returns the Storer interface
|
|
func NewStorer(cfg *config.Config) (Storer, error) {
|
|
if cfg.StorageURL != "" {
|
|
return storerFromURL(cfg.StorageURL, cfg)
|
|
}
|
|
|
|
return storerFromLegacyS3Config(cfg)
|
|
}
|
|
|
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
|
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
|
parsed, err := ParseStorageURL(rawURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing storage URL: %w", err)
|
|
}
|
|
|
|
switch parsed.Scheme {
|
|
case schemeFile:
|
|
return NewFileStorer(parsed.Prefix)
|
|
|
|
case schemeS3:
|
|
return storerFromParsedS3URL(parsed, cfg)
|
|
|
|
case schemeRclone:
|
|
return NewRcloneStorer(
|
|
context.Background(), parsed.RcloneRemote, parsed.Prefix)
|
|
|
|
default:
|
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedStorage, parsed.Scheme)
|
|
}
|
|
}
|
|
|
|
// storerFromParsedS3URL builds an S3 storer from a parsed s3:// URL,
|
|
// filling endpoint protocol and region defaults from the config.
|
|
//
|
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
|
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
|
|
// Build endpoint URL
|
|
endpoint := parsed.Endpoint
|
|
if endpoint == "" {
|
|
endpoint = defaultS3Endpoint
|
|
}
|
|
|
|
// Add protocol if not present
|
|
hasProtocol := strings.HasPrefix(endpoint, "https://") ||
|
|
strings.HasPrefix(endpoint, "http://")
|
|
if !hasProtocol {
|
|
if parsed.UseSSL {
|
|
endpoint = "https://" + endpoint
|
|
} else {
|
|
endpoint = "http://" + endpoint
|
|
}
|
|
}
|
|
|
|
region := parsed.Region
|
|
if region == "" {
|
|
region = cfg.S3.Region
|
|
if region == "" {
|
|
region = defaultS3Region
|
|
}
|
|
}
|
|
|
|
// Credentials come from config (not URL for security)
|
|
client, err := s3.NewClient(context.Background(), s3.Config{
|
|
Endpoint: endpoint,
|
|
Bucket: parsed.Bucket,
|
|
Prefix: parsed.Prefix,
|
|
AccessKeyID: cfg.S3.AccessKeyID,
|
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
|
Region: region,
|
|
PartSize: cfg.S3.PartSize.Int64(),
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating S3 client: %w", err)
|
|
}
|
|
|
|
return NewS3Storer(client), nil
|
|
}
|
|
|
|
//nolint:ireturn // factory intentionally returns the Storer interface
|
|
func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
|
|
endpoint := cfg.S3.Endpoint
|
|
|
|
// Ensure protocol is present. Absent an explicit use_ssl, default to TLS;
|
|
// plain HTTP only when use_ssl is written as false.
|
|
if !strings.HasPrefix(endpoint, "http://") &&
|
|
!strings.HasPrefix(endpoint, "https://") {
|
|
if cfg.S3.UseSSL == nil || *cfg.S3.UseSSL {
|
|
endpoint = "https://" + endpoint
|
|
} else {
|
|
endpoint = "http://" + endpoint
|
|
}
|
|
}
|
|
|
|
region := cfg.S3.Region
|
|
if region == "" {
|
|
region = defaultS3Region
|
|
}
|
|
|
|
client, err := s3.NewClient(context.Background(), s3.Config{
|
|
Endpoint: endpoint,
|
|
Bucket: cfg.S3.Bucket,
|
|
Prefix: cfg.S3.Prefix,
|
|
AccessKeyID: cfg.S3.AccessKeyID,
|
|
SecretAccessKey: cfg.S3.SecretAccessKey,
|
|
Region: region,
|
|
PartSize: cfg.S3.PartSize.Int64(),
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating S3 client: %w", err)
|
|
}
|
|
|
|
return NewS3Storer(client), nil
|
|
}
|