Files
vaultik/internal/storage/module.go
T
sneak 6e3d2e0941
check / check (push) Waiting to run
Pass s3.part_size to the multipart uploader (closes #232)
s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, the part sizes S3
accepts. The docs gave the default as 5MB, which the config file reads
as 5,000,000 bytes, below the minimum; they now say 5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.
Trap: at the 5MiB default the uploader's 10,000-part limit caps one
upload at about 48.8GiB, down from about 97.7GiB; blob_size_limit is
not checked against it.

Model: opus-5-5
2026-10-07 09:18:31 +00:00

146 lines
3.9 KiB
Go

package storage
import (
"context"
"fmt"
"strings"
"go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3"
)
// defaultS3Region is used when neither the URL nor the config specify one.
const defaultS3Region = "us-east-1"
// defaultS3Endpoint is the AWS endpoint used when none is configured.
const defaultS3Endpoint = "s3.amazonaws.com"
// Module exports storage functionality as an fx module.
// It provides a Storer implementation based on the configured storage URL
// or falls back to legacy S3 configuration.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("storage",
fx.Provide(NewStorer),
)
// NewStorer creates a Storer based on configuration.
// If StorageURL is set, it uses URL-based configuration.
// Otherwise, it falls back to legacy S3 configuration.
//
//nolint:ireturn // fx provider intentionally returns the Storer interface
func NewStorer(cfg *config.Config) (Storer, error) {
if cfg.StorageURL != "" {
return storerFromURL(cfg.StorageURL, cfg)
}
return storerFromLegacyS3Config(cfg)
}
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
parsed, err := ParseStorageURL(rawURL)
if err != nil {
return nil, fmt.Errorf("parsing storage URL: %w", err)
}
switch parsed.Scheme {
case schemeFile:
return NewFileStorer(parsed.Prefix)
case schemeS3:
return storerFromParsedS3URL(parsed, cfg)
case schemeRclone:
return NewRcloneStorer(
context.Background(), parsed.RcloneRemote, parsed.Prefix)
default:
return nil, fmt.Errorf("%w: %s", ErrUnsupportedStorage, parsed.Scheme)
}
}
// storerFromParsedS3URL builds an S3 storer from a parsed s3:// URL,
// filling endpoint protocol and region defaults from the config.
//
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
// Build endpoint URL
endpoint := parsed.Endpoint
if endpoint == "" {
endpoint = defaultS3Endpoint
}
// Add protocol if not present
hasProtocol := strings.HasPrefix(endpoint, "https://") ||
strings.HasPrefix(endpoint, "http://")
if !hasProtocol {
if parsed.UseSSL {
endpoint = "https://" + endpoint
} else {
endpoint = "http://" + endpoint
}
}
region := parsed.Region
if region == "" {
region = cfg.S3.Region
if region == "" {
region = defaultS3Region
}
}
// Credentials come from config (not URL for security)
client, err := s3.NewClient(context.Background(), s3.Config{
Endpoint: endpoint,
Bucket: parsed.Bucket,
Prefix: parsed.Prefix,
AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey,
Region: region,
PartSize: cfg.S3.PartSize.Int64(),
})
if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err)
}
return NewS3Storer(client), nil
}
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
endpoint := cfg.S3.Endpoint
// Ensure protocol is present. Absent an explicit use_ssl, default to TLS;
// plain HTTP only when use_ssl is written as false.
if !strings.HasPrefix(endpoint, "http://") &&
!strings.HasPrefix(endpoint, "https://") {
if cfg.S3.UseSSL == nil || *cfg.S3.UseSSL {
endpoint = "https://" + endpoint
} else {
endpoint = "http://" + endpoint
}
}
region := cfg.S3.Region
if region == "" {
region = defaultS3Region
}
client, err := s3.NewClient(context.Background(), s3.Config{
Endpoint: endpoint,
Bucket: cfg.S3.Bucket,
Prefix: cfg.S3.Prefix,
AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey,
Region: region,
PartSize: cfg.S3.PartSize.Int64(),
})
if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err)
}
return NewS3Storer(client), nil
}