Files
vaultik/internal/config/config.go
T
clawbot 8b22ae8d42
check / check (push) Waiting to run
Pass s3.part_size to the multipart uploader (closes #232)
s3.part_size was loaded and defaulted but never reached the S3 client,
whose uploader used a fixed 10MiB part. The client now takes the part
size from the config, for storage_url and for the s3.* fields, and
config load rejects a value below 5MiB or above 5GiB, an explicit 0
included. A blob too large for S3's limit of 10,000 parts at that size
is uploaded in larger parts, since the uploader cannot learn the size of
the reader it is given. The docs gave the default as 5MB, which the
config file reads as 5,000,000 bytes, below the minimum; they now say
5MiB.

Judgement call: the 5GiB maximum is enforced along with the 5MiB
minimum the issue names.

Model: opus-5-5
2026-10-07 14:29:10 +02:00

484 lines
15 KiB
Go

// Package config loads, validates, and provides the vaultik YAML
// configuration, including snapshot definitions, encryption recipients,
// and storage settings.
package config
import (
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"filippo.io/age"
"git.eeqj.de/sneak/smartconfig"
"github.com/adrg/xdg"
"go.uber.org/fx"
"gopkg.in/yaml.v3"
"sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log"
)
const appName = "vaultik"
// secretKeyPrefix marks an age secret (private) key. It is compared
// case-insensitively so a recipient entry that is actually a private key is
// caught and never passed to age or echoed back.
//
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
const secretKeyPrefix = "AGE-SECRET-KEY-"
// Defaults and validation bounds for tunable settings.
const (
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
defaultCompressionLevel = 3
minChunkSize = 1024 * 1024 // 1MB
minCompressionLevel = 1
maxCompressionLevel = 19
// S3 accepts a multipart upload part from 5MiB to 5GiB.
minS3PartSize = 5 * 1024 * 1024
maxS3PartSize = 5 * 1024 * 1024 * 1024
)
// Sentinel validation errors.
var (
errNoConfigPath = errors.New("config path not provided")
errRecipientIsSecretKey = errors.New(
"an age secret key was given where a public key (age1...) belongs")
errRecipientNotX25519 = errors.New(
"not a valid recipient; only X25519 age1... public keys are supported")
errNoSnapshots = errors.New(
"at least one snapshot must be configured (see config.example.yml)")
errSnapshotNoPaths = errors.New("snapshot must have at least one path")
errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB")
errBlobSizeTooSmall = errors.New(
"blob_size_limit must be at least the largest chunk the chunker can " +
"emit (chunk_size times the FastCDC size spread)")
errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://")
errStorageNotConfigured = errors.New(
"storage not configured; set storage_url or provide s3.endpoint + " +
"s3.bucket + credentials")
errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)")
errS3KeyIDRequired = errors.New("s3.access_key_id is required")
errS3SecretRequired = errors.New("s3.secret_access_key is required")
)
// expandTilde expands ~ at the start of a path to the user's home directory.
func expandTilde(path string) string {
if path == "~" {
home, _ := os.UserHomeDir()
return home
}
if strings.HasPrefix(path, "~/") {
home, _ := os.UserHomeDir()
return filepath.Join(home, path[2:])
}
return path
}
// expandTildeInURL expands ~ in file:// URLs.
func expandTildeInURL(url string) string {
if strings.HasPrefix(url, "file://~/") {
home, _ := os.UserHomeDir()
return "file://" + filepath.Join(home, url[9:])
}
return url
}
// SnapshotConfig represents configuration for a named snapshot.
// Each snapshot backs up one or more paths and can have its own exclude patterns
// in addition to the global excludes.
type SnapshotConfig struct {
Paths []string `yaml:"paths"`
Exclude []string `yaml:"exclude"` // Additional excludes for this snapshot
}
// GetExcludes returns the combined exclude patterns for a named snapshot.
// It merges global excludes with the snapshot-specific excludes.
func (c *Config) GetExcludes(snapshotName string) []string {
snap, ok := c.Snapshots[snapshotName]
if !ok {
return c.Exclude
}
if len(snap.Exclude) == 0 {
return c.Exclude
}
// Combine global and snapshot-specific excludes
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
combined = append(combined, c.Exclude...)
combined = append(combined, snap.Exclude...)
return combined
}
// SnapshotNames returns the names of all configured snapshots in sorted order.
func (c *Config) SnapshotNames() []string {
names := make([]string, 0, len(c.Snapshots))
for name := range c.Snapshots {
names = append(names, name)
}
// Sort for deterministic order
sort.Strings(names)
return names
}
// Names of the two places the age secret key can be configured, used by
// AgeSecretKeySourceName for error messages that must not echo the value.
//
//nolint:gosec // G101: these are the names of the config sources, not a key
const (
ageSecretKeySourceEnv = "VAULTIK_AGE_SECRET_KEY"
ageSecretKeySourceConfig = "age_secret_key"
)
// AgeSecretKeySourceName returns the human name of where AgeSecretKey was
// configured. A Config built directly (as in tests) has no recorded
// source, so it reports the config-file field name.
func (c *Config) AgeSecretKeySourceName() string {
if c.AgeSecretKeySource != "" {
return c.AgeSecretKeySource
}
return ageSecretKeySourceConfig
}
// Config represents the application configuration for Vaultik.
// It defines all settings for backup operations, including source directories,
// encryption recipients, storage configuration, and performance tuning parameters.
// Configuration is typically loaded from a YAML file.
//
//nolint:tagliatelle // snake_case is the established config-file format
type Config struct {
AgeRecipients []string `yaml:"age_recipients"`
AgeSecretKey string `yaml:"age_secret_key"`
// AgeSecretKeySource names where AgeSecretKey was configured
// ("VAULTIK_AGE_SECRET_KEY" or "age_secret_key") so a later parse
// failure can name the source without echoing the secret value. It is
// set by Load and never read from or written to the config file.
AgeSecretKeySource string `yaml:"-"`
BlobSizeLimit Size `yaml:"blob_size_limit"`
ChunkSize Size `yaml:"chunk_size"`
// Exclude holds global excludes applied to all snapshots.
Exclude []string `yaml:"exclude"`
Hostname string `yaml:"hostname"`
IndexPath string `yaml:"index_path"`
S3 S3Config `yaml:"s3"`
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
CompressionLevel int `yaml:"compression_level"`
// StorageURL specifies the storage backend using a URL format.
// Takes precedence over S3Config if set.
// Supported formats:
// - s3://bucket/prefix?endpoint=host&region=us-east-1
// - file:///path/to/backup
// For S3 URLs, credentials are still read from s3.access_key_id
// and s3.secret_access_key.
StorageURL string `yaml:"storage_url"`
}
// S3Config represents S3 storage configuration for backup storage.
// It supports both AWS S3 and S3-compatible storage services.
// All fields except UseSSL and PartSize are required.
//
//nolint:tagliatelle // snake_case is the established config-file format
type S3Config struct {
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
AccessKeyID string `yaml:"access_key_id"`
SecretAccessKey string `yaml:"secret_access_key"`
Region string `yaml:"region"`
// UseSSL selects HTTPS for a scheme-less endpoint. Omitted (nil) means
// the default, TLS; set it to false only to force plain HTTP.
UseSSL *bool `yaml:"use_ssl"`
PartSize Size `yaml:"part_size"`
}
// Path wraps the config file path for fx dependency injection.
// This type allows the config file path to be injected as a distinct type
// rather than a plain string, avoiding conflicts with other string dependencies.
type Path string
// New creates a new Config instance by loading from the specified path.
// This function is used by the fx dependency injection framework.
// Returns an error if the path is empty or if loading fails.
func New(path Path) (*Config, error) {
if path == "" {
return nil, errNoConfigPath
}
cfg, err := Load(string(path))
if err != nil {
return nil, fmt.Errorf("failed to load config: %w", err)
}
return cfg, nil
}
// Load reads and parses the configuration file from the specified path.
// It applies default values for optional fields, performs environment variable
// substitution using smartconfig, and validates the configuration.
// The configuration file should be in YAML format. Returns an error if the file
// cannot be read, parsed, or if validation fails.
func Load(path string) (*Config, error) {
// Load config using smartconfig for interpolation
sc, err := smartconfig.NewFromConfigPath(path)
if err != nil {
return nil, fmt.Errorf("failed to load config file: %w", err)
}
cfg := &Config{
// Set defaults
BlobSizeLimit: defaultBlobSizeLimit,
ChunkSize: defaultChunkSize,
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
CompressionLevel: defaultCompressionLevel,
S3: S3Config{PartSize: defaultS3PartSize},
}
// Convert smartconfig data to YAML then unmarshal
configData := sc.Data()
yamlBytes, err := yaml.Marshal(configData)
if err != nil {
return nil, fmt.Errorf("failed to marshal config data: %w", err)
}
err = yaml.Unmarshal(yamlBytes, cfg)
if err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
// Expand tilde in all path fields
cfg.IndexPath = expandTilde(cfg.IndexPath)
cfg.StorageURL = expandTildeInURL(cfg.StorageURL)
// Expand tildes in snapshot paths
for name, snap := range cfg.Snapshots {
for i, path := range snap.Paths {
snap.Paths[i] = expandTilde(path)
}
cfg.Snapshots[name] = snap
}
// Check for environment variable override for IndexPath
if envIndexPath := os.Getenv("VAULTIK_INDEX_PATH"); envIndexPath != "" {
cfg.IndexPath = expandTilde(envIndexPath)
}
cfg.setAgeSecretKey()
// Get hostname if not set
if cfg.Hostname == "" {
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
cfg.Hostname = hostname
}
// Set default S3 settings
if cfg.S3.Region == "" {
cfg.S3.Region = "us-east-1"
}
// Check config file permissions (warn if world or group readable)
//nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path)
if statErr == nil {
mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable
log.Warn("Config file has insecure permissions (contains S3 credentials)",
"path", path,
"mode", fmt.Sprintf("%04o", mode),
"recommendation", "chmod 600 "+path)
}
}
err = cfg.Validate()
if err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
// Validate checks if the configuration is valid and complete.
// It ensures all required fields are present and have valid values:
// - Every age recipient must parse as an X25519 age1... public key (so a
// bad entry fails at load, not mid-backup); errors name the position,
// never the value. An empty list is accepted, because only snapshot
// create needs a recipient and it checks for one itself
// - At least one snapshot must be configured with at least one path
// - Storage must be configured (either storage_url or s3.* fields)
// - Chunk size must be at least 1MB
// - Blob size limit must be at least the largest chunk the chunker can emit
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
// exceeds the configured limit
// - Compression level must be between 1 and 19
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
//
// Returns an error describing the first validation failure encountered.
func (c *Config) Validate() error {
for i, recipient := range c.AgeRecipients {
err := validateAgeRecipient(recipient)
if err != nil {
return fmt.Errorf("age_recipients[%d]: %w", i, err)
}
}
if len(c.Snapshots) == 0 {
return errNoSnapshots
}
for name, snap := range c.Snapshots {
if len(snap.Paths) == 0 {
return fmt.Errorf("%w: %q", errSnapshotNoPaths, name)
}
}
// Validate storage configuration
err := c.validateStorage()
if err != nil {
return err
}
if c.ChunkSize.Int64() < minChunkSize {
return errChunkSizeTooSmall
}
// The chunker can emit chunks up to chunk_size * ChunkSizeSpread, and the
// packer places a single such chunk into an otherwise empty blob. A limit
// below that bound would let a blob exceed it, so reject it.
largestChunk := c.ChunkSize.Int64() * chunker.ChunkSizeSpread
if c.BlobSizeLimit.Int64() < largestChunk {
return fmt.Errorf("%w: need at least %d bytes",
errBlobSizeTooSmall, largestChunk)
}
if c.CompressionLevel < minCompressionLevel ||
c.CompressionLevel > maxCompressionLevel {
return errBadCompression
}
if c.S3.PartSize.Int64() < minS3PartSize ||
c.S3.PartSize.Int64() > maxS3PartSize {
return errBadS3PartSize
}
return nil
}
// validateAgeRecipient parses one age_recipients entry with the age library
// and returns a value-free error on failure. A recipient string can be
// sensitive (an operator may paste a secret key by mistake), so neither the
// entry nor age's own error (which quotes its input) is ever included.
func validateAgeRecipient(recipient string) error {
if strings.HasPrefix(strings.ToUpper(recipient), secretKeyPrefix) {
return errRecipientIsSecretKey
}
_, err := age.ParseX25519Recipient(recipient)
if err != nil {
return errRecipientNotX25519
}
return nil
}
// setAgeSecretKey records the age secret key and where it came from. The
// value is stored raw and parsed only where decryption happens
// (internal/vaultik), so backup, list and prune keep working whatever the
// field holds. The environment variable overrides the config-file field.
func (c *Config) setAgeSecretKey() {
if c.AgeSecretKey != "" {
c.AgeSecretKeySource = ageSecretKeySourceConfig
}
if env := os.Getenv("VAULTIK_AGE_SECRET_KEY"); env != "" {
c.AgeSecretKey = env
c.AgeSecretKeySource = ageSecretKeySourceEnv
}
}
// validateStorage validates storage configuration.
// If StorageURL is set, it takes precedence. S3 URLs require credentials.
// File URLs don't require any S3 configuration.
// If StorageURL is not set, legacy S3 configuration is required.
func (c *Config) validateStorage() error {
if c.StorageURL != "" {
return c.validateStorageURL()
}
// Legacy S3 configuration
if c.S3.Endpoint == "" {
return errStorageNotConfigured
}
if c.S3.Bucket == "" {
return errS3BucketRequired
}
if c.S3.AccessKeyID == "" {
return errS3KeyIDRequired
}
if c.S3.SecretAccessKey == "" {
return errS3SecretRequired
}
return nil
}
// validateStorageURL validates URL-based storage configuration. File and
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
// credential fields.
func (c *Config) validateStorageURL() error {
switch {
case strings.HasPrefix(c.StorageURL, "file://"):
// File storage doesn't need S3 credentials
return nil
case strings.HasPrefix(c.StorageURL, "rclone://"):
// Rclone storage uses rclone's own config
return nil
case strings.HasPrefix(c.StorageURL, "s3://"):
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired)
}
return nil
default:
return errBadStorageScheme
}
}
// Module exports the config module for fx dependency injection.
// It provides the Config type to other modules in the application.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("config",
fx.Provide(New),
)