smartconfig/smartconfig.go
sneak cf13b275b7 Squashed commit of the following:
commit e15edaedd786254cf22c291a63a02f7cff33b119
Author: sneak <sneak@sneak.berlin>
Date:   Mon Jul 21 15:17:12 2025 +0200

    Implement YAML-first interpolation with type preservation

    Previously, interpolations were performed during string manipulation before
    YAML parsing, which caused issues with quoting and escaping. This commit
    fundamentally changes the approach:

    - Parse YAML first, then walk the structure to interpolate values
    - Preserve types: standalone interpolations can return numbers/booleans
    - Mixed content (text with embedded interpolations) always returns strings
    - Users control types through YAML syntax, not our quoting logic
    - Properly handle nested interpolations without quote accumulation

    This gives users explicit control over output types while eliminating
    the complex and error-prone manual quoting logic.
2025-07-21 15:19:28 +02:00

623 lines
17 KiB
Go

package smartconfig
import (
"fmt"
"io"
"os"
"regexp"
"strconv"
"strings"
"github.com/dustin/go-humanize"
"gopkg.in/yaml.v3"
)
const (
maxRecursionDepth = 3
interpolationPattern = `\$\{([^:]+?):(.*?)\}`
)
// Config represents the loaded configuration with support for interpolated values.
// It provides methods to load configuration from files or readers, access values,
// and register custom resolvers for extending interpolation capabilities.
type Config struct {
data map[string]interface{}
resolvers map[string]Resolver
}
// Resolver defines the interface for custom variable resolution.
// Implementations should resolve the given value and return the result.
// For example, an environment resolver would return the value of the
// environment variable specified in the value parameter.
type Resolver interface {
Resolve(value string) (string, error)
}
// NewFromAppName loads configuration from /etc/appname/config.yml.
// It creates a new Config instance, loads and interpolates the configuration file.
func NewFromAppName(appname string) (*Config, error) {
configPath := fmt.Sprintf("/etc/%s/config.yml", appname)
return NewFromConfigPath(configPath)
}
// NewFromConfigPath loads configuration from the specified file path.
// It creates a new Config instance, loads and interpolates the configuration file.
func NewFromConfigPath(path string) (*Config, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open config file: %w", err)
}
defer func() {
_ = file.Close()
}()
return NewFromReader(file)
}
// NewFromReader loads configuration from an io.Reader.
// It creates a new Config instance, loads and interpolates the configuration.
func NewFromReader(reader io.Reader) (*Config, error) {
c := newWithDefaults()
data, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to read config: %w", err)
}
// Parse YAML first
if err := yaml.Unmarshal(data, &c.data); err != nil {
return nil, fmt.Errorf("failed to parse YAML: %w", err)
}
// Walk and interpolate the parsed structure
interpolated, err := c.walkAndInterpolate(c.data)
if err != nil {
return nil, fmt.Errorf("failed to interpolate config: %w", err)
}
c.data = interpolated.(map[string]interface{})
// Handle environment variable injection
if err := c.injectEnvironment(); err != nil {
return nil, fmt.Errorf("failed to inject environment variables: %w", err)
}
return c, nil
}
// New creates an empty Config instance with default resolvers registered.
// For loading configuration, prefer using NewFromAppName, NewFromConfigPath, or NewFromReader.
func New() *Config {
return newWithDefaults()
}
func newWithDefaults() *Config {
c := &Config{
resolvers: make(map[string]Resolver),
data: make(map[string]interface{}),
}
// Register default resolvers
c.RegisterResolver("ENV", &EnvResolver{})
c.RegisterResolver("EXEC", &ExecResolver{})
c.RegisterResolver("FILE", &FileResolver{})
c.RegisterResolver("JSON", &JSONResolver{})
c.RegisterResolver("YAML", &YAMLResolver{})
c.RegisterResolver("AWSSM", &AWSSecretManagerResolver{})
c.RegisterResolver("GCPSM", &GCPSecretManagerResolver{})
c.RegisterResolver("VAULT", &VaultResolver{})
c.RegisterResolver("CONSUL", &ConsulResolver{})
c.RegisterResolver("AZURESM", &AzureKeyVaultResolver{})
c.RegisterResolver("K8SS", &K8SSecretResolver{})
c.RegisterResolver("ETCD", &EtcdResolver{})
return c
}
// RegisterResolver registers a custom resolver with the given name.
// The resolver will be available for use in interpolations with the syntax ${name:value}.
func (c *Config) RegisterResolver(name string, resolver Resolver) {
c.resolvers[name] = resolver
}
// LoadFromFile loads configuration from a YAML file at the specified path.
// DEPRECATED: Use NewFromConfigPath instead for cleaner API.
func (c *Config) LoadFromFile(path string) error {
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("failed to open config file: %w", err)
}
defer func() {
_ = file.Close()
}()
return c.LoadFromReader(file)
}
// LoadFromReader loads configuration from an io.Reader containing YAML data.
// DEPRECATED: Use NewFromReader instead for cleaner API.
func (c *Config) LoadFromReader(reader io.Reader) error {
data, err := io.ReadAll(reader)
if err != nil {
return fmt.Errorf("failed to read config: %w", err)
}
// Parse YAML first
if err := yaml.Unmarshal(data, &c.data); err != nil {
return fmt.Errorf("failed to parse YAML: %w", err)
}
// Walk and interpolate the parsed structure
interpolated, err := c.walkAndInterpolate(c.data)
if err != nil {
return fmt.Errorf("failed to interpolate config: %w", err)
}
c.data = interpolated.(map[string]interface{})
// Handle environment variable injection
if err := c.injectEnvironment(); err != nil {
return fmt.Errorf("failed to inject environment variables: %w", err)
}
return nil
}
func (c *Config) injectEnvironment() error {
envData, ok := c.data["env"]
if !ok {
return nil
}
envMap, ok := envData.(map[string]interface{})
if !ok {
return fmt.Errorf("env section must be a map")
}
for key, value := range envMap {
strValue, ok := value.(string)
if !ok {
strValue = fmt.Sprintf("%v", value)
}
if err := os.Setenv(key, strValue); err != nil {
return fmt.Errorf("failed to set environment variable %s: %w", key, err)
}
}
return nil
}
// Get retrieves a value from the configuration using dot notation.
// For example:
// - "database.host" retrieves config.database.host
// - "servers.0.name" retrieves the name of the first server in a list
//
// Returns the value and true if found, nil and false if not found.
func (c *Config) Get(key string) (interface{}, bool) {
return c.data[key], true
}
// GetString retrieves a string value from the configuration.
// Returns an error if the key doesn't exist. Numeric values are converted to strings.
func (c *Config) GetString(key string) (string, error) {
value, ok := c.data[key]
if !ok {
return "", fmt.Errorf("key %s not found", key)
}
// Try direct string conversion first
if strValue, ok := value.(string); ok {
return strValue, nil
}
// Convert other types to string
return fmt.Sprintf("%v", value), nil
}
// GetInt retrieves an integer value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to an integer.
func (c *Config) GetInt(key string) (int, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct int conversion first
if intValue, ok := value.(int); ok {
return intValue, nil
}
// Try float64 (common in JSON/YAML parsing)
if floatValue, ok := value.(float64); ok {
return int(floatValue), nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
intValue, err := strconv.Atoi(strValue)
if err != nil {
return 0, fmt.Errorf("cannot convert value %q to integer: %w", strValue, err)
}
return intValue, nil
}
return 0, fmt.Errorf("cannot convert value of type %T to integer", value)
}
// GetUint retrieves an unsigned integer value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to a uint.
func (c *Config) GetUint(key string) (uint, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct uint conversion first
if uintValue, ok := value.(uint); ok {
return uintValue, nil
}
// Try int conversion
if intValue, ok := value.(int); ok {
if intValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %d to uint", intValue)
}
return uint(intValue), nil
}
// Try float64 (common in JSON/YAML parsing)
if floatValue, ok := value.(float64); ok {
if floatValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %f to uint", floatValue)
}
return uint(floatValue), nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
uintValue, err := strconv.ParseUint(strValue, 10, 0)
if err != nil {
return 0, fmt.Errorf("cannot convert value %q to uint: %w", strValue, err)
}
return uint(uintValue), nil
}
return 0, fmt.Errorf("cannot convert value of type %T to uint", value)
}
// GetFloat retrieves a float64 value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to a float64.
func (c *Config) GetFloat(key string) (float64, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct float64 conversion first
if floatValue, ok := value.(float64); ok {
return floatValue, nil
}
// Try float32 conversion
if floatValue, ok := value.(float32); ok {
return float64(floatValue), nil
}
// Try int conversion
if intValue, ok := value.(int); ok {
return float64(intValue), nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
floatValue, err := strconv.ParseFloat(strValue, 64)
if err != nil {
return 0, fmt.Errorf("cannot convert value %q to float: %w", strValue, err)
}
return floatValue, nil
}
return 0, fmt.Errorf("cannot convert value of type %T to float", value)
}
// GetBool retrieves a boolean value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to a boolean.
func (c *Config) GetBool(key string) (bool, error) {
value, ok := c.data[key]
if !ok {
return false, fmt.Errorf("key %s not found", key)
}
// Try direct bool conversion first
if boolValue, ok := value.(bool); ok {
return boolValue, nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
boolValue, err := strconv.ParseBool(strValue)
if err != nil {
return false, fmt.Errorf("cannot convert value %q to boolean: %w", strValue, err)
}
return boolValue, nil
}
// Try numeric conversions (0 = false, non-zero = true)
if intValue, ok := value.(int); ok {
return intValue != 0, nil
}
if floatValue, ok := value.(float64); ok {
return floatValue != 0, nil
}
return false, fmt.Errorf("cannot convert value of type %T to boolean", value)
}
// GetBytes retrieves a byte size value from the configuration.
// Supports human-readable formats like "10G", "20KiB", "25TB", etc.
// Returns the size in bytes as uint64.
func (c *Config) GetBytes(key string) (uint64, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct numeric conversions first
if uintValue, ok := value.(uint64); ok {
return uintValue, nil
}
if intValue, ok := value.(int); ok {
if intValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %d to bytes", intValue)
}
return uint64(intValue), nil
}
if floatValue, ok := value.(float64); ok {
if floatValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %f to bytes", floatValue)
}
return uint64(floatValue), nil
}
// Try string conversion with humanize parsing
if strValue, ok := value.(string); ok {
// Try parsing as a plain number first
if bytesValue, err := strconv.ParseUint(strValue, 10, 64); err == nil {
return bytesValue, nil
}
// Try parsing with humanize
bytesValue, err := humanize.ParseBytes(strValue)
if err != nil {
return 0, fmt.Errorf("cannot parse value %q as bytes: %w", strValue, err)
}
return bytesValue, nil
}
return 0, fmt.Errorf("cannot convert value of type %T to bytes", value)
}
// Data returns the entire configuration as a map.
// This is useful for unmarshaling into custom structures.
func (c *Config) Data() map[string]interface{} {
return c.data
}
// walkAndInterpolate recursively walks through the parsed YAML structure
// and interpolates any string values containing ${...} patterns.
func (c *Config) walkAndInterpolate(data interface{}) (interface{}, error) {
switch v := data.(type) {
case string:
// Check if this string contains interpolation patterns
if strings.Contains(v, "${") && strings.Contains(v, "}") {
return c.interpolateString(v)
}
return v, nil
case map[string]interface{}:
// Recursively process map values
result := make(map[string]interface{})
for key, value := range v {
interpolated, err := c.walkAndInterpolate(value)
if err != nil {
return nil, err
}
result[key] = interpolated
}
return result, nil
case []interface{}:
// Recursively process array elements
result := make([]interface{}, len(v))
for i, value := range v {
interpolated, err := c.walkAndInterpolate(value)
if err != nil {
return nil, err
}
result[i] = interpolated
}
return result, nil
default:
// Return other types as-is (numbers, booleans, etc.)
return v, nil
}
}
// interpolateString handles interpolation of a single string value.
// It returns the appropriate type based on the resolved value.
func (c *Config) interpolateString(s string) (interface{}, error) {
// If the entire string is a single interpolation, we can return typed values
if match := regexp.MustCompile(`^\$\{([^:]+):(.+)\}$`).FindStringSubmatch(s); match != nil {
resolverName := match[1]
value := match[2]
// Handle nested interpolations in the value
if strings.Contains(value, "${") {
processedValue, err := c.interpolateValue(value, 0)
if err != nil {
return nil, err
}
value = processedValue
}
// Resolve the value
resolver, ok := c.resolvers[resolverName]
if !ok {
return nil, fmt.Errorf("unknown resolver: %s", resolverName)
}
resolved, err := resolver.Resolve(value)
if err != nil {
return nil, fmt.Errorf("failed to resolve %s:%s: %w", resolverName, value, err)
}
// Try to convert to appropriate type
return c.convertToType(resolved), nil
}
// Otherwise, it's a string with embedded interpolations
// We need to interpolate these as strings
return c.interpolateMixedContent(s, 0)
}
// interpolateValue handles nested interpolations for resolver values
func (c *Config) interpolateValue(s string, depth int) (string, error) {
if depth >= maxRecursionDepth {
return s, nil
}
result := s
changed := true
for changed && depth < maxRecursionDepth {
changed = false
positions := findInterpolations(result)
// Process from end to beginning
for i := len(positions) - 1; i >= 0; i-- {
pos := positions[i]
fullMatch := result[pos.start:pos.end]
inner := fullMatch[2 : len(fullMatch)-1]
colonIdx := strings.Index(inner, ":")
if colonIdx == -1 {
continue
}
resolverName := inner[:colonIdx]
value := inner[colonIdx+1:]
// Recursively handle nested interpolations
if strings.Contains(value, "${") {
interpolatedValue, err := c.interpolateValue(value, depth+1)
if err != nil {
return "", err
}
value = interpolatedValue
}
// Resolve the value
resolver, ok := c.resolvers[resolverName]
if !ok {
return "", fmt.Errorf("unknown resolver: %s", resolverName)
}
resolved, err := resolver.Resolve(value)
if err != nil {
return "", fmt.Errorf("failed to resolve %s:%s: %w", resolverName, value, err)
}
// Replace without quotes
result = result[:pos.start] + resolved + result[pos.end:]
changed = true
}
if changed {
depth++
}
}
return result, nil
}
// interpolateMixedContent handles strings with embedded interpolations
func (c *Config) interpolateMixedContent(s string, depth int) (string, error) {
if depth >= maxRecursionDepth {
return s, nil
}
result := s
positions := findInterpolations(result)
// Process from end to beginning
for i := len(positions) - 1; i >= 0; i-- {
pos := positions[i]
fullMatch := result[pos.start:pos.end]
inner := fullMatch[2 : len(fullMatch)-1]
colonIdx := strings.Index(inner, ":")
if colonIdx == -1 {
continue
}
resolverName := inner[:colonIdx]
value := inner[colonIdx+1:]
// Handle nested interpolations
if strings.Contains(value, "${") {
interpolatedValue, err := c.interpolateValue(value, depth+1)
if err != nil {
return "", err
}
value = interpolatedValue
}
// Resolve the value
resolver, ok := c.resolvers[resolverName]
if !ok {
return "", fmt.Errorf("unknown resolver: %s", resolverName)
}
resolved, err := resolver.Resolve(value)
if err != nil {
return "", fmt.Errorf("failed to resolve %s:%s: %w", resolverName, value, err)
}
// Replace without quotes for mixed content
result = result[:pos.start] + resolved + result[pos.end:]
}
return result, nil
}
// convertToType attempts to convert a string to its appropriate type
func (c *Config) convertToType(s string) interface{} {
// Try boolean
if s == "true" {
return true
}
if s == "false" {
return false
}
// Try integer
if i, err := strconv.Atoi(s); err == nil {
return i
}
// Try float
if f, err := strconv.ParseFloat(s, 64); err == nil {
// Check if it's actually an integer
if float64(int(f)) == f {
return int(f)
}
return f
}
// Default to string
return s
}