smartconfig/smartconfig.go
sneak 6b8c16dd1d Implement proper YAML path navigation and complex type support
- YAML resolver now supports full path navigation (e.g., production.primary.host)
- Both JSON and YAML resolvers return YAML-formatted data for complex types
- This allows proper type preservation when loading objects/arrays from files
- Updated convertToType to parse YAML returned by resolvers
- Added comprehensive tests for YAML path navigation including arrays
- Fixed JSON resolver to support "." path for entire document
- All README examples now work correctly

The key insight was that resolvers should return YAML strings for complex
types, which can then be parsed and merged into the configuration structure,
preserving the original types (maps, arrays) instead of flattening to strings.
2025-07-21 18:57:13 +02:00

645 lines
17 KiB
Go

package smartconfig
import (
"fmt"
"io"
"os"
"regexp"
"strconv"
"strings"
"github.com/dustin/go-humanize"
"gopkg.in/yaml.v3"
)
const (
maxRecursionDepth = 3
interpolationPattern = `\$\{([^:]+?):(.*?)\}`
)
// Config represents the loaded configuration with support for interpolated values.
// It provides methods to load configuration from files or readers, access values,
// and register custom resolvers for extending interpolation capabilities.
type Config struct {
data map[string]interface{}
resolvers map[string]Resolver
}
// Resolver defines the interface for custom variable resolution.
// Implementations should resolve the given value and return the result.
// For example, an environment resolver would return the value of the
// environment variable specified in the value parameter.
type Resolver interface {
Resolve(value string) (string, error)
}
// NewFromAppName loads configuration from /etc/appname/config.yml.
// It creates a new Config instance, loads and interpolates the configuration file.
func NewFromAppName(appname string) (*Config, error) {
configPath := fmt.Sprintf("/etc/%s/config.yml", appname)
return NewFromConfigPath(configPath)
}
// NewFromConfigPath loads configuration from the specified file path.
// It creates a new Config instance, loads and interpolates the configuration file.
func NewFromConfigPath(path string) (*Config, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open config file: %w", err)
}
defer func() {
_ = file.Close()
}()
return NewFromReader(file)
}
// NewFromReader loads configuration from an io.Reader.
// It creates a new Config instance, loads and interpolates the configuration.
func NewFromReader(reader io.Reader) (*Config, error) {
c := newWithDefaults()
data, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to read config: %w", err)
}
// Parse YAML first
if err := yaml.Unmarshal(data, &c.data); err != nil {
return nil, fmt.Errorf("failed to parse YAML: %w", err)
}
// Walk and interpolate the parsed structure
interpolated, err := c.walkAndInterpolate(c.data)
if err != nil {
return nil, fmt.Errorf("failed to interpolate config: %w", err)
}
c.data = interpolated.(map[string]interface{})
// Handle environment variable injection
if err := c.injectEnvironment(); err != nil {
return nil, fmt.Errorf("failed to inject environment variables: %w", err)
}
return c, nil
}
// New creates an empty Config instance with default resolvers registered.
// For loading configuration, prefer using NewFromAppName, NewFromConfigPath, or NewFromReader.
func New() *Config {
return newWithDefaults()
}
func newWithDefaults() *Config {
c := &Config{
resolvers: make(map[string]Resolver),
data: make(map[string]interface{}),
}
// Register default resolvers
c.RegisterResolver("ENV", &EnvResolver{})
c.RegisterResolver("EXEC", &ExecResolver{})
c.RegisterResolver("FILE", &FileResolver{})
c.RegisterResolver("JSON", &JSONResolver{})
c.RegisterResolver("YAML", &YAMLResolver{})
c.RegisterResolver("AWSSM", &AWSSecretManagerResolver{})
c.RegisterResolver("GCPSM", &GCPSecretManagerResolver{})
c.RegisterResolver("VAULT", &VaultResolver{})
c.RegisterResolver("CONSUL", &ConsulResolver{})
c.RegisterResolver("AZURESM", &AzureKeyVaultResolver{})
c.RegisterResolver("K8SS", &K8SSecretResolver{})
c.RegisterResolver("ETCD", &EtcdResolver{})
return c
}
// RegisterResolver registers a custom resolver with the given name.
// The resolver will be available for use in interpolations with the syntax ${name:value}.
func (c *Config) RegisterResolver(name string, resolver Resolver) {
c.resolvers[name] = resolver
}
// LoadFromFile loads configuration from a YAML file at the specified path.
// DEPRECATED: Use NewFromConfigPath instead for cleaner API.
func (c *Config) LoadFromFile(path string) error {
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("failed to open config file: %w", err)
}
defer func() {
_ = file.Close()
}()
return c.LoadFromReader(file)
}
// LoadFromReader loads configuration from an io.Reader containing YAML data.
// DEPRECATED: Use NewFromReader instead for cleaner API.
func (c *Config) LoadFromReader(reader io.Reader) error {
data, err := io.ReadAll(reader)
if err != nil {
return fmt.Errorf("failed to read config: %w", err)
}
// Parse YAML first
if err := yaml.Unmarshal(data, &c.data); err != nil {
return fmt.Errorf("failed to parse YAML: %w", err)
}
// Walk and interpolate the parsed structure
interpolated, err := c.walkAndInterpolate(c.data)
if err != nil {
return fmt.Errorf("failed to interpolate config: %w", err)
}
c.data = interpolated.(map[string]interface{})
// Handle environment variable injection
if err := c.injectEnvironment(); err != nil {
return fmt.Errorf("failed to inject environment variables: %w", err)
}
return nil
}
func (c *Config) injectEnvironment() error {
envData, ok := c.data["env"]
if !ok {
return nil
}
envMap, ok := envData.(map[string]interface{})
if !ok {
return fmt.Errorf("env section must be a map")
}
for key, value := range envMap {
strValue, ok := value.(string)
if !ok {
strValue = fmt.Sprintf("%v", value)
}
if err := os.Setenv(key, strValue); err != nil {
return fmt.Errorf("failed to set environment variable %s: %w", key, err)
}
}
return nil
}
// Get retrieves a value from the configuration using dot notation.
// For example:
// - "database.host" retrieves config.database.host
// - "servers.0.name" retrieves the name of the first server in a list
//
// Returns the value and true if found, nil and false if not found.
func (c *Config) Get(key string) (interface{}, bool) {
return c.data[key], true
}
// GetString retrieves a string value from the configuration.
// Returns an error if the key doesn't exist. Numeric values are converted to strings.
func (c *Config) GetString(key string) (string, error) {
value, ok := c.data[key]
if !ok {
return "", fmt.Errorf("key %s not found", key)
}
// Try direct string conversion first
if strValue, ok := value.(string); ok {
return strValue, nil
}
// Convert other types to string
return fmt.Sprintf("%v", value), nil
}
// GetInt retrieves an integer value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to an integer.
func (c *Config) GetInt(key string) (int, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct int conversion first
if intValue, ok := value.(int); ok {
return intValue, nil
}
// Try float64 (common in JSON/YAML parsing)
if floatValue, ok := value.(float64); ok {
return int(floatValue), nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
intValue, err := strconv.Atoi(strValue)
if err != nil {
return 0, fmt.Errorf("cannot convert value %q to integer: %w", strValue, err)
}
return intValue, nil
}
return 0, fmt.Errorf("cannot convert value of type %T to integer", value)
}
// GetUint retrieves an unsigned integer value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to a uint.
func (c *Config) GetUint(key string) (uint, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct uint conversion first
if uintValue, ok := value.(uint); ok {
return uintValue, nil
}
// Try int conversion
if intValue, ok := value.(int); ok {
if intValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %d to uint", intValue)
}
return uint(intValue), nil
}
// Try float64 (common in JSON/YAML parsing)
if floatValue, ok := value.(float64); ok {
if floatValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %f to uint", floatValue)
}
return uint(floatValue), nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
uintValue, err := strconv.ParseUint(strValue, 10, 0)
if err != nil {
return 0, fmt.Errorf("cannot convert value %q to uint: %w", strValue, err)
}
return uint(uintValue), nil
}
return 0, fmt.Errorf("cannot convert value of type %T to uint", value)
}
// GetFloat retrieves a float64 value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to a float64.
func (c *Config) GetFloat(key string) (float64, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct float64 conversion first
if floatValue, ok := value.(float64); ok {
return floatValue, nil
}
// Try float32 conversion
if floatValue, ok := value.(float32); ok {
return float64(floatValue), nil
}
// Try int conversion
if intValue, ok := value.(int); ok {
return float64(intValue), nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
floatValue, err := strconv.ParseFloat(strValue, 64)
if err != nil {
return 0, fmt.Errorf("cannot convert value %q to float: %w", strValue, err)
}
return floatValue, nil
}
return 0, fmt.Errorf("cannot convert value of type %T to float", value)
}
// GetBool retrieves a boolean value from the configuration.
// Returns an error if the key doesn't exist or if the value cannot be converted to a boolean.
func (c *Config) GetBool(key string) (bool, error) {
value, ok := c.data[key]
if !ok {
return false, fmt.Errorf("key %s not found", key)
}
// Try direct bool conversion first
if boolValue, ok := value.(bool); ok {
return boolValue, nil
}
// Try string conversion
if strValue, ok := value.(string); ok {
boolValue, err := strconv.ParseBool(strValue)
if err != nil {
return false, fmt.Errorf("cannot convert value %q to boolean: %w", strValue, err)
}
return boolValue, nil
}
// Try numeric conversions (0 = false, non-zero = true)
if intValue, ok := value.(int); ok {
return intValue != 0, nil
}
if floatValue, ok := value.(float64); ok {
return floatValue != 0, nil
}
return false, fmt.Errorf("cannot convert value of type %T to boolean", value)
}
// GetBytes retrieves a byte size value from the configuration.
// Supports human-readable formats like "10G", "20KiB", "25TB", etc.
// Returns the size in bytes as uint64.
func (c *Config) GetBytes(key string) (uint64, error) {
value, ok := c.data[key]
if !ok {
return 0, fmt.Errorf("key %s not found", key)
}
// Try direct numeric conversions first
if uintValue, ok := value.(uint64); ok {
return uintValue, nil
}
if intValue, ok := value.(int); ok {
if intValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %d to bytes", intValue)
}
return uint64(intValue), nil
}
if floatValue, ok := value.(float64); ok {
if floatValue < 0 {
return 0, fmt.Errorf("cannot convert negative value %f to bytes", floatValue)
}
return uint64(floatValue), nil
}
// Try string conversion with humanize parsing
if strValue, ok := value.(string); ok {
// Try parsing as a plain number first
if bytesValue, err := strconv.ParseUint(strValue, 10, 64); err == nil {
return bytesValue, nil
}
// Try parsing with humanize
bytesValue, err := humanize.ParseBytes(strValue)
if err != nil {
return 0, fmt.Errorf("cannot parse value %q as bytes: %w", strValue, err)
}
return bytesValue, nil
}
return 0, fmt.Errorf("cannot convert value of type %T to bytes", value)
}
// Data returns the entire configuration as a map.
// This is useful for unmarshaling into custom structures.
func (c *Config) Data() map[string]interface{} {
return c.data
}
// walkAndInterpolate recursively walks through the parsed YAML structure
// and interpolates any string values containing ${...} patterns.
func (c *Config) walkAndInterpolate(data interface{}) (interface{}, error) {
switch v := data.(type) {
case string:
// Check if this string contains interpolation patterns
if strings.Contains(v, "${") && strings.Contains(v, "}") {
return c.interpolateString(v)
}
return v, nil
case map[string]interface{}:
// Recursively process map values
result := make(map[string]interface{})
for key, value := range v {
interpolated, err := c.walkAndInterpolate(value)
if err != nil {
return nil, err
}
result[key] = interpolated
}
return result, nil
case []interface{}:
// Recursively process array elements
result := make([]interface{}, len(v))
for i, value := range v {
interpolated, err := c.walkAndInterpolate(value)
if err != nil {
return nil, err
}
result[i] = interpolated
}
return result, nil
default:
// Return other types as-is (numbers, booleans, etc.)
return v, nil
}
}
// interpolateString handles interpolation of a single string value.
// It returns the appropriate type based on the resolved value.
func (c *Config) interpolateString(s string) (interface{}, error) {
// If the entire string is a single interpolation, we can return typed values
if match := regexp.MustCompile(`^\$\{([^:]+):(.+)\}$`).FindStringSubmatch(s); match != nil {
resolverName := match[1]
value := match[2]
// Handle nested interpolations in the value
if strings.Contains(value, "${") {
processedValue, err := c.interpolateValue(value, 0)
if err != nil {
return nil, err
}
value = processedValue
}
// Resolve the value
resolver, ok := c.resolvers[resolverName]
if !ok {
return nil, fmt.Errorf("unknown resolver: %s", resolverName)
}
resolved, err := resolver.Resolve(value)
if err != nil {
return nil, fmt.Errorf("failed to resolve %s:%s: %w", resolverName, value, err)
}
// Try to convert to appropriate type
return c.convertToType(resolved), nil
}
// Otherwise, it's a string with embedded interpolations
// We need to interpolate these as strings
return c.interpolateMixedContent(s, 0)
}
// interpolateValue handles nested interpolations for resolver values
func (c *Config) interpolateValue(s string, depth int) (string, error) {
if depth >= maxRecursionDepth {
return s, nil
}
result := s
changed := true
for changed && depth < maxRecursionDepth {
changed = false
positions := findInterpolations(result)
// Process from end to beginning
for i := len(positions) - 1; i >= 0; i-- {
pos := positions[i]
fullMatch := result[pos.start:pos.end]
inner := fullMatch[2 : len(fullMatch)-1]
colonIdx := strings.Index(inner, ":")
if colonIdx == -1 {
continue
}
resolverName := inner[:colonIdx]
value := inner[colonIdx+1:]
// Recursively handle nested interpolations
if strings.Contains(value, "${") {
interpolatedValue, err := c.interpolateValue(value, depth+1)
if err != nil {
return "", err
}
value = interpolatedValue
}
// Resolve the value
resolver, ok := c.resolvers[resolverName]
if !ok {
return "", fmt.Errorf("unknown resolver: %s", resolverName)
}
resolved, err := resolver.Resolve(value)
if err != nil {
return "", fmt.Errorf("failed to resolve %s:%s: %w", resolverName, value, err)
}
// Replace without quotes
result = result[:pos.start] + resolved + result[pos.end:]
changed = true
}
if changed {
depth++
}
}
return result, nil
}
// interpolateMixedContent handles strings with embedded interpolations
func (c *Config) interpolateMixedContent(s string, depth int) (string, error) {
if depth >= maxRecursionDepth {
return s, nil
}
result := s
positions := findInterpolations(result)
// Process from end to beginning
for i := len(positions) - 1; i >= 0; i-- {
pos := positions[i]
fullMatch := result[pos.start:pos.end]
inner := fullMatch[2 : len(fullMatch)-1]
colonIdx := strings.Index(inner, ":")
if colonIdx == -1 {
continue
}
resolverName := inner[:colonIdx]
value := inner[colonIdx+1:]
// Handle nested interpolations
if strings.Contains(value, "${") {
interpolatedValue, err := c.interpolateValue(value, depth+1)
if err != nil {
return "", err
}
value = interpolatedValue
}
// Resolve the value
resolver, ok := c.resolvers[resolverName]
if !ok {
return "", fmt.Errorf("unknown resolver: %s", resolverName)
}
resolved, err := resolver.Resolve(value)
if err != nil {
return "", fmt.Errorf("failed to resolve %s:%s: %w", resolverName, value, err)
}
// Replace without quotes for mixed content
result = result[:pos.start] + resolved + result[pos.end:]
}
return result, nil
}
// convertToType attempts to convert a string to its appropriate type
func (c *Config) convertToType(s string) interface{} {
// First try to parse as YAML - this handles complex structures
// returned by JSON/YAML resolvers
var yamlData interface{}
if err := yaml.Unmarshal([]byte(s), &yamlData); err == nil {
// If it's a complex type (map or slice), return it directly
switch yamlData.(type) {
case map[string]interface{}, map[interface{}]interface{}, []interface{}:
return yamlData
}
// For simple types parsed from YAML, continue with normal conversion
}
// Try boolean
if s == "true" {
return true
}
if s == "false" {
return false
}
// Try integer
if i, err := strconv.Atoi(s); err == nil {
// Verify the conversion is lossless
if strconv.Itoa(i) == s {
return i
}
}
// Try float
if f, err := strconv.ParseFloat(s, 64); err == nil {
// Check if it's actually an integer
if float64(int(f)) == f {
// Verify integer conversion is lossless
if strconv.Itoa(int(f)) == s {
return int(f)
}
}
// For floats, use FormatFloat to check if conversion is lossless
// Using -1 precision means use the smallest number of digits necessary
if strconv.FormatFloat(f, 'f', -1, 64) == s {
return f
}
}
// Default to string
return s
}