Implement proper YAML path navigation and complex type support

- YAML resolver now supports full path navigation (e.g., production.primary.host)
- Both JSON and YAML resolvers return YAML-formatted data for complex types
- This allows proper type preservation when loading objects/arrays from files
- Updated convertToType to parse YAML returned by resolvers
- Added comprehensive tests for YAML path navigation including arrays
- Fixed JSON resolver to support "." path for entire document
- All README examples now work correctly

The key insight was that resolvers should return YAML strings for complex
types, which can then be parsed and merged into the configuration structure,
preserving the original types (maps, arrays) instead of flattening to strings.
This commit is contained in:
Jeffrey Paul 2025-07-21 18:57:13 +02:00
parent e7fa389634
commit 6b8c16dd1d
8 changed files with 575 additions and 11 deletions

175
readme_examples_test.go Normal file
View File

@ -0,0 +1,175 @@
package smartconfig
import (
"fmt"
"os"
"strings"
"testing"
)
func TestREADMEExamples(t *testing.T) {
// Create test YAML files that match README examples
// Database config file
databaseYAML := `production:
primary:
host: prod-primary.db.example.com
port: 5432
replica:
host: prod-replica.db.example.com
port: 5432
staging:
primary:
host: staging.db.example.com
port: 5432
`
err := os.WriteFile("./test/readme_database.yml", []byte(databaseYAML), 0644)
if err != nil {
t.Fatalf("Failed to create database test file: %v", err)
}
defer func() { _ = os.Remove("./test/readme_database.yml") }()
// Features config file
featuresYAML := `features:
analytics:
enabled: true
provider: google
rate_limiting:
enabled: false
`
err = os.WriteFile("./test/readme_features.yaml", []byte(featuresYAML), 0644)
if err != nil {
t.Fatalf("Failed to create features test file: %v", err)
}
defer func() { _ = os.Remove("./test/readme_features.yaml") }()
// Test the exact examples from README
tests := []struct {
name string
yaml string
expected map[string]interface{}
}{
{
name: "YAML resolver examples from README",
yaml: `
db_config: ${YAML:./test/readme_database.yml:production.primary}
replica_host: ${YAML:./test/readme_database.yml:production.replica.host}
analytics: ${YAML:./test/readme_features.yaml:features.analytics.enabled}
`,
expected: map[string]interface{}{
"db_config": "map[host:prod-primary.db.example.com port:5432]",
"replica_host": "prod-replica.db.example.com",
"analytics": "true",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := NewFromReader(strings.NewReader(tt.yaml))
if err != nil {
t.Fatalf("Failed to load config: %v", err)
}
for key, expectedValue := range tt.expected {
val, exists := config.Get(key)
if !exists {
t.Errorf("Key %s not found in config", key)
continue
}
// Convert to string for comparison
strVal := ""
switch v := val.(type) {
case string:
strVal = v
case bool:
strVal = fmt.Sprintf("%t", v)
default:
strVal = fmt.Sprintf("%v", v)
}
if strVal != expectedValue {
t.Errorf("Key %s: expected %q, got %q", key, expectedValue, strVal)
}
}
})
}
}
func TestCompleteExampleFromREADME(t *testing.T) {
// This tests the complete example from the README
err := os.WriteFile("./test/services.json", []byte(`{
"production": {
"api": {
"endpoint": "https://api.example.com"
}
}
}`), 0644)
if err != nil {
t.Fatalf("Failed to create services.json: %v", err)
}
defer func() { _ = os.Remove("./test/services.json") }()
err = os.WriteFile("./test/features.json", []byte(`{
"production": {
"new_ui": false,
"rate_limiting": true
},
"staging": {
"new_ui": true
}
}`), 0644)
if err != nil {
t.Fatalf("Failed to create features.json: %v", err)
}
defer func() { _ = os.Remove("./test/features.json") }()
// Set environment variables for the test
_ = os.Setenv("ENVIRONMENT", "production")
defer func() { _ = os.Unsetenv("ENVIRONMENT") }()
configYAML := `
# External service configuration from JSON config file
services: ${JSON:./test/services.json:production}
# Feature flags from JSON file
features: ${JSON:./test/features.json:${ENV:ENVIRONMENT}}
`
config, err := NewFromReader(strings.NewReader(configYAML))
if err != nil {
t.Fatalf("Failed to load config: %v", err)
}
// Check services were loaded
services, exists := config.Get("services")
if !exists {
t.Fatal("services key not found")
}
// Check it's a map
if _, ok := services.(map[string]interface{}); !ok {
t.Errorf("Expected services to be a map, got %T", services)
}
// Check features were loaded with correct environment
features, exists := config.Get("features")
if !exists {
t.Fatal("features key not found")
}
// Check features content
if featuresMap, ok := features.(map[string]interface{}); ok {
if newUI, exists := featuresMap["new_ui"]; exists {
if newUI != false {
t.Errorf("Expected new_ui to be false, got %v", newUI)
}
} else {
t.Error("new_ui not found in features")
}
} else {
t.Errorf("Expected features to be a map, got %T", features)
}
}

View File

@ -1,11 +1,13 @@
package smartconfig
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/tidwall/gjson"
"gopkg.in/yaml.v3"
)
// JSONResolver reads values from JSON files.
@ -13,6 +15,7 @@ import (
type JSONResolver struct{}
// Resolve reads a JSON file and extracts the value at the specified path.
// Returns the value as a YAML string that can be parsed back into the config.
func (r *JSONResolver) Resolve(value string) (string, error) {
parts := strings.SplitN(value, ":", 2)
if len(parts) != 2 {
@ -28,9 +31,18 @@ func (r *JSONResolver) Resolve(value string) (string, error) {
}
// gjson supports JSON5 syntax including comments, trailing commas, etc.
// Special case: if path is ".", return the entire JSON as a string
// Special case: if path is ".", return the entire JSON
if jsonPath == "." {
return string(data), nil
// Parse and convert to YAML for consistency
var jsonData interface{}
if err := json.Unmarshal(data, &jsonData); err != nil {
return "", fmt.Errorf("failed to parse JSON: %w", err)
}
yamlBytes, err := yaml.Marshal(jsonData)
if err != nil {
return "", fmt.Errorf("failed to marshal to YAML: %w", err)
}
return strings.TrimSpace(string(yamlBytes)), nil
}
result := gjson.GetBytes(data, jsonPath)
@ -38,6 +50,20 @@ func (r *JSONResolver) Resolve(value string) (string, error) {
return "", fmt.Errorf("path %s not found in JSON file", jsonPath)
}
// Return the raw value as a string
// For complex types (objects/arrays), we need to parse and convert to YAML
if result.IsObject() || result.IsArray() {
// Use result.Value() to get the underlying interface{}
jsonData := result.Value()
// Convert to YAML
yamlBytes, err := yaml.Marshal(jsonData)
if err != nil {
return "", fmt.Errorf("failed to marshal to YAML: %w", err)
}
return strings.TrimSpace(string(yamlBytes)), nil
}
// For simple types, just return the string representation
return result.String(), nil
}

View File

@ -3,6 +3,7 @@ package smartconfig
import (
"fmt"
"os"
"strconv"
"strings"
"gopkg.in/yaml.v3"
@ -13,6 +14,7 @@ import (
type YAMLResolver struct{}
// Resolve reads a YAML file and extracts the value at the specified path.
// Returns the value as a YAML string that can be parsed back into the config.
func (r *YAMLResolver) Resolve(value string) (string, error) {
parts := strings.SplitN(value, ":", 2)
if len(parts) != 2 {
@ -22,6 +24,11 @@ func (r *YAMLResolver) Resolve(value string) (string, error) {
filePath := parts[0]
yamlPath := parts[1]
// Check for empty path
if yamlPath == "" {
return "", fmt.Errorf("empty YAML path")
}
data, err := os.ReadFile(filePath)
if err != nil {
return "", fmt.Errorf("failed to read YAML file %s: %w", filePath, err)
@ -32,12 +39,75 @@ func (r *YAMLResolver) Resolve(value string) (string, error) {
return "", fmt.Errorf("failed to parse YAML: %w", err)
}
// Simple YAML path evaluation (would need a proper library for complex paths)
if yamlPath == "." {
return fmt.Sprintf("%v", yamlData), nil
// Navigate the path
result, err := navigateYAMLPath(yamlData, yamlPath)
if err != nil {
return "", err
}
// This is a simplified implementation
// In production, use a proper YAML path library
return fmt.Sprintf("%v", yamlData), nil
// Convert the result back to YAML
yamlBytes, err := yaml.Marshal(result)
if err != nil {
return "", fmt.Errorf("failed to marshal result to YAML: %w", err)
}
// Return as YAML string (trim trailing newline for cleaner output)
return strings.TrimSpace(string(yamlBytes)), nil
}
// navigateYAMLPath traverses the YAML structure following the dot-separated path
func navigateYAMLPath(data interface{}, path string) (interface{}, error) {
// Handle root document request
if path == "." {
return data, nil
}
// Split path by dots
parts := strings.Split(path, ".")
current := data
for i, part := range parts {
switch v := current.(type) {
case map[string]interface{}:
// Navigate map
next, exists := v[part]
if !exists {
return nil, fmt.Errorf("path not found: %s (failed at '%s')", path, strings.Join(parts[:i+1], "."))
}
current = next
case map[interface{}]interface{}:
// YAML can have non-string keys, handle them
found := false
for key, value := range v {
if fmt.Sprintf("%v", key) == part {
current = value
found = true
break
}
}
if !found {
return nil, fmt.Errorf("path not found: %s (failed at '%s')", path, strings.Join(parts[:i+1], "."))
}
case []interface{}:
// Handle array access
index, err := strconv.Atoi(part)
if err != nil {
return nil, fmt.Errorf("invalid array index '%s' in path %s", part, path)
}
if index < 0 || index >= len(v) {
return nil, fmt.Errorf("array index out of bounds: %d (array length: %d)", index, len(v))
}
current = v[index]
default:
// Can't navigate further
if i < len(parts)-1 {
return nil, fmt.Errorf("cannot navigate path %s: '%s' is not a map or array", path, strings.Join(parts[:i], "."))
}
}
}
return current, nil
}

246
resolver_yaml_test.go Normal file
View File

@ -0,0 +1,246 @@
package smartconfig
import (
"os"
"testing"
)
func TestYAMLResolverPathNavigation(t *testing.T) {
resolver := &YAMLResolver{}
tests := []struct {
name string
input string
expected string
wantErr bool
}{
// Basic path navigation
{
name: "top level key",
input: "./test/features.yaml:settings",
expected: "debug: false\nlog_level: info",
wantErr: false,
},
{
name: "nested path with dot notation",
input: "./test/database.yaml:production.primary",
expected: "host: prod-db-primary.example.com\nport: 5432\nusername: prod_user",
wantErr: false,
},
{
name: "deeply nested path",
input: "./test/database.yaml:production.replica.host",
expected: "prod-db-replica.example.com",
wantErr: false,
},
{
name: "boolean value",
input: "./test/features.yaml:features.analytics.enabled",
expected: "true",
wantErr: false,
},
{
name: "string value",
input: "./test/features.yaml:features.analytics.provider",
expected: "google",
wantErr: false,
},
{
name: "numeric value",
input: "./test/features.yaml:features.rate_limiting.requests_per_minute",
expected: "100",
wantErr: false,
},
{
name: "root document with dot",
input: "./test/features.yaml:.",
expected: "features:\n analytics:\n enabled: true\n provider: google\n tracking_id: UA-123456\n new_ui:\n enabled: false\n rollout_percentage: 25\n rate_limiting:\n enabled: true\n requests_per_minute: 100\nsettings:\n debug: false\n log_level: info",
wantErr: false,
},
// Error cases
{
name: "non-existent path",
input: "./test/database.yaml:production.nonexistent.key",
expected: "",
wantErr: true,
},
{
name: "invalid file",
input: "./test/nonexistent.yaml:some.path",
expected: "",
wantErr: true,
},
{
name: "missing path separator",
input: "./test/database.yaml",
expected: "",
wantErr: true,
},
{
name: "empty path",
input: "./test/database.yaml:",
expected: "",
wantErr: true,
},
// Edge cases with special characters
{
name: "path with spaces (if exists)",
input: "./test/features.yaml:settings.log_level",
expected: "info",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := resolver.Resolve(tt.input)
if tt.wantErr {
if err == nil {
t.Errorf("Expected error but got none, result: %s", result)
}
} else {
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if result != tt.expected {
t.Errorf("Expected %q, got %q", tt.expected, result)
}
}
})
}
}
func TestYAMLResolverArrayAccess(t *testing.T) {
// Create a test YAML file with arrays
yamlContent := `servers:
- name: server1
host: 192.168.1.1
port: 8080
- name: server2
host: 192.168.1.2
port: 8081
users:
- admin
- user1
- user2
`
// Write test file
err := os.WriteFile("./test/arrays.yaml", []byte(yamlContent), 0644)
if err != nil {
t.Fatalf("Failed to create test file: %v", err)
}
defer func() { _ = os.Remove("./test/arrays.yaml") }()
resolver := &YAMLResolver{}
tests := []struct {
name string
input string
expected string
wantErr bool
}{
{
name: "array index access",
input: "./test/arrays.yaml:servers.0",
expected: "host: 192.168.1.1\nname: server1\nport: 8080",
wantErr: false,
},
{
name: "array element property",
input: "./test/arrays.yaml:servers.1.host",
expected: "192.168.1.2",
wantErr: false,
},
{
name: "simple array element",
input: "./test/arrays.yaml:users.0",
expected: "admin",
wantErr: false,
},
{
name: "out of bounds array access",
input: "./test/arrays.yaml:servers.10",
expected: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := resolver.Resolve(tt.input)
if tt.wantErr {
if err == nil {
t.Errorf("Expected error but got none, result: %s", result)
}
} else {
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if result != tt.expected {
t.Errorf("Expected %q, got %q", tt.expected, result)
}
}
})
}
}
func TestYAMLResolverComplexPaths(t *testing.T) {
// Test quoted paths and special cases
yamlContent := `special:
"dotted.key": value1
"key with spaces": value2
123: numeric_key
nested:
level1:
level2:
level3:
level4:
deepvalue: "found it!"
`
err := os.WriteFile("./test/complex.yaml", []byte(yamlContent), 0644)
if err != nil {
t.Fatalf("Failed to create test file: %v", err)
}
defer func() { _ = os.Remove("./test/complex.yaml") }()
resolver := &YAMLResolver{}
tests := []struct {
name string
input string
expected string
wantErr bool
}{
{
name: "deeply nested value",
input: "./test/complex.yaml:nested.level1.level2.level3.level4.deepvalue",
expected: "found it!",
wantErr: false,
},
// Note: Quoted keys would require special handling in the path parser
// For now, we'll stick to simple dot notation
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := resolver.Resolve(tt.input)
if tt.wantErr {
if err == nil {
t.Errorf("Expected error but got none, result: %s", result)
}
} else {
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if result != tt.expected {
t.Errorf("Expected %q, got %q", tt.expected, result)
}
}
})
}
}

View File

@ -595,6 +595,18 @@ func (c *Config) interpolateMixedContent(s string, depth int) (string, error) {
// convertToType attempts to convert a string to its appropriate type
func (c *Config) convertToType(s string) interface{} {
// First try to parse as YAML - this handles complex structures
// returned by JSON/YAML resolvers
var yamlData interface{}
if err := yaml.Unmarshal([]byte(s), &yamlData); err == nil {
// If it's a complex type (map or slice), return it directly
switch yamlData.(type) {
case map[string]interface{}, map[interface{}]interface{}, []interface{}:
return yamlData
}
// For simple types parsed from YAML, continue with normal conversion
}
// Try boolean
if s == "true" {
return true

View File

@ -14,8 +14,8 @@ api:
tls_host: ${JSON:./test/hosts.json:tls_sni_host.0}
database:
host: ${YAML:./test/data.yaml:'.database.host'}
version: ${YAML:./test/data.yaml:'.version'}
host: ${YAML:./test/data.yaml:database.host}
version: ${YAML:./test/data.yaml:version}
nested:
value: ${ENV:NESTED_${ENV:TEST_ENV_SUFFIX}}

20
test/database.yaml Normal file
View File

@ -0,0 +1,20 @@
production:
primary:
host: prod-db-primary.example.com
port: 5432
username: prod_user
replica:
host: prod-db-replica.example.com
port: 5432
staging:
primary:
host: staging-db.example.com
port: 5432
username: staging_user
development:
primary:
host: localhost
port: 5432
username: dev_user

15
test/features.yaml Normal file
View File

@ -0,0 +1,15 @@
features:
analytics:
enabled: true
provider: google
tracking_id: UA-123456
new_ui:
enabled: false
rollout_percentage: 25
rate_limiting:
enabled: true
requests_per_minute: 100
settings:
debug: false
log_level: info