Merge pull request #30 from asecurityteam/feature/var-substitution

implement var substitution feature

README.md

@@ -9,14 +9,15 @@
 <!-- TOC -->
 
 - [Settings - Typed Configuration Toolkit For Go](#settings---typed-configuration-toolkit-for-go)
-    - [Overview](#overview)
-    - [Data Sources](#data-sources)
-    - [Component API](#component-api)
-    - [Hierarchy API](#hierarchy-api)
-    - [Adapter API](#adapter-api)
-    - [Contributing](#contributing)
-        - [License](#license)
-        - [Contributing Agreement](#contributing-agreement)
+  - [Overview](#overview)
+  - [Data Sources](#data-sources)
+  - [Component API](#component-api)
+  - [Hierarchy API](#hierarchy-api)
+  - [Adapter API](#adapter-api)
+  - [Special Type Parsing and Casting](#special-type-parsing-and-casting)
+  - [Contributing](#contributing)
+    - [License](#license)
+    - [Contributing Agreement](#contributing-agreement)
 
 <!-- /TOC -->
 
@@ -91,12 +92,50 @@ prefixedEnv := &settings.PrefixSource{
     Prefix: []string{"APP"},
 }
 // Create an ordered lookup where ENV is first with a fallback to
-// YAML which can fallback to JSON.
+// YAML which can fallback to JSON.  The first found is used.
 finalSource := []settings.MultiSource{prefixedEnv, yamlSource, jsonSource}
 
 v, found := finalSource.Get(context.Background(), "setting")
 ```
 
+The built-in sources, including `MultiSource`, support variable substitution, so that you can effectively perform variable mapping.  For example, in the following, the final value of `B_BB` key will
+be `envValue`.
+
+*config.yaml* (be sure to wrap reference values in quotes so the yaml parser interprets it as a string)
+```yaml
+b:
+  bb: "${A_AA}"
+```
+
+*Environment Variable*
+```shell
+A_AA="envValue"
+```
+
+```golang
+yamlSource, _ := settings.NewFileSource("config.yaml")
+envSource := settings.NewEnvSource(os.Environ())
+finalSource := []settings.MultiSource{yamlSource, envSource}
+```
+
+Recursion protection is such that recursing to a depth of ten will result in the value
+at the "top" of the recursion stack being returned.  For example, the following will
+return a literal unexpanded value `${b}` when getting key `a` due to infinite recursion
+protection.
+
+```yaml
+a: "${b}"
+b: "${c}"
+c: "${a}"
+```
+
+Similarly, the literal value is returned when no expansion is possible.  The following will
+return a literal unexpanded value `${b}` when getting key `a`:
+
+```yaml
+a: "${b}"
+```
+
 Sources may be used as-is by passing them around to components that need to fetch
 values. However, the values returned from `Get()` are opaque and highly dependent on
 the implementation. For example, the ENV source will always return a string

source.go

@@ -6,11 +6,20 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	infiniteRecursionDepthLimit = 10
+)
+
+// envPattern is used for matching strings the lib user intends to have
+// substituted by recursing through the sources to find the "final" value
+var envPattern = regexp.MustCompile(`\${[^}]+}`)
+
 // Source is the main entry point for fetching configuration
 // values. The boolean value must be false if the source is
 // unable to find an entry for the given path. If found, the
@@ -25,7 +34,8 @@ type Source interface {
 // sources which would be JSON, YAML, and ENV.
 //
 // Note: All keys should lower case (if applicable for the character set)
-// 		 as lower case will also be applied to all lookup paths.
+//
+//	as lower case will also be applied to all lookup paths.
 type MapSource struct {
 	Map map[string]interface{}
 }
@@ -67,8 +77,16 @@ func NewMapSource(m map[string]interface{}) *MapSource {
 }
 
 // Get traverses a configuration map until it finds the requested element
-// or reaches a dead end.
+// or reaches a dead end.  Variable expansion is supported when the value
+// is a string with ${} wrapped around a key.
 func (s *MapSource) Get(_ context.Context, path ...string) (interface{}, bool) {
+	return s.getRecursive(nil, 0, path...)
+}
+
+func (s *MapSource) getRecursive(valueAtTopOfRecursionStack interface{}, recursionDepth int, path ...string) (interface{}, bool) {
+	if recursionDepth > infiniteRecursionDepthLimit {
+		return nil, false
+	}
 	location := s.Map
 	for x := 0; x < len(path)-1; x = x + 1 {
 		pth := strings.ToLower(path[x])
@@ -81,6 +99,25 @@ func (s *MapSource) Get(_ context.Context, path ...string) (interface{}, bool) {
 		}
 	}
 	v, ok := location[strings.ToLower(path[len(path)-1])]
+	if !ok {
+		return valueAtTopOfRecursionStack, valueAtTopOfRecursionStack != nil
+	}
+
+	if vString, okString := v.(string); okString && ok && envPattern.Match([]byte(vString)) {
+		// value is a string wrapped in "${}"; turn the value into a path and keep searching,
+		// but keep the value at the top of the recursion stack in case we don't find anything
+		// and need to return it as-is
+		if valueAtTopOfRecursionStack == nil {
+			valueAtTopOfRecursionStack = vString
+		}
+		key := unwrap([]byte(vString))
+		subValue, subFound := s.getRecursive(valueAtTopOfRecursionStack, recursionDepth+1, strings.Split(string(key), "_")...)
+		if !subFound {
+			return valueAtTopOfRecursionStack, true
+		}
+		return subValue, subFound
+	}
+
 	return v, ok
 }
 
@@ -211,7 +248,7 @@ func NewEnvSource(env []string) (*MapSource, error) {
 	return NewMapSource(m), nil
 }
 
-// PrefixSource is a wrapper for other Source implementaions that adds
+// PrefixSource is a wrapper for other Source implementations that adds
 // a path element to the front of every lookup.
 type PrefixSource struct {
 	Source Source
@@ -231,13 +268,49 @@ func (s *PrefixSource) Get(ctx context.Context, path ...string) (interface{}, bo
 // value or will return false for found.
 type MultiSource []Source
 
-// Get a value from the ordered set of Sources.
-func (s MultiSource) Get(ctx context.Context, path ...string) (interface{}, bool) {
-	for _, ss := range s {
-		v, found := ss.Get(ctx, path...)
+// Get a value from the ordered set of Sources.  Variable expansion is supported when the value
+// is a string with ${} wrapped around a key.
+func (ms MultiSource) Get(ctx context.Context, path ...string) (interface{}, bool) {
+	return ms.getRecursive(ctx, nil, 0, path...)
+}
+
+func (ms MultiSource) getRecursive(ctx context.Context, valueAtTopOfRecursionStack interface{}, recursionDepth int, path ...string) (interface{}, bool) {
+	if recursionDepth > infiniteRecursionDepthLimit {
+		return nil, false
+	}
+	var v interface{}
+	var found bool
+	for _, ss := range ms {
+		v, found = ss.Get(ctx, path...)
 		if found {
-			return v, found
+			break
+		}
+	}
+
+	if vString, ok := v.(string); ok && envPattern.Match([]byte(vString)) {
+		// value is a string wrapped in "${}"; turn the value into a path and keep searching,
+		// but keep the value at the top of the recursion stack in case we don't find anything
+		// and need to return it as-is
+		if valueAtTopOfRecursionStack == nil {
+			valueAtTopOfRecursionStack = vString
 		}
+		key := unwrap([]byte(vString))
+		subValue, subFound := ms.getRecursive(ctx, valueAtTopOfRecursionStack, recursionDepth+1, strings.Split(string(key), "_")...)
+		if !subFound {
+			return valueAtTopOfRecursionStack, true
+		}
+		return subValue, subFound
+	}
+
+	if !found {
+		return valueAtTopOfRecursionStack, valueAtTopOfRecursionStack != nil
 	}
-	return nil, false
+
+	return v, found
+}
+
+func unwrap(source []byte) []byte {
+	return envPattern.ReplaceAllFunc(source, func(match []byte) []byte {
+		return match[2 : len(match)-1] // strip ${}
+	})
 }