Add support for variables in bundle config

bundle/config/interpolation/interpolation.go

@@ -174,7 +174,7 @@ func (a *accumulator) Resolve(path string, seenPaths []string, fns ...LookupFunc
 	// fetch the string node to resolve
 	field, ok := a.strings[path]
 	if !ok {
-		return fmt.Errorf("could not find string field with path %s", path)
+		return fmt.Errorf("could not resolve reference %s", path)
 	}
 
 	// return early if the string field has no variables to interpolate

bundle/config/mutator/load_variables_from_environment.go

@@ -0,0 +1,55 @@
+package mutator
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/databricks/bricks/bundle"
+)
+
+const bundleVarPrefix = "BUNDLE_VAR_"
+
+type loadVariablesFromEnvironment struct{}
+
+func LoadVariablesFromEnvionment() bundle.Mutator {
+	return &loadVariablesFromEnvironment{}
+}
+
+func (m *loadVariablesFromEnvironment) Name() string {
+	return "LoadVariablesFromEnvionment"
+}
+
+func parseBundleVar(s string) (key string, val string, err error) {
+	// env var name show have the bundle prefix to parse it
+	if !strings.HasPrefix(s, bundleVarPrefix) {
+		return "", "", fmt.Errorf("environment variable %s does not have expected prefix %s", s, bundleVarPrefix)
+	}
+
+	// split the env var entry at '=' to get the variable name and variable value
+	components := strings.Split(s, "=")
+	if len(components) != 2 {
+		return "", "", fmt.Errorf("unexpected format for environment variable: %s", s)
+	}
+	key = strings.ToLower(
+		strings.TrimPrefix(components[0], bundleVarPrefix),
+	)
+	val = components[1]
+	return
+}
+
+func (m *loadVariablesFromEnvironment) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
+	envVars := os.Environ()
+	for _, envVar := range envVars {
+		if !strings.HasPrefix(envVar, bundleVarPrefix) {
+			continue
+		}
+		varName, varVal, err := parseBundleVar(envVar)
+		if err != nil {
+			return nil, err
+		}
+		b.Config.Variables[varName] = varVal
+	}
+	return nil, nil
+}

bundle/config/mutator/load_variables_from_environment_test.go

@@ -0,0 +1,27 @@
+package mutator
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestVariablesParseBundleVar(t *testing.T) {
+	s := `BUNDLE_VAR_FOO=Bar`
+	key, val, err := parseBundleVar(s)
+	assert.NoError(t, err)
+	assert.Equal(t, "foo", key)
+	assert.Equal(t, "Bar", val)
+}
+
+func TestVariablesParseBundleVarOnInvalidPrefix(t *testing.T) {
+	s := `INVALID_PREFIX_FOO=Bar`
+	_, _, err := parseBundleVar(s)
+	assert.ErrorContains(t, err, "environment variable INVALID_PREFIX_FOO=Bar does not have expected prefix BUNDLE_VAR_")
+}
+
+func TestVariablesParseBundleVarOnInvalidFormat(t *testing.T) {
+	s := `BUNDLE_VAR_FOO=Bar=bar`
+	_, _, err := parseBundleVar(s)
+	assert.ErrorContains(t, err, "unexpected format for environment variable: BUNDLE_VAR_FOO=Bar=bar")
+}

bundle/config/root.go

@@ -16,6 +16,10 @@ type Root struct {
 	// It is set when loading `bundle.yml`.
 	Path string `json:"-" bundle:"readonly"`
 
+	// Contains user defined variables, either defined in the bundle config or
+	// loaded through env vars
+	Variables map[string]string `json:"variables"`
+
 	// Bundle contains details about this bundle, such as its name,
 	// version of the spec (TODO), default cluster, default warehouse, etc.
 	Bundle Bundle `json:"bundle"`

bundle/phases/initialize.go

@@ -18,9 +18,11 @@ func Initialize() bundle.Mutator {
 			mutator.DefineDefaultWorkspaceRoot(),
 			mutator.ExpandWorkspaceRoot(),
 			mutator.DefineDefaultWorkspacePaths(),
+			mutator.LoadVariablesFromEnvionment(),
 			interpolation.Interpolate(
 				interpolation.IncludeLookupsInPath("bundle"),
 				interpolation.IncludeLookupsInPath("workspace"),
+				interpolation.IncludeLookupsInPath("variables"),
 			),
 			mutator.TranslatePaths(),
 			terraform.Initialize(),

bundle/tests/variables/bundle.yml

@@ -0,0 +1,7 @@
+variables:
+  a: foo
+  b: bar
+
+
+bundle:
+  name: "name: ${variables.a} ${variables.b} ${variables.c}"

bundle/tests/variables_test.go

@@ -0,0 +1,42 @@
+package config_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/bundle/config/interpolation"
+	"github.com/databricks/bricks/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVariables(t *testing.T) {
+	t.Setenv("BUNDLE_VAR_B", "bar_from_env")
+	t.Setenv("BUNDLE_VAR_C", "world")
+	b := load(t, "./variables")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.LoadVariablesFromEnvionment(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath("bundle"),
+			interpolation.IncludeLookupsInPath("workspace"),
+			interpolation.IncludeLookupsInPath("variables"),
+		)})
+	require.NoError(t, err)
+	assert.Equal(t, "name: foo bar_from_env world", b.Config.Bundle.Name)
+	assert.Equal(t, "foo", b.Config.Variables["a"])
+	assert.Equal(t, "bar_from_env", b.Config.Variables["b"])
+	assert.Equal(t, "world", b.Config.Variables["c"])
+}
+
+func TestVariablesWhenUndefinedVarIsUsed(t *testing.T) {
+	b := load(t, "./variables")
+	err := bundle.Apply(context.Background(), b, []bundle.Mutator{
+		mutator.LoadVariablesFromEnvionment(),
+		interpolation.Interpolate(
+			interpolation.IncludeLookupsInPath("bundle"),
+			interpolation.IncludeLookupsInPath("workspace"),
+			interpolation.IncludeLookupsInPath("variables"),
+		)})
+	assert.ErrorContains(t, err, "could not resolve reference variables.c")
+}