fix: allow values.yaml to include go template functions

* also support referencing logical Parameters in a `parameters.yaml` file which can include a logical structure + schema (for nice install tooling) which then contains inline values for simple values or URLs to vault/local secret files for better secret management

fixes #4328

Signed-off-by: James Strachan <james.strachan@gmail.com>

go.sum

@@ -43,6 +43,7 @@ github.com/Masterminds/semver v1.3.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/sprig v0.0.0-20180403013413-6b2a58267f6a/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
+github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w=
 github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Microsoft/go-winio v0.4.6 h1:Tu8dlnF1wvUKKqr011GFneCoyIn7D+Q2uq6AKmQnGrA=
 github.com/Microsoft/go-winio v0.4.6/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@@ -85,6 +86,7 @@ github.com/antham/chyle v1.4.0 h1:RBCXpnmj3Wcl43bKbcuRU3LXkarhYwSigWAPUyWEjvY=
 github.com/antham/chyle v1.4.0/go.mod h1:D94Z4aE/ECudyNoTHwkhqu77mjGPZtfPG8dNoeIG9CU=
 github.com/antham/envh v1.2.0/go.mod h1:ocIRPHuwwjyBVBtuUJOJc2TYzGg+d23xSAZexl4y9hQ=
 github.com/antham/strumt v0.0.0-20171215230529-6776189777d3/go.mod h1:sE7EYIUE0nQzPiv5zQAmw2aVkei0j2xmb4gTIIqSFSI=
+github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
 github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/appscode/jsonpatch v0.0.0-20180911074601-5af499cf01c8/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M=
@@ -391,6 +393,7 @@ github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c h1:kp3AxgXgDOmIJFR7b
 github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13 h1:0XE8qtre7NNhsKWo+PuYJNmoR3szkSzpDtZLEW+5HE0=
 github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=

pkg/cmd/opts/common.go

@@ -1142,3 +1142,20 @@ func (o *CommonOptions) IsFlagExplicitlySet(flagName string) bool {
 	o.Cmd.Flags().Visit(explicitlySetFunc)
 	return explicit
 }
+
+// GetClusterName returns the current cluster name
+func (o *CommonOptions) GetClusterName() (string, error) {
+	kubeClient, ns, err := o.KubeClientAndDevNamespace()
+	if err != nil {
+		return "", err
+	}
+	data, err := kube.ReadInstallValues(kubeClient, ns)
+	if err != nil {
+		return "", err
+	}
+	answer := data[kube.ClusterName]
+	if answer == "" {
+		answer = "default"
+	}
+	return answer, nil
+}

pkg/cmd/opts/helm.go

@@ -10,6 +10,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jenkins-x/jx/pkg/io/secrets"
+	"github.com/jenkins-x/jx/pkg/secreturl"
+	"github.com/jenkins-x/jx/pkg/secreturl/localvault"
 	"github.com/pborman/uuid"
 
 	"github.com/jenkins-x/jx/pkg/environments"
@@ -403,11 +406,28 @@ func (o *CommonOptions) InstallChartWithOptionsAndTimeout(options helm.InstallCh
 			return err
 		}
 	}
+	secretUrlClient, err := o.GetSecretURLClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to create a Secret RL client")
+	}
+	return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, secretUrlClient)
+}
+
+// GetSecretURLClient create a new secret URL client
+func (o *CommonOptions) GetSecretURLClient() (secreturl.Client, error) {
+	if o.GetSecretsLocation() == secrets.FileSystemLocationKind {
+		clusterName, err := o.GetClusterName()
+		if err != nil {
+			return nil, err
+		}
+		dir, err := util.LocalFileSystemSecretsDir(clusterName)
+		return localvault.NewFileSystemClient(dir), nil
+	}
 	vaultClient, err := o.SystemVaultClient(o.devNamespace)
 	if err != nil {
 		vaultClient = nil
 	}
-	return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, vaultClient)
+	return vaultClient, nil
 }
 
 // CloneJXVersionsRepo clones the jenkins-x versions repo to a local working dir

pkg/cmd/step/helm/step_helm_apply.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/jenkins-x/jx/pkg/cmd/helper"
-
 	"github.com/jenkins-x/jx/pkg/cmd/opts"
 	"github.com/jenkins-x/jx/pkg/cmd/templates"
 	"github.com/jenkins-x/jx/pkg/helm"
@@ -194,7 +193,11 @@ func (o *StepHelmApplyOptions) Run() error {
 		}()
 	}
 
-	chartValues, err := helm.GenerateValues(dir, nil, true)
+	secretUrlClient, err := o.GetSecretURLClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to create a Secret RL client")
+	}
+	chartValues, err := helm.GenerateValues(dir, nil, true, secretUrlClient)
 	if err != nil {
 		return errors.Wrapf(err, "generating values.yaml for tree from %s", dir)
 	}

pkg/helm/helm_helpers.go

@@ -47,6 +47,10 @@ const (
 	// TemplatesDirName is the default name for the templates directory
 	TemplatesDirName = "templates"
 
+	// ParametersYAMLFile contains logical parameters (values or secrets) which can be fetched from a Secret URL or
+	// inlined if not a secret which can be referenced from a 'values.yaml` file via a `{{ .Parameters.foo.bar }}` expression
+	ParametersYAMLFile = "parameters.yaml"
+
 	// InClusterHelmRepositoryURL is the default cluster local helm repo
 	InClusterHelmRepositoryURL = "http://jenkins-x-chartmuseum:8080"
 
@@ -595,9 +599,12 @@ func DecorateWithSecrets(options *InstallChartOptions, vaultClient secreturl.Cli
 			if err != nil {
 				return cleanup, errors.Wrapf(err, "reading file %s", valueFile)
 			}
-			newValues, err := vaultClient.ReplaceURIs(string(bytes))
-			if err != nil {
-				return cleanup, errors.Wrapf(err, "replacing vault URIs")
+			newValues := string(bytes)
+			if vaultClient != nil {
+				newValues, err = vaultClient.ReplaceURIs(newValues)
+				if err != nil {
+					return cleanup, errors.Wrapf(err, "replacing vault URIs")
+				}
 			}
 			err = ioutil.WriteFile(newValuesFile.Name(), []byte(newValues), 0600)
 			if err != nil {
@@ -610,6 +617,35 @@ func DecorateWithSecrets(options *InstallChartOptions, vaultClient secreturl.Cli
 	return cleanup, nil
 }
 
+// LoadParameters loads the 'parameters.yaml' file if it exists in the current directory
+func LoadParameters(dir string, vaultClient secreturl.Client) (chartutil.Values, error) {
+	fileName := filepath.Join(dir, ParametersYAMLFile)
+	exists, err := util.FileExists(fileName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "checking %s exists", fileName)
+	}
+	m := map[string]interface{}{}
+	if exists {
+		data, err := ioutil.ReadFile(fileName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "reading %s", fileName)
+		}
+		if vaultClient != nil {
+			text, err := vaultClient.ReplaceURIs(string(data))
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to convert secret URLs in parameters file %s", fileName)
+			}
+			data = []byte(text)
+		}
+
+		m, err = LoadValues(data)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unmarshaling %s", fileName)
+		}
+	}
+	return chartutil.Values(m), err
+}
+
 // AddHelmRepoIfMissing will add the helm repo if there is no helm repo with that url present.
 // It will generate the repoName from the url (using the host name) if the repoName is empty.
 // The repo name may have a suffix added in order to prevent name collisions, and is returned for this reason.

pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/adminUser.yaml

@@ -0,0 +1 @@
+password-passthrough: myDockerRegistryPassword

pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/dockerRegistry.yaml

@@ -0,0 +1 @@
+password-passthrough: myAdminPassword

pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/pipelineUser.yaml

@@ -0,0 +1 @@
+token-passthrough: myPipelineUserToken

pkg/helm/test_data/tree_of_values_yaml_templates/parameters.yaml

@@ -0,0 +1,17 @@
+adminUser:
+  password: local:/my-cheese-cluster/adminUser:password-passthrough
+  username: admin
+docker:
+  password: local:/my-cheese-cluster/dockerRegistry:password-passthrough
+  url: https://index.docker.io/v1/
+  username: james
+enableDocker: true
+enableGpg: false
+gitProvider: github
+pipelineUser:
+  github:
+    host: github.com
+    password: local:/my-cheese-cluster/pipelineUser:token-passthrough
+    username: james
+prow:
+  hmacToken: abc

pkg/helm/test_data/tree_of_values_yaml_templates/prow/values.yaml

@@ -0,0 +1 @@
+hmacToken: {{ .Parameters.prow.hmacToken }}

pkg/helm/test_data/tree_of_values_yaml_templates/tekton/values.yaml

@@ -0,0 +1,4 @@
+auth:
+  git:
+    username: {{ .Parameters.pipelineUser.github.username }}
+    password: {{ .Parameters.pipelineUser.github.password }}

pkg/helm/test_data/tree_of_values_yaml_templates/values.yaml

@@ -0,0 +1 @@
+dummy: cheese

pkg/helm/values_tree.go

@@ -1,13 +1,19 @@
 package helm
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+	"text/template"
 
+	"github.com/jenkins-x/jx/pkg/secreturl"
 	"github.com/jenkins-x/jx/pkg/util"
+	"github.com/pkg/errors"
+	"k8s.io/helm/pkg/chartutil"
+	"k8s.io/helm/pkg/engine"
 
 	"github.com/ghodss/yaml"
 
@@ -26,7 +32,7 @@ var DefaultValuesTreeIgnores = []string{
 // Any keys used that match files with the same name in the directory (
 // and have empty values) will be inlined as block scalars.
 // Standard UNIX glob patterns can be passed to IgnoreFile directories.
-func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error) {
+func GenerateValues(dir string, ignores []string, verbose bool, secretUrlClient secreturl.Client) ([]byte, error) {
 	info, err := os.Stat(dir)
 	if err != nil {
 		return nil, err
@@ -35,6 +41,15 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error)
 	} else if !info.IsDir() {
 		return nil, fmt.Errorf("%s is not a directory", dir)
 	}
+
+	// load the parameter values if there are any
+	params, err := LoadParameters(dir, secretUrlClient)
+	if err != nil {
+		return nil, err
+	}
+	funcMap := engine.FuncMap()
+	funcMap["hashPassword"] = util.HashPassword
+
 	if ignores == nil {
 		ignores = DefaultValuesTreeIgnores
 	}
@@ -54,7 +69,7 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error)
 			if rDir != "" {
 				// If it's values.yaml, then read and parse it
 				if file == "values.yaml" {
-					b, err := ioutil.ReadFile(path)
+					b, err := ReadValuesYamlFileTemplateOutput(path, params, funcMap)
 					if err != nil {
 						return err
 					}
@@ -141,6 +156,25 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error)
 	return yaml.Marshal(rootValues)
 }
 
+// ReadValuesYamlFileTemplateOutput evaluates the given values.yaml file as a go template and returns the output data
+func ReadValuesYamlFileTemplateOutput(templateFile string, params chartutil.Values, funcMap template.FuncMap) ([]byte, error) {
+	tmpl, err := template.New(ValuesFileName).Option("missingkey=error").Funcs(funcMap).ParseFiles(templateFile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse Secrets template: %s", templateFile)
+	}
+
+	templateData := map[string]interface{}{
+		"Parameters": chartutil.Values(params),
+	}
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, templateData)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to execute Secrets template: %s", templateFile)
+	}
+	data := buf.Bytes()
+	return data, nil
+}
+
 // HandleExternalFileRefs recursively scans the element map structure,
 // looking for nested maps. If it finds keys that match any key-value pair in possibles it will call the handler.
 // The jsonPath is used for referencing the path in the map structure when reporting errors.

pkg/helm/values_tree_template_test.go

@@ -0,0 +1,33 @@
+package helm_test
+
+import (
+	"path"
+	"testing"
+
+	"github.com/jenkins-x/jx/pkg/helm"
+	"github.com/jenkins-x/jx/pkg/secreturl/localvault"
+	"github.com/stretchr/testify/assert"
+)
+
+var expectedTemplatedValuesTree = `dummy: cheese
+prow:
+  hmacToken: abc
+tekton:
+  auth:
+    git:
+      password: myPipelineUserToken
+      username: james
+`
+
+func TestValuesTreeTemplates(t *testing.T) {
+	t.Parallel()
+
+	testData := path.Join("test_data", "tree_of_values_yaml_templates")
+
+	localVaultDir := path.Join(testData, "local_vault_files")
+	secretUrlClient := localvault.NewFileSystemClient(localVaultDir)
+
+	result, err := helm.GenerateValues(testData, nil, true, secretUrlClient)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedTemplatedValuesTree, string(result))
+}

pkg/helm/values_tree_test.go

@@ -29,7 +29,7 @@ meat:
 		assert.NoError(t, err)
 	}()
 	assert.NoError(t, err)
-	result, err := helm.GenerateValues(dir, nil, true)
+	result, err := helm.GenerateValues(dir, nil, true, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedOutput, string(result))
 }
@@ -55,7 +55,7 @@ people: pete
 		assert.NoError(t, err)
 	}()
 	assert.NoError(t, err)
-	result, err := helm.GenerateValues(dir, nil, true)
+	result, err := helm.GenerateValues(dir, nil, true, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedOutput, string(result))
 }
@@ -84,7 +84,7 @@ func TestValuesTreeWithFileRefs(t *testing.T) {
 		assert.NoError(t, err)
 	}()
 	assert.NoError(t, err)
-	result, err := helm.GenerateValues(dir, nil, true)
+	result, err := helm.GenerateValues(dir, nil, true, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedOutput, string(result))
 }

pkg/util/dirs.go

@@ -45,6 +45,15 @@ func ConfigDir() (string, error) {
 	return path, nil
 }
 
+// LocalFileSystemSecretsDir returns the default local file system secrets location for the file system alternative to vault
+func LocalFileSystemSecretsDir(clusterName string) (string, error) {
+	home, err := ConfigDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, "localSecrets", clusterName), nil
+}
+
 // KubeConfigFile gets the .kube/config file
 func KubeConfigFile() string {
 	path := os.Getenv("KUBECONFIG")

pkg/vault/helpers_test.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"github.com/jenkins-x/jx/pkg/secreturl"
 	"github.com/pborman/uuid"
 
 	"github.com/stretchr/testify/assert"
@@ -35,7 +34,7 @@ func TestReplaceURIs(t *testing.T) {
 	pegomock.When(vaultClient.Read(pegomock.EqString(path))).ThenReturn(map[string]interface{}{
 		key: secret,
 	}, nil)
-	result, err := secreturl.ReplaceURIs(valuesyaml, vaultClient)
+	result, err := vaultClient.ReplaceURIs(valuesyaml)
 	assert.NoError(t, err)
 	assert.NoError(t, err)
 	assert.Equal(t, fmt.Sprintf(`foo:
@@ -63,7 +62,7 @@ func TestReplaceRealExampleURI(t *testing.T) {
 	pegomock.When(vaultClient.Read(pegomock.EqString(path))).ThenReturn(map[string]interface{}{
 		key: secret,
 	}, nil)
-	result, err := secreturl.ReplaceURIs(valuesyaml, vaultClient)
+	result, err := vaultClient.ReplaceURIs(valuesyaml)
 	assert.NoError(t, err)
 	assert.NoError(t, err)
 	assert.Equal(t, fmt.Sprintf(`foo: