fix: refactor out a sub-interface from Vault for VaultURL injection

go.sum

@@ -43,6 +43,7 @@ github.com/Masterminds/semver v1.3.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/sprig v0.0.0-20180403013413-6b2a58267f6a/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
+github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w=
 github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Microsoft/go-winio v0.4.6 h1:Tu8dlnF1wvUKKqr011GFneCoyIn7D+Q2uq6AKmQnGrA=
 github.com/Microsoft/go-winio v0.4.6/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@@ -85,6 +86,7 @@ github.com/antham/chyle v1.4.0 h1:RBCXpnmj3Wcl43bKbcuRU3LXkarhYwSigWAPUyWEjvY=
 github.com/antham/chyle v1.4.0/go.mod h1:D94Z4aE/ECudyNoTHwkhqu77mjGPZtfPG8dNoeIG9CU=
 github.com/antham/envh v1.2.0/go.mod h1:ocIRPHuwwjyBVBtuUJOJc2TYzGg+d23xSAZexl4y9hQ=
 github.com/antham/strumt v0.0.0-20171215230529-6776189777d3/go.mod h1:sE7EYIUE0nQzPiv5zQAmw2aVkei0j2xmb4gTIIqSFSI=
+github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
 github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/appscode/jsonpatch v0.0.0-20180911074601-5af499cf01c8/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M=
@@ -391,6 +393,7 @@ github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c h1:kp3AxgXgDOmIJFR7b
 github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13 h1:0XE8qtre7NNhsKWo+PuYJNmoR3szkSzpDtZLEW+5HE0=
 github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=

pkg/apps/values.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/jenkins-x/jx/pkg/secreturl"
 	"gopkg.in/AlecAivazis/survey.v1/terminal"
 
 	"github.com/ghodss/yaml"
@@ -187,7 +188,7 @@ func GenerateQuestions(schema []byte, batchMode bool, askExisting bool, basePath
 			secrets = append(secrets, secret)
 			if passthrough {
 				if useVault {
-					return vault.ToURI(secret.Path, secret.Key), nil
+					return secreturl.ToURI(secret.Path, secret.Key), nil
 				}
 				return value, nil
 			}

pkg/cmd/opts/common.go

@@ -1142,3 +1142,16 @@ func (o *CommonOptions) IsFlagExplicitlySet(flagName string) bool {
 	o.Cmd.Flags().Visit(explicitlySetFunc)
 	return explicit
 }
+
+// GetClusterName returns the current cluster name
+func (o *CommonOptions) GetClusterName() (string, error) {
+	kubeClient, ns, err := o.KubeClientAndDevNamespace()
+	if err != nil {
+		return "", err
+	}
+	data, err := kube.ReadInstallValues(kubeClient, ns)
+	if err != nil {
+		return "", err
+	}
+	return data[kube.ClusterName], nil
+}

pkg/cmd/opts/helm.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jenkins-x/jx/pkg/secreturl"
+	"github.com/jenkins-x/jx/pkg/secreturl/localvault"
 	"github.com/pborman/uuid"
 
 	"github.com/jenkins-x/jx/pkg/environments"
@@ -24,8 +26,8 @@ import (
 	"github.com/jenkins-x/jx/pkg/util"
 	version2 "github.com/jenkins-x/jx/pkg/version"
 	"github.com/pkg/errors"
-	survey "gopkg.in/AlecAivazis/survey.v1"
-	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/AlecAivazis/survey.v1"
+	"gopkg.in/src-d/go-git.v4"
 	gitconfig "gopkg.in/src-d/go-git.v4/config"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -403,11 +405,32 @@ func (o *CommonOptions) InstallChartWithOptionsAndTimeout(options helm.InstallCh
 			return err
 		}
 	}
+	secretURLClient, err := o.GetSecretURLClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to create a Secret RL client")
+	}
+	return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, secretURLClient)
+}
+
+// GetSecretURLClient create a new secret URL client
+func (o *CommonOptions) GetSecretURLClient() (secreturl.Client, error) {
 	vaultClient, err := o.SystemVaultClient(o.devNamespace)
 	if err != nil {
 		vaultClient = nil
 	}
-	return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, vaultClient)
+	if vaultClient != nil {
+		return vaultClient, nil
+	}
+	clusterName, err := o.GetClusterName()
+	if err != nil || clusterName == "" {
+		// we could be bootstrapping the cluster
+		clusterName = os.Getenv("JX_CLUSTER_NAME")
+		if clusterName == "" {
+			clusterName = "default-cluster"
+		}
+	}
+	dir, err := util.LocalFileSystemSecretsDir(clusterName)
+	return localvault.NewFileSystemClient(dir), nil
 }
 
 // CloneJXVersionsRepo clones the jenkins-x versions repo to a local working dir

pkg/cmd/step/create/step_create_install_values.go

@@ -23,11 +23,11 @@ import (
 
 var (
 	createInstallValuesLong = templates.LongDesc(`
-		Creates the installation values.yaml file from an init-values.yaml defaulting any missing values from the cluster itself
+		Creates any mising cluster values into the cluster/values.yaml file 
 `)
 
 	createInstallValuesExample = templates.Examples(`
-		# create the values.yaml file in the current directory
+		# populate the cluster/values.yaml file
 		jx step create install vales
 
 			`)
@@ -62,7 +62,7 @@ func NewCmdStepCreateInstallValues(commonOpts *opts.CommonOptions) *cobra.Comman
 
 	cmd := &cobra.Command{
 		Use:     "install values",
-		Short:   "Creates the installation values.yaml file from an init-values.yaml defaulting any missing values from the cluster itself",
+		Short:   "Creates any mising cluster values into the cluster/values.yaml file ",
 		Long:    createInstallValuesLong,
 		Example: createInstallValuesExample,
 		Aliases: []string{"version pullrequest"},
@@ -93,30 +93,28 @@ func (o *StepCreateInstallValuesOptions) Run() error {
 			return err
 		}
 	}
-	valuesFile := filepath.Join(o.Dir, "values.yaml")
-	initValuesFile := filepath.Join(o.Dir, "init-values.yaml")
+	clusterDir := filepath.Join(o.Dir, "cluster")
+	err = os.MkdirAll(clusterDir, util.DefaultWritePermissions)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create cluster dir: %s", clusterDir)
+	}
 
-	exists, err := util.FileExists(initValuesFile)
+	valuesFile := filepath.Join(clusterDir, helm.ValuesFileName)
+	values, err := helm.LoadValuesFile(valuesFile)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to load helm values: %s", valuesFile)
 	}
-	if exists {
-		values, err := helm.LoadValuesFile(initValuesFile)
-		if err != nil {
-			return errors.Wrapf(err, "failed to load helm values: %s", initValuesFile)
-		}
 
-		values, err = o.defaultMissingValues(values)
-		if err != nil {
-			return errors.Wrapf(err, "failed to default helm values into: %s", initValuesFile)
-		}
+	values, err = o.defaultMissingValues(values)
+	if err != nil {
+		return errors.Wrapf(err, "failed to default helm values into: %s", valuesFile)
+	}
 
-		err = helm.SaveFile(valuesFile, values)
-		if err != nil {
-			return errors.Wrapf(err, "failed to save helm values: %s", valuesFile)
-		}
-		log.Logger().Infof("wrote %s\n", util.ColorInfo(valuesFile))
+	err = helm.SaveFile(valuesFile, values)
+	if err != nil {
+		return errors.Wrapf(err, "failed to save helm values: %s", valuesFile)
 	}
+	log.Logger().Infof("wrote %s\n", util.ColorInfo(valuesFile))
 	return nil
 }
 
@@ -126,23 +124,23 @@ func (o *StepCreateInstallValuesOptions) defaultMissingValues(values map[string]
 		ns = os.Getenv("DEPLOY_NAMESPACE")
 	}
 	if ns != "" {
-		current := util.GetMapValueAsStringViaPath(values, "cluster.namespaceSubDomain")
+		current := util.GetMapValueAsStringViaPath(values, "namespaceSubDomain")
 		if current == "" {
 			subDomain := "." + ns + "."
-			util.SetMapValueViaPath(values, "cluster.namespaceSubDomain", subDomain)
+			util.SetMapValueViaPath(values, "namespaceSubDomain", subDomain)
 		}
 	}
 
-	domain := util.GetMapValueAsStringViaPath(values, "cluster.domain")
+	domain := util.GetMapValueAsStringViaPath(values, "domain")
 	if domain == "" {
 		domain, err := o.discoverIngressDomain(values)
 		if err != nil {
 			return values, errors.Wrapf(err, "failed to discover the Ingress domain")
 		}
 		if domain == "" {
-			return values, fmt.Errorf("could not detect a domain. Pleae configure one at 'cluster.domain' in the init-values.yaml")
+			return values, fmt.Errorf("could not detect a domain. Pleae configure one at 'domain' in the init-values.yaml")
 		}
-		util.SetMapValueViaPath(values, "cluster.domain", domain)
+		util.SetMapValueViaPath(values, "domain", domain)
 	}
 	return values, nil
 }

pkg/cmd/step/create/step_create_install_values_test.go

@@ -81,7 +81,7 @@ func TestCreateInstallValues(t *testing.T) {
 	err = o.Run()
 	require.NoError(t, err, "failed to run step")
 
-	fileName := filepath.Join(outputDir, "values.yaml")
+	fileName := filepath.Join(outputDir, "cluster", "values.yaml")
 
 	t.Logf("Generated values file at %s\n", fileName)
 
@@ -90,8 +90,8 @@ func TestCreateInstallValues(t *testing.T) {
 	values, err := helm.LoadValuesFile(fileName)
 	require.NoError(t, err, "failed to load file %s", fileName)
 
-	AssertMapPathValueAsString(t, values, "cluster.namespaceSubDomain", ".jx.")
-	AssertMapPathValueAsString(t, values, "cluster.domain", expectedDomain)
+	AssertMapPathValueAsString(t, values, "namespaceSubDomain", ".jx.")
+	AssertMapPathValueAsString(t, values, "domain", expectedDomain)
 }
 
 func AssertMapPathValueAsString(t *testing.T, values map[string]interface{}, path string, expected string) {

pkg/cmd/step/helm/step_helm_apply.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/jenkins-x/jx/pkg/cmd/helper"
-
 	"github.com/jenkins-x/jx/pkg/cmd/opts"
 	"github.com/jenkins-x/jx/pkg/cmd/templates"
 	"github.com/jenkins-x/jx/pkg/helm"
@@ -194,7 +193,11 @@ func (o *StepHelmApplyOptions) Run() error {
 		}()
 	}
 
-	chartValues, err := helm.GenerateValues(dir, nil, true)
+	secretURLClient, err := o.GetSecretURLClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to create a Secret RL client")
+	}
+	chartValues, err := helm.GenerateValues(dir, nil, true, secretURLClient)
 	if err != nil {
 		return errors.Wrapf(err, "generating values.yaml for tree from %s", dir)
 	}

pkg/helm/helm_helpers.go

@@ -15,22 +15,21 @@ import (
 	"strconv"
 	"strings"
 
-	survey "gopkg.in/AlecAivazis/survey.v1"
+	"gopkg.in/AlecAivazis/survey.v1"
 	"gopkg.in/AlecAivazis/survey.v1/terminal"
 
 	"github.com/pborman/uuid"
 
+	"github.com/jenkins-x/jx/pkg/kube"
+	"github.com/jenkins-x/jx/pkg/log"
+	"github.com/jenkins-x/jx/pkg/secreturl"
 	"github.com/jenkins-x/jx/pkg/table"
-	"github.com/jenkins-x/jx/pkg/vault"
-
+	"github.com/jenkins-x/jx/pkg/util"
 	"github.com/jenkins-x/jx/pkg/version"
 
-	"github.com/jenkins-x/jx/pkg/kube"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/ghodss/yaml"
-	"github.com/jenkins-x/jx/pkg/log"
-	"github.com/jenkins-x/jx/pkg/util"
 	"github.com/pkg/errors"
 	"k8s.io/helm/pkg/chartutil"
 	"k8s.io/helm/pkg/proto/hapi/chart"
@@ -48,6 +47,10 @@ const (
 	// TemplatesDirName is the default name for the templates directory
 	TemplatesDirName = "templates"
 
+	// ParametersYAMLFile contains logical parameters (values or secrets) which can be fetched from a Secret URL or
+	// inlined if not a secret which can be referenced from a 'values.yaml` file via a `{{ .Parameters.foo.bar }}` expression
+	ParametersYAMLFile = "parameters.yaml"
+
 	// InClusterHelmRepositoryURL is the default cluster local helm repo
 	InClusterHelmRepositoryURL = "http://jenkins-x-chartmuseum:8080"
 
@@ -308,8 +311,10 @@ func LoadChart(data []byte) (*chart.Metadata, error) {
 
 // LoadValues loads the values from some data
 func LoadValues(data []byte) (map[string]interface{}, error) {
-	r := make(map[string]interface{})
-
+	r := map[string]interface{}{}
+	if data == nil || len(data) == 0 {
+		return r, nil
+	}
 	return r, yaml.Unmarshal(data, &r)
 }
 
@@ -518,7 +523,7 @@ type InstallChartOptions struct {
 // respecting the installTimeout, looking up or updating Vault with the username and password for the repo.
 // If vaultClient is nil then username and passwords for repos will not be looked up in Vault.
 func InstallFromChartOptions(options InstallChartOptions, helmer Helmer, kubeClient kubernetes.Interface,
-	installTimeout string, vaultClient vault.Client) error {
+	installTimeout string, vaultClient secreturl.Client) error {
 	chart := options.Chart
 	if options.Version == "" {
 		versionsDir := options.VersionsDir
@@ -574,7 +579,7 @@ type HelmRepoCredential struct {
 
 // DecorateWithSecrets will replace any vault: URIs with the secret from vault. Safe to call with a nil client (
 // no replacement will take place).
-func DecorateWithSecrets(options *InstallChartOptions, vaultClient vault.Client) (func(), error) {
+func DecorateWithSecrets(options *InstallChartOptions, vaultClient secreturl.Client) (func(), error) {
 	cleanup := func() {
 	}
 	if vaultClient != nil {
@@ -596,9 +601,12 @@ func DecorateWithSecrets(options *InstallChartOptions, vaultClient vault.Client)
 			if err != nil {
 				return cleanup, errors.Wrapf(err, "reading file %s", valueFile)
 			}
-			newValues, err := vault.ReplaceURIs(string(bytes), vaultClient)
-			if err != nil {
-				return cleanup, errors.Wrapf(err, "replacing vault URIs")
+			newValues := string(bytes)
+			if vaultClient != nil {
+				newValues, err = vaultClient.ReplaceURIs(newValues)
+				if err != nil {
+					return cleanup, errors.Wrapf(err, "replacing vault URIs")
+				}
 			}
 			err = ioutil.WriteFile(newValuesFile.Name(), []byte(newValues), 0600)
 			if err != nil {
@@ -611,12 +619,41 @@ func DecorateWithSecrets(options *InstallChartOptions, vaultClient vault.Client)
 	return cleanup, nil
 }
 
+// LoadParameters loads the 'parameters.yaml' file if it exists in the current directory
+func LoadParameters(dir string, vaultClient secreturl.Client) (chartutil.Values, error) {
+	fileName := filepath.Join(dir, ParametersYAMLFile)
+	exists, err := util.FileExists(fileName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "checking %s exists", fileName)
+	}
+	m := map[string]interface{}{}
+	if exists {
+		data, err := ioutil.ReadFile(fileName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "reading %s", fileName)
+		}
+		if vaultClient != nil {
+			text, err := vaultClient.ReplaceURIs(string(data))
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to convert secret URLs in parameters file %s", fileName)
+			}
+			data = []byte(text)
+		}
+
+		m, err = LoadValues(data)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unmarshaling %s", fileName)
+		}
+	}
+	return chartutil.Values(m), err
+}
+
 // AddHelmRepoIfMissing will add the helm repo if there is no helm repo with that url present.
 // It will generate the repoName from the url (using the host name) if the repoName is empty.
 // The repo name may have a suffix added in order to prevent name collisions, and is returned for this reason.
 // The username and password will be stored in vault for the URL (if vault is enabled).
 func AddHelmRepoIfMissing(helmURL, repoName, username, password string, helmer Helmer,
-	vaultClient vault.Client, in terminal.FileReader,
+	vaultClient secreturl.Client, in terminal.FileReader,
 	out terminal.FileWriter, outErr io.Writer) (string, error) {
 	missing, existingName, err := helmer.IsRepoMissing(helmURL)
 	if err != nil {
@@ -663,7 +700,7 @@ func AddHelmRepoIfMissing(helmURL, repoName, username, password string, helmer H
 }
 
 // DecorateWithCredentials will, if vault is installed, store or replace the username or password
-func DecorateWithCredentials(repo string, username string, password string, vaultClient vault.Client, in terminal.FileReader,
+func DecorateWithCredentials(repo string, username string, password string, vaultClient secreturl.Client, in terminal.FileReader,
 	out terminal.FileWriter, outErr io.Writer) (string,
 	string, error) {
 	if repo != "" && vaultClient != nil {