diff --git a/go.sum b/go.sum index 6a2fa221a5..e9c63dc695 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,7 @@ github.com/Masterminds/semver v1.3.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v0.0.0-20180403013413-6b2a58267f6a/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.4.6 h1:Tu8dlnF1wvUKKqr011GFneCoyIn7D+Q2uq6AKmQnGrA= github.com/Microsoft/go-winio v0.4.6/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -85,6 +86,7 @@ github.com/antham/chyle v1.4.0 h1:RBCXpnmj3Wcl43bKbcuRU3LXkarhYwSigWAPUyWEjvY= github.com/antham/chyle v1.4.0/go.mod h1:D94Z4aE/ECudyNoTHwkhqu77mjGPZtfPG8dNoeIG9CU= github.com/antham/envh v1.2.0/go.mod h1:ocIRPHuwwjyBVBtuUJOJc2TYzGg+d23xSAZexl4y9hQ= github.com/antham/strumt v0.0.0-20171215230529-6776189777d3/go.mod h1:sE7EYIUE0nQzPiv5zQAmw2aVkei0j2xmb4gTIIqSFSI= +github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appscode/jsonpatch v0.0.0-20180911074601-5af499cf01c8/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= @@ -391,6 +393,7 @@ github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c h1:kp3AxgXgDOmIJFR7b github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13 h1:0XE8qtre7NNhsKWo+PuYJNmoR3szkSzpDtZLEW+5HE0= github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= diff --git a/pkg/apps/values.go b/pkg/apps/values.go index 69c20181c6..f0cca26b8f 100644 --- a/pkg/apps/values.go +++ b/pkg/apps/values.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" + "github.com/jenkins-x/jx/pkg/secreturl" "gopkg.in/AlecAivazis/survey.v1/terminal" "github.com/ghodss/yaml" @@ -187,7 +188,7 @@ func GenerateQuestions(schema []byte, batchMode bool, askExisting bool, basePath secrets = append(secrets, secret) if passthrough { if useVault { - return vault.ToURI(secret.Path, secret.Key), nil + return secreturl.ToURI(secret.Path, secret.Key), nil } return value, nil } diff --git a/pkg/cmd/opts/helm.go b/pkg/cmd/opts/helm.go index db5647c027..5eae24656c 100644 --- a/pkg/cmd/opts/helm.go +++ b/pkg/cmd/opts/helm.go @@ -10,6 +10,9 @@ import ( "strings" "time" + "github.com/jenkins-x/jx/pkg/kube/cluster" + "github.com/jenkins-x/jx/pkg/secreturl" + "github.com/jenkins-x/jx/pkg/secreturl/localvault" "github.com/pborman/uuid" "github.com/jenkins-x/jx/pkg/environments" @@ -24,8 +27,8 @@ import ( "github.com/jenkins-x/jx/pkg/util" version2 "github.com/jenkins-x/jx/pkg/version" "github.com/pkg/errors" - survey "gopkg.in/AlecAivazis/survey.v1" - git "gopkg.in/src-d/go-git.v4" + "gopkg.in/AlecAivazis/survey.v1" + "gopkg.in/src-d/go-git.v4" gitconfig "gopkg.in/src-d/go-git.v4/config" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -403,11 +406,32 @@ func (o *CommonOptions) InstallChartWithOptionsAndTimeout(options helm.InstallCh return err } } + secretURLClient, err := o.GetSecretURLClient() + if err != nil { + return errors.Wrap(err, "failed to create a Secret RL client") + } + return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, secretURLClient) +} + +// GetSecretURLClient create a new secret URL client +func (o *CommonOptions) GetSecretURLClient() (secreturl.Client, error) { vaultClient, err := o.SystemVaultClient(o.devNamespace) if err != nil { vaultClient = nil } - return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, vaultClient) + if vaultClient != nil { + return vaultClient, nil + } + clusterName, err := cluster.Name(o.Kube()) + if err != nil || clusterName == "" { + // we could be bootstrapping the cluster + clusterName = os.Getenv("JX_CLUSTER_NAME") + if clusterName == "" { + clusterName = "default-cluster" + } + } + dir, err := util.LocalFileSystemSecretsDir(clusterName) + return localvault.NewFileSystemClient(dir), nil } // CloneJXVersionsRepo clones the jenkins-x versions repo to a local working dir diff --git a/pkg/cmd/step/create/step_create_install_values.go b/pkg/cmd/step/create/step_create_install_values.go index 05c9346818..db4c33cd6c 100644 --- a/pkg/cmd/step/create/step_create_install_values.go +++ b/pkg/cmd/step/create/step_create_install_values.go @@ -23,11 +23,11 @@ import ( var ( createInstallValuesLong = templates.LongDesc(` - Creates the installation values.yaml file from an init-values.yaml defaulting any missing values from the cluster itself + Creates any mising cluster values into the cluster/values.yaml file `) createInstallValuesExample = templates.Examples(` - # create the values.yaml file in the current directory + # populate the cluster/values.yaml file jx step create install vales `) @@ -62,7 +62,7 @@ func NewCmdStepCreateInstallValues(commonOpts *opts.CommonOptions) *cobra.Comman cmd := &cobra.Command{ Use: "install values", - Short: "Creates the installation values.yaml file from an init-values.yaml defaulting any missing values from the cluster itself", + Short: "Creates any mising cluster values into the cluster/values.yaml file ", Long: createInstallValuesLong, Example: createInstallValuesExample, Aliases: []string{"version pullrequest"}, @@ -93,30 +93,28 @@ func (o *StepCreateInstallValuesOptions) Run() error { return err } } - valuesFile := filepath.Join(o.Dir, "values.yaml") - initValuesFile := filepath.Join(o.Dir, "init-values.yaml") + clusterDir := filepath.Join(o.Dir, "cluster") + err = os.MkdirAll(clusterDir, util.DefaultWritePermissions) + if err != nil { + return errors.Wrapf(err, "failed to create cluster dir: %s", clusterDir) + } - exists, err := util.FileExists(initValuesFile) + valuesFile := filepath.Join(clusterDir, helm.ValuesFileName) + values, err := helm.LoadValuesFile(valuesFile) if err != nil { - return err + return errors.Wrapf(err, "failed to load helm values: %s", valuesFile) } - if exists { - values, err := helm.LoadValuesFile(initValuesFile) - if err != nil { - return errors.Wrapf(err, "failed to load helm values: %s", initValuesFile) - } - values, err = o.defaultMissingValues(values) - if err != nil { - return errors.Wrapf(err, "failed to default helm values into: %s", initValuesFile) - } + values, err = o.defaultMissingValues(values) + if err != nil { + return errors.Wrapf(err, "failed to default helm values into: %s", valuesFile) + } - err = helm.SaveFile(valuesFile, values) - if err != nil { - return errors.Wrapf(err, "failed to save helm values: %s", valuesFile) - } - log.Logger().Infof("wrote %s\n", util.ColorInfo(valuesFile)) + err = helm.SaveFile(valuesFile, values) + if err != nil { + return errors.Wrapf(err, "failed to save helm values: %s", valuesFile) } + log.Logger().Infof("wrote %s\n", util.ColorInfo(valuesFile)) return nil } @@ -126,23 +124,23 @@ func (o *StepCreateInstallValuesOptions) defaultMissingValues(values map[string] ns = os.Getenv("DEPLOY_NAMESPACE") } if ns != "" { - current := util.GetMapValueAsStringViaPath(values, "cluster.namespaceSubDomain") + current := util.GetMapValueAsStringViaPath(values, "namespaceSubDomain") if current == "" { subDomain := "." + ns + "." - util.SetMapValueViaPath(values, "cluster.namespaceSubDomain", subDomain) + util.SetMapValueViaPath(values, "namespaceSubDomain", subDomain) } } - domain := util.GetMapValueAsStringViaPath(values, "cluster.domain") + domain := util.GetMapValueAsStringViaPath(values, "domain") if domain == "" { domain, err := o.discoverIngressDomain(values) if err != nil { return values, errors.Wrapf(err, "failed to discover the Ingress domain") } if domain == "" { - return values, fmt.Errorf("could not detect a domain. Pleae configure one at 'cluster.domain' in the init-values.yaml") + return values, fmt.Errorf("could not detect a domain. Pleae configure one at 'domain' in the init-values.yaml") } - util.SetMapValueViaPath(values, "cluster.domain", domain) + util.SetMapValueViaPath(values, "domain", domain) } return values, nil } diff --git a/pkg/cmd/step/create/step_create_install_values_test.go b/pkg/cmd/step/create/step_create_install_values_test.go index 37ea152c9d..b26d691426 100644 --- a/pkg/cmd/step/create/step_create_install_values_test.go +++ b/pkg/cmd/step/create/step_create_install_values_test.go @@ -81,7 +81,7 @@ func TestCreateInstallValues(t *testing.T) { err = o.Run() require.NoError(t, err, "failed to run step") - fileName := filepath.Join(outputDir, "values.yaml") + fileName := filepath.Join(outputDir, "cluster", "values.yaml") t.Logf("Generated values file at %s\n", fileName) @@ -90,8 +90,8 @@ func TestCreateInstallValues(t *testing.T) { values, err := helm.LoadValuesFile(fileName) require.NoError(t, err, "failed to load file %s", fileName) - AssertMapPathValueAsString(t, values, "cluster.namespaceSubDomain", ".jx.") - AssertMapPathValueAsString(t, values, "cluster.domain", expectedDomain) + AssertMapPathValueAsString(t, values, "namespaceSubDomain", ".jx.") + AssertMapPathValueAsString(t, values, "domain", expectedDomain) } func AssertMapPathValueAsString(t *testing.T, values map[string]interface{}, path string, expected string) { diff --git a/pkg/cmd/step/create/test_data/step_create_install_values/init-values.yaml b/pkg/cmd/step/create/test_data/step_create_install_values/values.yaml similarity index 100% rename from pkg/cmd/step/create/test_data/step_create_install_values/init-values.yaml rename to pkg/cmd/step/create/test_data/step_create_install_values/values.yaml diff --git a/pkg/cmd/step/helm/step_helm_apply.go b/pkg/cmd/step/helm/step_helm_apply.go index 764a77c349..11e9632086 100644 --- a/pkg/cmd/step/helm/step_helm_apply.go +++ b/pkg/cmd/step/helm/step_helm_apply.go @@ -10,7 +10,6 @@ import ( "github.com/google/uuid" "github.com/jenkins-x/jx/pkg/cmd/helper" - "github.com/jenkins-x/jx/pkg/cmd/opts" "github.com/jenkins-x/jx/pkg/cmd/templates" "github.com/jenkins-x/jx/pkg/helm" @@ -194,7 +193,11 @@ func (o *StepHelmApplyOptions) Run() error { }() } - chartValues, err := helm.GenerateValues(dir, nil, true) + secretURLClient, err := o.GetSecretURLClient() + if err != nil { + return errors.Wrap(err, "failed to create a Secret RL client") + } + chartValues, err := helm.GenerateValues(dir, nil, true, secretURLClient) if err != nil { return errors.Wrapf(err, "generating values.yaml for tree from %s", dir) } diff --git a/pkg/helm/helm_helpers.go b/pkg/helm/helm_helpers.go index a6d76e77ba..473b64505a 100644 --- a/pkg/helm/helm_helpers.go +++ b/pkg/helm/helm_helpers.go @@ -15,22 +15,21 @@ import ( "strconv" "strings" - survey "gopkg.in/AlecAivazis/survey.v1" + "gopkg.in/AlecAivazis/survey.v1" "gopkg.in/AlecAivazis/survey.v1/terminal" "github.com/pborman/uuid" + "github.com/jenkins-x/jx/pkg/kube" + "github.com/jenkins-x/jx/pkg/log" + "github.com/jenkins-x/jx/pkg/secreturl" "github.com/jenkins-x/jx/pkg/table" - "github.com/jenkins-x/jx/pkg/vault" - + "github.com/jenkins-x/jx/pkg/util" "github.com/jenkins-x/jx/pkg/version" - "github.com/jenkins-x/jx/pkg/kube" "k8s.io/client-go/kubernetes" "github.com/ghodss/yaml" - "github.com/jenkins-x/jx/pkg/log" - "github.com/jenkins-x/jx/pkg/util" "github.com/pkg/errors" "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" @@ -48,6 +47,10 @@ const ( // TemplatesDirName is the default name for the templates directory TemplatesDirName = "templates" + // ParametersYAMLFile contains logical parameters (values or secrets) which can be fetched from a Secret URL or + // inlined if not a secret which can be referenced from a 'values.yaml` file via a `{{ .Parameters.foo.bar }}` expression + ParametersYAMLFile = "parameters.yaml" + // InClusterHelmRepositoryURL is the default cluster local helm repo InClusterHelmRepositoryURL = "http://jenkins-x-chartmuseum:8080" @@ -308,8 +311,10 @@ func LoadChart(data []byte) (*chart.Metadata, error) { // LoadValues loads the values from some data func LoadValues(data []byte) (map[string]interface{}, error) { - r := make(map[string]interface{}) - + r := map[string]interface{}{} + if data == nil || len(data) == 0 { + return r, nil + } return r, yaml.Unmarshal(data, &r) } @@ -516,9 +521,9 @@ type InstallChartOptions struct { // InstallFromChartOptions uses the helmer and kubeClient interfaces to install the chart from the options, // respecting the installTimeout, looking up or updating Vault with the username and password for the repo. -// If vaultClient is nil then username and passwords for repos will not be looked up in Vault. +// If secretURLClient is nil then username and passwords for repos will not be looked up in Vault. func InstallFromChartOptions(options InstallChartOptions, helmer Helmer, kubeClient kubernetes.Interface, - installTimeout string, vaultClient vault.Client) error { + installTimeout string, secretURLClient secreturl.Client) error { chart := options.Chart if options.Version == "" { versionsDir := options.VersionsDir @@ -539,7 +544,7 @@ func InstallFromChartOptions(options InstallChartOptions, helmer Helmer, kubeCli } log.Logger().Debugf("Helm repository update done.") } - cleanup, err := DecorateWithSecrets(&options, vaultClient) + cleanup, err := DecorateWithSecrets(&options, secretURLClient) defer cleanup() if err != nil { return errors.WithStack(err) @@ -574,10 +579,10 @@ type HelmRepoCredential struct { // DecorateWithSecrets will replace any vault: URIs with the secret from vault. Safe to call with a nil client ( // no replacement will take place). -func DecorateWithSecrets(options *InstallChartOptions, vaultClient vault.Client) (func(), error) { +func DecorateWithSecrets(options *InstallChartOptions, secretURLClient secreturl.Client) (func(), error) { cleanup := func() { } - if vaultClient != nil { + if secretURLClient != nil { newValuesFiles := make([]string, 0) cleanup = func() { for _, f := range newValuesFiles { @@ -596,9 +601,12 @@ func DecorateWithSecrets(options *InstallChartOptions, vaultClient vault.Client) if err != nil { return cleanup, errors.Wrapf(err, "reading file %s", valueFile) } - newValues, err := vault.ReplaceURIs(string(bytes), vaultClient) - if err != nil { - return cleanup, errors.Wrapf(err, "replacing vault URIs") + newValues := string(bytes) + if secretURLClient != nil { + newValues, err = secretURLClient.ReplaceURIs(newValues) + if err != nil { + return cleanup, errors.Wrapf(err, "replacing vault URIs") + } } err = ioutil.WriteFile(newValuesFile.Name(), []byte(newValues), 0600) if err != nil { @@ -611,12 +619,41 @@ func DecorateWithSecrets(options *InstallChartOptions, vaultClient vault.Client) return cleanup, nil } +// LoadParameters loads the 'parameters.yaml' file if it exists in the current directory +func LoadParameters(dir string, secretURLClient secreturl.Client) (chartutil.Values, error) { + fileName := filepath.Join(dir, ParametersYAMLFile) + exists, err := util.FileExists(fileName) + if err != nil { + return nil, errors.Wrapf(err, "checking %s exists", fileName) + } + m := map[string]interface{}{} + if exists { + data, err := ioutil.ReadFile(fileName) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", fileName) + } + if secretURLClient != nil { + text, err := secretURLClient.ReplaceURIs(string(data)) + if err != nil { + return nil, errors.Wrapf(err, "failed to convert secret URLs in parameters file %s", fileName) + } + data = []byte(text) + } + + m, err = LoadValues(data) + if err != nil { + return nil, errors.Wrapf(err, "unmarshaling %s", fileName) + } + } + return chartutil.Values(m), err +} + // AddHelmRepoIfMissing will add the helm repo if there is no helm repo with that url present. // It will generate the repoName from the url (using the host name) if the repoName is empty. // The repo name may have a suffix added in order to prevent name collisions, and is returned for this reason. // The username and password will be stored in vault for the URL (if vault is enabled). func AddHelmRepoIfMissing(helmURL, repoName, username, password string, helmer Helmer, - vaultClient vault.Client, in terminal.FileReader, + secretURLClient secreturl.Client, in terminal.FileReader, out terminal.FileWriter, outErr io.Writer) (string, error) { missing, existingName, err := helmer.IsRepoMissing(helmURL) if err != nil { @@ -647,7 +684,7 @@ func AddHelmRepoIfMissing(helmURL, repoName, username, password string, helmer H } } log.Logger().Infof("Adding missing Helm repo: %s %s", util.ColorInfo(repoName), util.ColorInfo(helmURL)) - username, password, err = DecorateWithCredentials(helmURL, username, password, vaultClient, in, out, outErr) + username, password, err = DecorateWithCredentials(helmURL, username, password, secretURLClient, in, out, outErr) if err != nil { return "", errors.WithStack(err) } @@ -663,12 +700,12 @@ func AddHelmRepoIfMissing(helmURL, repoName, username, password string, helmer H } // DecorateWithCredentials will, if vault is installed, store or replace the username or password -func DecorateWithCredentials(repo string, username string, password string, vaultClient vault.Client, in terminal.FileReader, +func DecorateWithCredentials(repo string, username string, password string, secretURLClient secreturl.Client, in terminal.FileReader, out terminal.FileWriter, outErr io.Writer) (string, string, error) { - if repo != "" && vaultClient != nil { + if repo != "" && secretURLClient != nil { creds := HelmRepoCredentials{} - if err := vaultClient.ReadObject(RepoVaultPath, &creds); err != nil { + if err := secretURLClient.ReadObject(RepoVaultPath, &creds); err != nil { return "", "", errors.Wrapf(err, "reading repo credentials from vault %s", RepoVaultPath) } var existingCred, cred HelmRepoCredential @@ -692,7 +729,7 @@ func DecorateWithCredentials(repo string, username string, password string, vaul if cred.Password != existingCred.Password || cred.Username != existingCred.Username { log.Logger().Infof("Storing credentials for %s in vault %s", repo, RepoVaultPath) creds[repo] = cred - _, err := vaultClient.WriteObject(RepoVaultPath, creds) + _, err := secretURLClient.WriteObject(RepoVaultPath, creds) if err != nil { return "", "", errors.Wrapf(err, "updating repo credentials in vault %s", RepoVaultPath) } diff --git a/pkg/helm/helm_helpers_test.go b/pkg/helm/helm_helpers_test.go index 9ea31bcd01..9c389d740c 100644 --- a/pkg/helm/helm_helpers_test.go +++ b/pkg/helm/helm_helpers_test.go @@ -3,15 +3,18 @@ package helm_test import ( "fmt" "io/ioutil" + "path" "strings" "testing" + "github.com/jenkins-x/jx/pkg/secreturl/localvault" "github.com/pborman/uuid" "github.com/petergtz/pegomock" + "github.com/stretchr/testify/require" "github.com/jenkins-x/jx/pkg/helm" + secreturl_test "github.com/jenkins-x/jx/pkg/secreturl/mocks" "github.com/jenkins-x/jx/pkg/util" - vault_test "github.com/jenkins-x/jx/pkg/vault/mocks" "github.com/magiconair/properties/assert" assert2 "github.com/stretchr/testify/assert" ) @@ -96,7 +99,7 @@ func TestSetValuesToMap(t *testing.T) { func TestStoreCredentials(t *testing.T) { pegomock.RegisterMockTestingT(t) - vaultClient := vault_test.NewMockClient() + vaultClient := secreturl_test.NewMockClient() repository := "http://charts.acme.com" username := uuid.New() password := uuid.New() @@ -112,7 +115,7 @@ func TestStoreCredentials(t *testing.T) { func TestRetrieveCredentials(t *testing.T) { pegomock.RegisterMockTestingT(t) - vaultClient := vault_test.NewMockClient() + vaultClient := secreturl_test.NewMockClient() repository := "http://charts.acme.com" username := uuid.New() password := uuid.New() @@ -139,7 +142,7 @@ func TestRetrieveCredentials(t *testing.T) { func TestOverrideCredentials(t *testing.T) { pegomock.RegisterMockTestingT(t) - vaultClient := vault_test.NewMockClient() + vaultClient := secreturl_test.NewMockClient() repository := "http://charts.acme.com" username := uuid.New() password := uuid.New() @@ -173,7 +176,7 @@ func TestOverrideCredentials(t *testing.T) { func TestReplaceVaultURI(t *testing.T) { pegomock.RegisterMockTestingT(t) - vaultClient := vault_test.NewMockClient() + vaultClient := secreturl_test.NewMockClient() path := "/baz/qux" key := "cheese" secret := uuid.New() @@ -196,6 +199,49 @@ func TestReplaceVaultURI(t *testing.T) { pegomock.When(vaultClient.Read(pegomock.EqString(path))).ThenReturn(map[string]interface{}{ key: secret, }, nil) + pegomock.When(vaultClient.ReplaceURIs(pegomock.EqString(valuesyaml))).ThenReturn(fmt.Sprintf(`foo: + bar: %s +`, secret), nil) + cleanup, err := helm.DecorateWithSecrets(&options, vaultClient) + defer cleanup() + assert2.Len(t, options.ValueFiles, 1) + newValuesYaml, err := ioutil.ReadFile(options.ValueFiles[0]) + assert2.NoError(t, err) + assert2.Equal(t, fmt.Sprintf(`foo: + bar: %s +`, secret), string(newValuesYaml)) +} + +func TestReplaceVaultURIWithLocalFile(t *testing.T) { + vaultClient := localvault.NewFileSystemClient(path.Join("test_data", "local_vault_files")) + path := "/baz/qux" + key := "cheese" + secret := "Edam" + valuesyaml := fmt.Sprintf(`foo: + bar: local:%s:%s +`, path, key) + valuesFile, err := ioutil.TempFile("", "values.yaml") + defer func() { + err := util.DeleteFile(valuesFile.Name()) + assert2.NoError(t, err) + }() + assert2.NoError(t, err) + err = ioutil.WriteFile(valuesFile.Name(), []byte(valuesyaml), 0600) + assert2.NoError(t, err) + options := helm.InstallChartOptions{ + ValueFiles: []string{ + valuesFile.Name(), + }, + } + + actual, err := vaultClient.Read(path) + expected := map[string]interface{}{ + key: secret, + } + + require.NoError(t, err, "reading vault client on path %s", path) + assert2.Equal(t, expected, actual, "vault read at path %s", path) + cleanup, err := helm.DecorateWithSecrets(&options, vaultClient) defer cleanup() assert2.Len(t, options.ValueFiles, 1) diff --git a/pkg/helm/test_data/local_vault_files/baz/qux.yaml b/pkg/helm/test_data/local_vault_files/baz/qux.yaml new file mode 100644 index 0000000000..b8dde1ea26 --- /dev/null +++ b/pkg/helm/test_data/local_vault_files/baz/qux.yaml @@ -0,0 +1 @@ +cheese: Edam \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/adminUser.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/adminUser.yaml new file mode 100644 index 0000000000..c7e58179a5 --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/adminUser.yaml @@ -0,0 +1 @@ +password-passthrough: myDockerRegistryPassword \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/dockerRegistry.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/dockerRegistry.yaml new file mode 100644 index 0000000000..97123e3709 --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/dockerRegistry.yaml @@ -0,0 +1 @@ +password-passthrough: myAdminPassword \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/pipelineUser.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/pipelineUser.yaml new file mode 100644 index 0000000000..b6b3b212d9 --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/local_vault_files/my-cheese-cluster/pipelineUser.yaml @@ -0,0 +1 @@ +token-passthrough: myPipelineUserToken \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/parameters.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/parameters.yaml new file mode 100644 index 0000000000..d478c337ee --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/parameters.yaml @@ -0,0 +1,17 @@ +adminUser: + password: local:/my-cheese-cluster/adminUser:password-passthrough + username: admin +docker: + password: local:/my-cheese-cluster/dockerRegistry:password-passthrough + url: https://index.docker.io/v1/ + username: james +enableDocker: true +enableGpg: false +gitProvider: github +pipelineUser: + github: + host: github.com + password: local:/my-cheese-cluster/pipelineUser:token-passthrough + username: james +prow: + hmacToken: abc \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/prow/values.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/prow/values.yaml new file mode 100644 index 0000000000..048214ed8e --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/prow/values.yaml @@ -0,0 +1 @@ +hmacToken: {{ .Parameters.prow.hmacToken }} \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/tekton/values.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/tekton/values.yaml new file mode 100644 index 0000000000..33f057da8c --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/tekton/values.yaml @@ -0,0 +1,4 @@ +auth: + git: + username: {{ .Parameters.pipelineUser.github.username }} + password: {{ .Parameters.pipelineUser.github.password }} \ No newline at end of file diff --git a/pkg/helm/test_data/tree_of_values_yaml_templates/values.yaml b/pkg/helm/test_data/tree_of_values_yaml_templates/values.yaml new file mode 100644 index 0000000000..aed250130d --- /dev/null +++ b/pkg/helm/test_data/tree_of_values_yaml_templates/values.yaml @@ -0,0 +1,3 @@ +JenkinsXGitHub: + username: "{{ .Parameters.pipelineUser.github.username }}" + password: "{{ .Parameters.pipelineUser.github.password }}" diff --git a/pkg/helm/values_tree.go b/pkg/helm/values_tree.go index a755a7fd64..d379861fef 100644 --- a/pkg/helm/values_tree.go +++ b/pkg/helm/values_tree.go @@ -1,13 +1,19 @@ package helm import ( + "bytes" "fmt" "io/ioutil" "os" "path/filepath" "strings" + "text/template" + "github.com/jenkins-x/jx/pkg/secreturl" "github.com/jenkins-x/jx/pkg/util" + "github.com/pkg/errors" + "k8s.io/helm/pkg/chartutil" + "k8s.io/helm/pkg/engine" "github.com/ghodss/yaml" @@ -26,7 +32,7 @@ var DefaultValuesTreeIgnores = []string{ // Any keys used that match files with the same name in the directory ( // and have empty values) will be inlined as block scalars. // Standard UNIX glob patterns can be passed to IgnoreFile directories. -func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error) { +func GenerateValues(dir string, ignores []string, verbose bool, secretURLClient secreturl.Client) ([]byte, error) { info, err := os.Stat(dir) if err != nil { return nil, err @@ -35,6 +41,15 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error) } else if !info.IsDir() { return nil, fmt.Errorf("%s is not a directory", dir) } + + // load the parameter values if there are any + params, err := LoadParameters(dir, secretURLClient) + if err != nil { + return nil, err + } + funcMap := engine.FuncMap() + funcMap["hashPassword"] = util.HashPassword + if ignores == nil { ignores = DefaultValuesTreeIgnores } @@ -54,7 +69,7 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error) if rDir != "" { // If it's values.yaml, then read and parse it if file == "values.yaml" { - b, err := ioutil.ReadFile(path) + b, err := ReadValuesYamlFileTemplateOutput(path, params, funcMap) if err != nil { return err } @@ -85,7 +100,18 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error) } // Load the root values.yaml rootValuesFileName := filepath.Join(dir, ValuesFileName) - rootValues, err := LoadValuesFile(rootValuesFileName) + rootData := []byte{} + exists, err := util.FileExists(rootValuesFileName) + if err != nil { + return nil, errors.Wrapf(err, "failed to find %s", rootValuesFileName) + } + if exists { + rootData, err = ReadValuesYamlFileTemplateOutput(rootValuesFileName, params, funcMap) + if err != nil { + return nil, errors.Wrapf(err, "failed to render template of file %s", rootValuesFileName) + } + } + rootValues, err := LoadValues(rootData) if err != nil { return nil, err } @@ -141,6 +167,25 @@ func GenerateValues(dir string, ignores []string, verbose bool) ([]byte, error) return yaml.Marshal(rootValues) } +// ReadValuesYamlFileTemplateOutput evaluates the given values.yaml file as a go template and returns the output data +func ReadValuesYamlFileTemplateOutput(templateFile string, params chartutil.Values, funcMap template.FuncMap) ([]byte, error) { + tmpl, err := template.New(ValuesFileName).Option("missingkey=error").Funcs(funcMap).ParseFiles(templateFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse Secrets template: %s", templateFile) + } + + templateData := map[string]interface{}{ + "Parameters": chartutil.Values(params), + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, templateData) + if err != nil { + return nil, errors.Wrapf(err, "failed to execute Secrets template: %s", templateFile) + } + data := buf.Bytes() + return data, nil +} + // HandleExternalFileRefs recursively scans the element map structure, // looking for nested maps. If it finds keys that match any key-value pair in possibles it will call the handler. // The jsonPath is used for referencing the path in the map structure when reporting errors. diff --git a/pkg/helm/values_tree_template_test.go b/pkg/helm/values_tree_template_test.go new file mode 100644 index 0000000000..d87244caac --- /dev/null +++ b/pkg/helm/values_tree_template_test.go @@ -0,0 +1,35 @@ +package helm_test + +import ( + "path" + "testing" + + "github.com/jenkins-x/jx/pkg/helm" + "github.com/jenkins-x/jx/pkg/secreturl/localvault" + "github.com/stretchr/testify/assert" +) + +var expectedTemplatedValuesTree = `JenkinsXGitHub: + password: myPipelineUserToken + username: james +prow: + hmacToken: abc +tekton: + auth: + git: + password: myPipelineUserToken + username: james +` + +func TestValuesTreeTemplates(t *testing.T) { + t.Parallel() + + testData := path.Join("test_data", "tree_of_values_yaml_templates") + + localVaultDir := path.Join(testData, "local_vault_files") + secretURLClient := localvault.NewFileSystemClient(localVaultDir) + + result, err := helm.GenerateValues(testData, nil, true, secretURLClient) + assert.NoError(t, err) + assert.Equal(t, expectedTemplatedValuesTree, string(result)) +} diff --git a/pkg/helm/values_tree_test.go b/pkg/helm/values_tree_test.go index fbeacb9de5..54435a9deb 100644 --- a/pkg/helm/values_tree_test.go +++ b/pkg/helm/values_tree_test.go @@ -29,7 +29,7 @@ meat: assert.NoError(t, err) }() assert.NoError(t, err) - result, err := helm.GenerateValues(dir, nil, true) + result, err := helm.GenerateValues(dir, nil, true, nil) assert.NoError(t, err) assert.Equal(t, expectedOutput, string(result)) } @@ -55,7 +55,7 @@ people: pete assert.NoError(t, err) }() assert.NoError(t, err) - result, err := helm.GenerateValues(dir, nil, true) + result, err := helm.GenerateValues(dir, nil, true, nil) assert.NoError(t, err) assert.Equal(t, expectedOutput, string(result)) } @@ -84,7 +84,7 @@ func TestValuesTreeWithFileRefs(t *testing.T) { assert.NoError(t, err) }() assert.NoError(t, err) - result, err := helm.GenerateValues(dir, nil, true) + result, err := helm.GenerateValues(dir, nil, true, nil) assert.NoError(t, err) assert.Equal(t, expectedOutput, string(result)) } diff --git a/pkg/secreturl/client.go b/pkg/secreturl/client.go new file mode 100644 index 0000000000..948012f073 --- /dev/null +++ b/pkg/secreturl/client.go @@ -0,0 +1,20 @@ +package secreturl + +// Client is a simple interface for acessing vault-like secret storage URLs such as `vault.Client` or a file system we can use to +// access secret files and values in helm. +//go:generate pegomock generate github.com/jenkins-x/jx/pkg/secreturl Client -o mocks/secreturl_client.go +type Client interface { + // Read reads a named secret from the vault + Read(secretName string) (map[string]interface{}, error) + + // ReadObject reads a generic named object from vault. + // The secret _must_ be serializable to JSON. + ReadObject(secretName string, secret interface{}) error + + // WriteObject writes a generic named object to the vault. + // The secret _must_ be serializable to JSON. + WriteObject(secretName string, secret interface{}) (map[string]interface{}, error) + + // ReplaceURIs will replace any vault: URIs in a string (or whatever URL scheme the secret URL client supports + ReplaceURIs(text string) (string, error) +} diff --git a/pkg/secreturl/helpers.go b/pkg/secreturl/helpers.go new file mode 100644 index 0000000000..740ce24488 --- /dev/null +++ b/pkg/secreturl/helpers.go @@ -0,0 +1,55 @@ +package secreturl + +import ( + "fmt" + "regexp" + "strings" + + "github.com/jenkins-x/jx/pkg/util" + "github.com/pkg/errors" +) + +// ReplaceURIs will replace any URIs with the given regular expression and scheme using the secret URL client +func ReplaceURIs(s string, client Client, r *regexp.Regexp, schemePrefix string) (string, error) { + if !strings.HasSuffix(schemePrefix, ":") { + return s, fmt.Errorf("the scheme prefix should end with ':' but was %s", schemePrefix) + } + var err error + answer := r.ReplaceAllStringFunc(s, func(found string) string { + // Stop once we have an error + if err == nil { + pathAndKey := strings.Trim(strings.TrimPrefix(found, schemePrefix), "\"") + parts := strings.Split(pathAndKey, ":") + if len(parts) != 2 { + err = errors.Errorf("cannot parse %s as path:key", pathAndKey) + return "" + } + secret, err1 := client.Read(parts[0]) + if err1 != nil { + err = errors.Wrapf(err1, "reading %s from vault", parts[0]) + return "" + } + v, ok := secret[parts[1]] + if !ok { + err = errors.Errorf("unable to find %s in secret at %s", parts[1], parts[0]) + return "" + } + result, err1 := util.AsString(v) + if err1 != nil { + err = errors.Wrapf(err1, "converting %v to string", v) + return "" + } + return result + } + return found + }) + if err != nil { + return "", errors.Wrapf(err, "replacing vault paths in %s", s) + } + return answer, nil +} + +// ToURI constructs a vault: URI for the given path and key +func ToURI(path string, key string) string { + return fmt.Sprintf("vault:%s:%s", path, key) +} diff --git a/pkg/secreturl/localvault/client.go b/pkg/secreturl/localvault/client.go new file mode 100644 index 0000000000..7d8b2a37cc --- /dev/null +++ b/pkg/secreturl/localvault/client.go @@ -0,0 +1,79 @@ +package localvault + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/jenkins-x/jx/pkg/helm" + "github.com/jenkins-x/jx/pkg/secreturl" + "github.com/jenkins-x/jx/pkg/util" + "github.com/pkg/errors" +) + +var localURIRegex = regexp.MustCompile(`local:[-_\w\/:]*`) + +// FileSystemClient a local file system based client loading/saving content from the given URL +type FileSystemClient struct { + Dir string +} + +// NewFileSystemClient create a new local file system based client loading content from the given URL +func NewFileSystemClient(dir string) secreturl.Client { + return &FileSystemClient{ + Dir: dir, + } +} + +// Read reads a named secret from the vault +func (c *FileSystemClient) Read(secretName string) (map[string]interface{}, error) { + name := c.fileName(secretName) + exists, err := util.FileExists(name) + if err != nil { + return nil, errors.Wrapf(err, "failed to check if file exists %s", name) + } + if !exists { + return nil, fmt.Errorf("local vault file does not exist: %s", name) + } + return helm.LoadValuesFile(name) +} + +// ReadObject reads a generic named object from vault. +// The secret _must_ be serializable to JSON. +func (c *FileSystemClient) ReadObject(secretName string, secret interface{}) error { + m, err := c.Read(secretName) + if err != nil { + return errors.Wrapf(err, "reading the secret %q from vault", secretName) + } + err = util.ToStructFromMapStringInterface(m, &secret) + if err != nil { + return errors.Wrapf(err, "deserializing the secret %q from vault", secretName) + } + return nil +} + +// WriteObject writes a generic named object to the vault. +// The secret _must_ be serializable to JSON. +func (c *FileSystemClient) WriteObject(secretName string, secret interface{}) (map[string]interface{}, error) { + path := c.fileName(secretName) + dir, _ := filepath.Split(path) + err := os.MkdirAll(dir, util.DefaultWritePermissions) + if err != nil { + return nil, errors.Wrapf(err, "failed to ensure that parent directory exists %s", dir) + } + err = helm.SaveFile(path, secret) + if err != nil { + return nil, err + } + return c.Read(secretName) +} + +// ReplaceURIs will replace any local: URIs in a string +func (c *FileSystemClient) ReplaceURIs(s string) (string, error) { + return secreturl.ReplaceURIs(s, c, localURIRegex, "local:") +} + +func (c *FileSystemClient) fileName(secretName string) string { + return filepath.Join(c.Dir, secretName+".yaml") +} diff --git a/pkg/secreturl/mocks/secreturl_client.go b/pkg/secreturl/mocks/secreturl_client.go new file mode 100644 index 0000000000..9073a1dd55 --- /dev/null +++ b/pkg/secreturl/mocks/secreturl_client.go @@ -0,0 +1,250 @@ +// Code generated by pegomock. DO NOT EDIT. +// Source: github.com/jenkins-x/jx/pkg/secreturl (interfaces: Client) + +package secreturl_test + +import ( + pegomock "github.com/petergtz/pegomock" + "reflect" + "time" +) + +type MockClient struct { + fail func(message string, callerSkip ...int) +} + +func NewMockClient(options ...pegomock.Option) *MockClient { + mock := &MockClient{} + for _, option := range options { + option.Apply(mock) + } + return mock +} + +func (mock *MockClient) SetFailHandler(fh pegomock.FailHandler) { mock.fail = fh } +func (mock *MockClient) FailHandler() pegomock.FailHandler { return mock.fail } + +func (mock *MockClient) Read(_param0 string) (map[string]interface{}, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{_param0} + result := pegomock.GetGenericMockFrom(mock).Invoke("Read", params, []reflect.Type{reflect.TypeOf((*map[string]interface{})(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 map[string]interface{} + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(map[string]interface{}) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + +func (mock *MockClient) ReadObject(_param0 string, _param1 interface{}) error { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{_param0, _param1} + result := pegomock.GetGenericMockFrom(mock).Invoke("ReadObject", params, []reflect.Type{reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(error) + } + } + return ret0 +} + +func (mock *MockClient) ReplaceURIs(_param0 string) (string, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{_param0} + result := pegomock.GetGenericMockFrom(mock).Invoke("ReplaceURIs", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 string + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(string) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + +func (mock *MockClient) WriteObject(_param0 string, _param1 interface{}) (map[string]interface{}, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{_param0, _param1} + result := pegomock.GetGenericMockFrom(mock).Invoke("WriteObject", params, []reflect.Type{reflect.TypeOf((*map[string]interface{})(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 map[string]interface{} + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(map[string]interface{}) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + +func (mock *MockClient) VerifyWasCalledOnce() *VerifierMockClient { + return &VerifierMockClient{ + mock: mock, + invocationCountMatcher: pegomock.Times(1), + } +} + +func (mock *MockClient) VerifyWasCalled(invocationCountMatcher pegomock.Matcher) *VerifierMockClient { + return &VerifierMockClient{ + mock: mock, + invocationCountMatcher: invocationCountMatcher, + } +} + +func (mock *MockClient) VerifyWasCalledInOrder(invocationCountMatcher pegomock.Matcher, inOrderContext *pegomock.InOrderContext) *VerifierMockClient { + return &VerifierMockClient{ + mock: mock, + invocationCountMatcher: invocationCountMatcher, + inOrderContext: inOrderContext, + } +} + +func (mock *MockClient) VerifyWasCalledEventually(invocationCountMatcher pegomock.Matcher, timeout time.Duration) *VerifierMockClient { + return &VerifierMockClient{ + mock: mock, + invocationCountMatcher: invocationCountMatcher, + timeout: timeout, + } +} + +type VerifierMockClient struct { + mock *MockClient + invocationCountMatcher pegomock.Matcher + inOrderContext *pegomock.InOrderContext + timeout time.Duration +} + +func (verifier *VerifierMockClient) Read(_param0 string) *MockClient_Read_OngoingVerification { + params := []pegomock.Param{_param0} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Read", params, verifier.timeout) + return &MockClient_Read_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_Read_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_Read_OngoingVerification) GetCapturedArguments() string { + _param0 := c.GetAllCapturedArguments() + return _param0[len(_param0)-1] +} + +func (c *MockClient_Read_OngoingVerification) GetAllCapturedArguments() (_param0 []string) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]string, len(params[0])) + for u, param := range params[0] { + _param0[u] = param.(string) + } + } + return +} + +func (verifier *VerifierMockClient) ReadObject(_param0 string, _param1 interface{}) *MockClient_ReadObject_OngoingVerification { + params := []pegomock.Param{_param0, _param1} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ReadObject", params, verifier.timeout) + return &MockClient_ReadObject_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_ReadObject_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_ReadObject_OngoingVerification) GetCapturedArguments() (string, interface{}) { + _param0, _param1 := c.GetAllCapturedArguments() + return _param0[len(_param0)-1], _param1[len(_param1)-1] +} + +func (c *MockClient_ReadObject_OngoingVerification) GetAllCapturedArguments() (_param0 []string, _param1 []interface{}) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]string, len(params[0])) + for u, param := range params[0] { + _param0[u] = param.(string) + } + _param1 = make([]interface{}, len(params[1])) + for u, param := range params[1] { + _param1[u] = param.(interface{}) + } + } + return +} + +func (verifier *VerifierMockClient) ReplaceURIs(_param0 string) *MockClient_ReplaceURIs_OngoingVerification { + params := []pegomock.Param{_param0} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ReplaceURIs", params, verifier.timeout) + return &MockClient_ReplaceURIs_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_ReplaceURIs_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_ReplaceURIs_OngoingVerification) GetCapturedArguments() string { + _param0 := c.GetAllCapturedArguments() + return _param0[len(_param0)-1] +} + +func (c *MockClient_ReplaceURIs_OngoingVerification) GetAllCapturedArguments() (_param0 []string) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]string, len(params[0])) + for u, param := range params[0] { + _param0[u] = param.(string) + } + } + return +} + +func (verifier *VerifierMockClient) WriteObject(_param0 string, _param1 interface{}) *MockClient_WriteObject_OngoingVerification { + params := []pegomock.Param{_param0, _param1} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "WriteObject", params, verifier.timeout) + return &MockClient_WriteObject_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_WriteObject_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_WriteObject_OngoingVerification) GetCapturedArguments() (string, interface{}) { + _param0, _param1 := c.GetAllCapturedArguments() + return _param0[len(_param0)-1], _param1[len(_param1)-1] +} + +func (c *MockClient_WriteObject_OngoingVerification) GetAllCapturedArguments() (_param0 []string, _param1 []interface{}) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]string, len(params[0])) + for u, param := range params[0] { + _param0[u] = param.(string) + } + _param1 = make([]interface{}, len(params[1])) + for u, param := range params[1] { + _param1[u] = param.(interface{}) + } + } + return +} diff --git a/pkg/util/dirs.go b/pkg/util/dirs.go index 7c9c83d600..4bef21d173 100644 --- a/pkg/util/dirs.go +++ b/pkg/util/dirs.go @@ -45,6 +45,15 @@ func ConfigDir() (string, error) { return path, nil } +// LocalFileSystemSecretsDir returns the default local file system secrets location for the file system alternative to vault +func LocalFileSystemSecretsDir(clusterName string) (string, error) { + home, err := ConfigDir() + if err != nil { + return "", err + } + return filepath.Join(home, "localSecrets", clusterName), nil +} + // KubeConfigFile gets the .kube/config file func KubeConfigFile() string { path := os.Getenv("KUBECONFIG") diff --git a/pkg/vault/helpers.go b/pkg/vault/helpers.go index d8b109fd22..6d5bf4b8b3 100644 --- a/pkg/vault/helpers.go +++ b/pkg/vault/helpers.go @@ -1,11 +1,8 @@ package vault import ( - "fmt" "io/ioutil" "path/filepath" - "regexp" - "strings" "github.com/jenkins-x/jx/pkg/config" "github.com/jenkins-x/jx/pkg/util" @@ -17,8 +14,6 @@ const ( passwordKey = "Password" ) -var vaultURIRegex = regexp.MustCompile(`vault:[-_\w\/:]*`) - // WriteYAMLFiles stores the given YAML files in vault. The final secret path is // a concatenation of the 'path' with the file name. func WriteYamlFiles(client Client, path string, files ...string) error { @@ -63,45 +58,3 @@ func WriteMap(client Client, path string, secret map[string]interface{}) error { } return nil } - -// ToURI constructs a vault: URI for the given path and key -func ToURI(path string, key string) string { - return fmt.Sprintf("vault:%s:%s", path, key) -} - -// ReplaceURIs will replace any vault: URIs in a string, using the vault client -func ReplaceURIs(s string, client Client) (string, error) { - var err error - answer := vaultURIRegex.ReplaceAllStringFunc(s, func(found string) string { - // Stop once we have an error - if err == nil { - pathAndKey := strings.Trim(strings.TrimPrefix(found, "vault:"), "\"") - parts := strings.Split(pathAndKey, ":") - if len(parts) != 2 { - err = errors.Errorf("cannot parse %s as path:key", pathAndKey) - return "" - } - secret, err1 := client.Read(parts[0]) - if err1 != nil { - err = errors.Wrapf(err1, "reading %s from vault", parts[0]) - return "" - } - if v, ok := secret[parts[1]]; !ok { - err = errors.Errorf("unable to find %s in secret at %s", parts[1], parts[0]) - return "" - } else { - result, err1 := util.AsString(v) - if err1 != nil { - err = errors.Wrapf(err1, "converting %v to string", v) - return "" - } - return result - } - } - return found - }) - if err != nil { - return "", errors.Wrapf(err, "replacing vault paths in %s", s) - } - return answer, nil -} diff --git a/pkg/vault/helpers_test.go b/pkg/vault/helpers_test.go index 9ca757b20d..1f79827377 100644 --- a/pkg/vault/helpers_test.go +++ b/pkg/vault/helpers_test.go @@ -7,8 +7,6 @@ import ( "github.com/pborman/uuid" - "github.com/jenkins-x/jx/pkg/vault" - "github.com/stretchr/testify/assert" "github.com/jenkins-x/jx/pkg/util" @@ -36,7 +34,10 @@ func TestReplaceURIs(t *testing.T) { pegomock.When(vaultClient.Read(pegomock.EqString(path))).ThenReturn(map[string]interface{}{ key: secret, }, nil) - result, err := vault.ReplaceURIs(valuesyaml, vaultClient) + pegomock.When(vaultClient.ReplaceURIs(pegomock.EqString(valuesyaml))).ThenReturn(fmt.Sprintf(`foo: + bar: %s +`, secret), nil) + result, err := vaultClient.ReplaceURIs(valuesyaml) assert.NoError(t, err) assert.NoError(t, err) assert.Equal(t, fmt.Sprintf(`foo: @@ -64,7 +65,10 @@ func TestReplaceRealExampleURI(t *testing.T) { pegomock.When(vaultClient.Read(pegomock.EqString(path))).ThenReturn(map[string]interface{}{ key: secret, }, nil) - result, err := vault.ReplaceURIs(valuesyaml, vaultClient) + pegomock.When(vaultClient.ReplaceURIs(pegomock.EqString(valuesyaml))).ThenReturn(fmt.Sprintf(`foo: + bar: %s +`, secret), nil) + result, err := vaultClient.ReplaceURIs(valuesyaml) assert.NoError(t, err) assert.NoError(t, err) assert.Equal(t, fmt.Sprintf(`foo: diff --git a/pkg/vault/mocks/vault_client.go b/pkg/vault/mocks/vault_client.go index 1b90ebdef8..cb028b90c5 100644 --- a/pkg/vault/mocks/vault_client.go +++ b/pkg/vault/mocks/vault_client.go @@ -120,6 +120,25 @@ func (mock *MockClient) ReadYaml(_param0 string) (string, error) { return ret0, ret1 } +func (mock *MockClient) ReplaceURIs(_param0 string) (string, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{_param0} + result := pegomock.GetGenericMockFrom(mock).Invoke("ReplaceURIs", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 string + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(string) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + func (mock *MockClient) Write(_param0 string, _param1 map[string]interface{}) (map[string]interface{}, error) { if mock == nil { panic("mock must not be nil. Use myMock := NewMockClient().") @@ -343,6 +362,33 @@ func (c *MockClient_ReadYaml_OngoingVerification) GetAllCapturedArguments() (_pa return } +func (verifier *VerifierMockClient) ReplaceURIs(_param0 string) *MockClient_ReplaceURIs_OngoingVerification { + params := []pegomock.Param{_param0} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ReplaceURIs", params, verifier.timeout) + return &MockClient_ReplaceURIs_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_ReplaceURIs_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_ReplaceURIs_OngoingVerification) GetCapturedArguments() string { + _param0 := c.GetAllCapturedArguments() + return _param0[len(_param0)-1] +} + +func (c *MockClient_ReplaceURIs_OngoingVerification) GetAllCapturedArguments() (_param0 []string) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]string, len(params[0])) + for u, param := range params[0] { + _param0[u] = param.(string) + } + } + return +} + func (verifier *VerifierMockClient) Write(_param0 string, _param1 map[string]interface{}) *MockClient_Write_OngoingVerification { params := []pegomock.Param{_param0, _param1} methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Write", params, verifier.timeout) diff --git a/pkg/vault/vault_client.go b/pkg/vault/vault_client.go index 93987ad736..d1efa92b20 100644 --- a/pkg/vault/vault_client.go +++ b/pkg/vault/vault_client.go @@ -4,8 +4,10 @@ import ( "encoding/base64" "fmt" "net/url" + "regexp" "github.com/hashicorp/vault/api" + "github.com/jenkins-x/jx/pkg/secreturl" "github.com/jenkins-x/jx/pkg/util" "github.com/pkg/errors" ) @@ -14,6 +16,8 @@ const ( yamlDataKey = "yaml" ) +var vaultURIRegex = regexp.MustCompile(`vault:[-_\w\/:]*`) + // Client is an interface for interacting with Vault //go:generate pegomock generate github.com/jenkins-x/jx/pkg/vault Client -o mocks/vault_client.go type Client interface { @@ -33,7 +37,7 @@ type Client interface { // Read reads a named secret from the vault Read(secretName string) (map[string]interface{}, error) - // ReadObject reads a generic named objec from vault. + // ReadObject reads a generic named object from vault. // The secret _must_ be serializable to JSON. ReadObject(secretName string, secret interface{}) error @@ -42,6 +46,9 @@ type Client interface { // Config gets the config required for configuring the official Vault CLI Config() (vaultURL url.URL, vaultToken string, err error) + + // ReplaceURIs will replace any vault: URIs in a string (or whatever URL scheme the secret URL client supports + ReplaceURIs(text string) (string, error) } // client is a hand wrapper around the official Vault API @@ -171,3 +178,8 @@ func (v *client) Config() (vaultURL url.URL, vaultToken string, err error) { parsed, err := url.Parse(v.client.Address()) return *parsed, v.client.Token(), err } + +// ReplaceURIs will replace any vault: URIs in a string (or whatever URL scheme the secret URL client supports +func (v *client) ReplaceURIs(s string) (string, error) { + return secreturl.ReplaceURIs(s, v, vaultURIRegex, "vault:") +}